diff --git a/src/extension-host/cutover-inventory.md b/src/extension-host/cutover-inventory.md index 80b2428f676..088ae501794 100644 --- a/src/extension-host/cutover-inventory.md +++ b/src/extension-host/cutover-inventory.md @@ -39,6 +39,7 @@ This is an implementation checklist, not a future-design spec. | Loader module-export, config-validation, and memory-slot decisions | `src/plugins/loader.ts` | `src/extension-host/loader-runtime.ts` | `partial` | Module export resolution, export-metadata application, config validation, and early or final memory-slot decisions now delegate through host-owned loader-runtime helpers. | | Loader post-import planning and register execution | `src/plugins/loader.ts` | `src/extension-host/loader-register.ts` | `partial` | Definition application, post-import validation planning, and `register(...)` execution now delegate through host-owned loader-register helpers while preserving current plugin behavior. | | Loader per-candidate orchestration | `src/plugins/loader.ts` | `src/extension-host/loader-flow.ts` | `partial` | The per-candidate load flow now runs through a host-owned orchestrator that composes planning, import, runtime validation, register execution, and record-state helpers. | +| Loader top-level load orchestration | `src/plugins/loader.ts` | `src/extension-host/loader-orchestrator.ts` | `partial` | Cache hits, runtime creation, discovery, manifest loading, candidate ordering, candidate processing, and finalization now route through a host-owned loader orchestrator while `src/plugins/loader.ts` remains the compatibility facade. | | Loader record-state transitions | `src/plugins/loader.ts` | `src/extension-host/loader-state.ts` | `partial` | Disabled, error, validate-only, and registered plugin-record state transitions now delegate through host-owned loader-state helpers, including explicit compatibility `lifecycleState` mapping; a real lifecycle state machine still does not exist. | | Loader final cache, warning, and activation finalization | `src/plugins/loader.ts` | `src/extension-host/loader-finalize.ts` | `partial` | Cache writes, untracked-extension warnings, final memory-slot warnings, and registry activation now delegate through a host-owned loader-finalize helper; the lifecycle state machine is still pending. | | Channel lookup | `src/channels/plugins/index.ts`, `src/channels/plugins/registry-loader.ts`, `src/channels/registry.ts` | extension-host-backed registries plus kernel channel contracts | `partial` | Readers now consume the host-owned active registry, but writes still originate from plugin registration. | @@ -86,14 +87,14 @@ That pattern has been used for: - active registry ownership - normalized extension schema and resolved-extension records - static consumers such as skills, validation, auto-enable, and config baseline generation -- loader compatibility, cache control, initial candidate planning, entry-path import, policy, runtime decisions, post-import register flow, per-candidate orchestration, record-state transitions with explicit compatibility lifecycle mapping, and final cache plus activation finalization +- loader compatibility, cache control, initial candidate planning, entry-path import, policy, runtime decisions, post-import register flow, per-candidate orchestration, top-level load orchestration, record-state transitions with explicit compatibility lifecycle mapping, and final cache plus activation finalization ## Immediate Next Targets These are the next lowest-risk cutover steps: 1. Replace remaining static-only manifest-registry injections with resolved-extension registry inputs where practical. -2. Grow the compatibility `lifecycleState` mapping into an explicit lifecycle state machine and move any remaining activation-state or policy orchestration into `src/extension-host/*`. +2. Grow the compatibility `lifecycleState` mapping into an explicit lifecycle state machine and move the remaining activation-state and policy ownership into `src/extension-host/*`. 3. Introduce explicit host-owned registration surfaces for runtime writes, starting with the least-coupled registries. 4. Move minimal SDK compatibility and loader normalization into `src/extension-host/*` without breaking current `openclaw/plugin-sdk/*` loading. 5. Start the first pilot on `extensions/thread-ownership` only after the host-side registry and lifecycle seams are explicit. diff --git a/src/extension-host/loader-orchestrator.ts b/src/extension-host/loader-orchestrator.ts new file mode 100644 index 00000000000..0f0ffcd88ab --- /dev/null +++ b/src/extension-host/loader-orchestrator.ts @@ -0,0 +1,229 @@ +import { createJiti } from "jiti"; +import type { OpenClawConfig } from "../config/config.js"; +import { activateExtensionHostRegistry } from "../extension-host/activation.js"; +import { + buildExtensionHostRegistryCacheKey, + clearExtensionHostRegistryCache, + getCachedExtensionHostRegistry, + setCachedExtensionHostRegistry, +} from "../extension-host/loader-cache.js"; +import { finalizeExtensionHostRegistryLoad } from "../extension-host/loader-finalize.js"; +import { processExtensionHostPluginCandidate } from "../extension-host/loader-flow.js"; +import { + buildExtensionHostProvenanceIndex, + compareExtensionHostDuplicateCandidateOrder, + pushExtensionHostDiagnostics, + warnWhenExtensionAllowlistIsOpen, +} from "../extension-host/loader-policy.js"; +import type { GatewayRequestHandler } from "../gateway/server-methods/types.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; +import { clearPluginCommands } from "../plugins/commands.js"; +import { applyTestPluginDefaults, normalizePluginsConfig } from "../plugins/config-state.js"; +import { discoverOpenClawPlugins } from "../plugins/discovery.js"; +import { loadPluginManifestRegistry } from "../plugins/manifest-registry.js"; +import { + createPluginRegistry, + type PluginRecord, + type PluginRegistry, +} from "../plugins/registry.js"; +import { createPluginRuntime, type CreatePluginRuntimeOptions } from "../plugins/runtime/index.js"; +import type { PluginRuntime } from "../plugins/runtime/types.js"; +import type { OpenClawPluginModule, PluginLogger } from "../plugins/types.js"; +import { resolvePluginSdkAlias, resolvePluginSdkScopedAliasMap } from "./loader-compat.js"; + +export type ExtensionHostPluginLoadOptions = { + config?: OpenClawConfig; + workspaceDir?: string; + env?: NodeJS.ProcessEnv; + logger?: PluginLogger; + coreGatewayHandlers?: Record; + runtimeOptions?: CreatePluginRuntimeOptions; + cache?: boolean; + mode?: "full" | "validate"; +}; + +const openAllowlistWarningCache = new Set(); + +const defaultLogger = () => createSubsystemLogger("plugins"); + +export function clearExtensionHostLoaderState(): void { + clearExtensionHostRegistryCache(); + openAllowlistWarningCache.clear(); +} + +export function loadExtensionHostPluginRegistry( + options: ExtensionHostPluginLoadOptions = {}, +): PluginRegistry { + const env = options.env ?? process.env; + // Test env: default-disable plugins unless explicitly configured. + // This keeps unit/gateway suites fast and avoids loading heavyweight plugin deps by accident. + const cfg = applyTestPluginDefaults(options.config ?? {}, env); + const logger = options.logger ?? defaultLogger(); + const validateOnly = options.mode === "validate"; + const normalized = normalizePluginsConfig(cfg.plugins); + const cacheKey = buildExtensionHostRegistryCacheKey({ + workspaceDir: options.workspaceDir, + plugins: normalized, + installs: cfg.plugins?.installs, + env, + }); + const cacheEnabled = options.cache !== false; + if (cacheEnabled) { + const cached = getCachedExtensionHostRegistry(cacheKey); + if (cached) { + activateExtensionHostRegistry(cached, cacheKey); + return cached; + } + } + + // Clear previously registered plugin commands before reloading. + clearPluginCommands(); + + // Lazily initialize the runtime so startup paths that discover/skip plugins do + // not eagerly load every channel runtime dependency. + let resolvedRuntime: PluginRuntime | null = null; + const resolveRuntime = (): PluginRuntime => { + resolvedRuntime ??= createPluginRuntime(options.runtimeOptions); + return resolvedRuntime; + }; + const runtime = new Proxy({} as PluginRuntime, { + get(_target, prop, receiver) { + return Reflect.get(resolveRuntime(), prop, receiver); + }, + set(_target, prop, value, receiver) { + return Reflect.set(resolveRuntime(), prop, value, receiver); + }, + has(_target, prop) { + return Reflect.has(resolveRuntime(), prop); + }, + ownKeys() { + return Reflect.ownKeys(resolveRuntime() as object); + }, + getOwnPropertyDescriptor(_target, prop) { + return Reflect.getOwnPropertyDescriptor(resolveRuntime() as object, prop); + }, + defineProperty(_target, prop, attributes) { + return Reflect.defineProperty(resolveRuntime() as object, prop, attributes); + }, + deleteProperty(_target, prop) { + return Reflect.deleteProperty(resolveRuntime() as object, prop); + }, + getPrototypeOf() { + return Reflect.getPrototypeOf(resolveRuntime() as object); + }, + }); + const { registry, createApi } = createPluginRegistry({ + logger, + runtime, + coreGatewayHandlers: options.coreGatewayHandlers as Record, + }); + + const discovery = discoverOpenClawPlugins({ + workspaceDir: options.workspaceDir, + extraPaths: normalized.loadPaths, + cache: options.cache, + env, + }); + const manifestRegistry = loadPluginManifestRegistry({ + config: cfg, + workspaceDir: options.workspaceDir, + cache: options.cache, + env, + candidates: discovery.candidates, + diagnostics: discovery.diagnostics, + }); + pushExtensionHostDiagnostics(registry.diagnostics, manifestRegistry.diagnostics); + warnWhenExtensionAllowlistIsOpen({ + logger, + pluginsEnabled: normalized.enabled, + allow: normalized.allow, + warningCacheKey: cacheKey, + warningCache: openAllowlistWarningCache, + discoverablePlugins: manifestRegistry.plugins.map((plugin) => ({ + id: plugin.id, + source: plugin.source, + origin: plugin.origin, + })), + }); + const provenance = buildExtensionHostProvenanceIndex({ + config: cfg, + normalizedLoadPaths: normalized.loadPaths, + env, + }); + + // Lazy: avoid creating the Jiti loader when all plugins are disabled (common in unit tests). + let jitiLoader: ReturnType | null = null; + const getJiti = () => { + if (jitiLoader) { + return jitiLoader; + } + const pluginSdkAlias = resolvePluginSdkAlias(); + const aliasMap = { + ...(pluginSdkAlias ? { "openclaw/plugin-sdk": pluginSdkAlias } : {}), + ...resolvePluginSdkScopedAliasMap(), + }; + jitiLoader = createJiti(import.meta.url, { + interopDefault: true, + extensions: [".ts", ".tsx", ".mts", ".cts", ".mtsx", ".ctsx", ".js", ".mjs", ".cjs", ".json"], + ...(Object.keys(aliasMap).length > 0 + ? { + alias: aliasMap, + } + : {}), + }); + return jitiLoader; + }; + + const manifestByRoot = new Map( + manifestRegistry.plugins.map((record) => [record.rootDir, record]), + ); + const orderedCandidates = [...discovery.candidates].toSorted((left, right) => { + return compareExtensionHostDuplicateCandidateOrder({ + left, + right, + manifestByRoot, + provenance, + env, + }); + }); + + const seenIds = new Map(); + const memorySlot = normalized.slots.memory; + let selectedMemoryPluginId: string | null = null; + let memorySlotMatched = false; + + for (const candidate of orderedCandidates) { + const manifestRecord = manifestByRoot.get(candidate.rootDir); + if (!manifestRecord) { + continue; + } + const processed = processExtensionHostPluginCandidate({ + candidate, + manifestRecord, + normalizedConfig: normalized, + rootConfig: cfg, + validateOnly, + logger, + registry, + seenIds, + selectedMemoryPluginId, + createApi, + loadModule: (safeSource) => getJiti()(safeSource) as OpenClawPluginModule, + }); + selectedMemoryPluginId = processed.selectedMemoryPluginId; + memorySlotMatched ||= processed.memorySlotMatched; + } + + return finalizeExtensionHostRegistryLoad({ + registry, + memorySlot, + memorySlotMatched, + provenance, + logger, + env, + cacheEnabled, + cacheKey, + setCachedRegistry: setCachedExtensionHostRegistry, + activateRegistry: activateExtensionHostRegistry, + }); +} diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 776c57ed2f8..332cd036fa3 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -1,64 +1,25 @@ -import { createJiti } from "jiti"; -import type { OpenClawConfig } from "../config/config.js"; -import { activateExtensionHostRegistry } from "../extension-host/activation.js"; -import { - buildExtensionHostRegistryCacheKey, - clearExtensionHostRegistryCache, - getCachedExtensionHostRegistry, - MAX_EXTENSION_HOST_REGISTRY_CACHE_ENTRIES, - setCachedExtensionHostRegistry, -} from "../extension-host/loader-cache.js"; import { listPluginSdkAliasCandidates, listPluginSdkExportedSubpaths, - resolvePluginSdkAlias, resolvePluginSdkAliasCandidateOrder, resolvePluginSdkAliasFile, - resolvePluginSdkScopedAliasMap, } from "../extension-host/loader-compat.js"; -import { finalizeExtensionHostRegistryLoad } from "../extension-host/loader-finalize.js"; -import { processExtensionHostPluginCandidate } from "../extension-host/loader-flow.js"; import { - buildExtensionHostProvenanceIndex, - compareExtensionHostDuplicateCandidateOrder, - pushExtensionHostDiagnostics, - warnWhenExtensionAllowlistIsOpen, -} from "../extension-host/loader-policy.js"; -import type { GatewayRequestHandler } from "../gateway/server-methods/types.js"; -import { createSubsystemLogger } from "../logging/subsystem.js"; -import { clearPluginCommands } from "./commands.js"; -import { applyTestPluginDefaults, normalizePluginsConfig } from "./config-state.js"; -import { discoverOpenClawPlugins } from "./discovery.js"; -import { loadPluginManifestRegistry } from "./manifest-registry.js"; -import { createPluginRegistry, type PluginRecord, type PluginRegistry } from "./registry.js"; -import { createPluginRuntime, type CreatePluginRuntimeOptions } from "./runtime/index.js"; -import type { PluginRuntime } from "./runtime/types.js"; -import type { OpenClawPluginModule, PluginLogger } from "./types.js"; + clearExtensionHostLoaderState, + type ExtensionHostPluginLoadOptions, + loadExtensionHostPluginRegistry, + MAX_EXTENSION_HOST_REGISTRY_CACHE_ENTRIES, +} from "../extension-host/loader-orchestrator.js"; +import type { PluginRegistry } from "./registry.js"; export type PluginLoadResult = PluginRegistry; -export type PluginLoadOptions = { - config?: OpenClawConfig; - workspaceDir?: string; - // Allows callers to resolve plugin roots and load paths against an explicit env - // instead of the process-global environment. - env?: NodeJS.ProcessEnv; - logger?: PluginLogger; - coreGatewayHandlers?: Record; - runtimeOptions?: CreatePluginRuntimeOptions; - cache?: boolean; - mode?: "full" | "validate"; -}; - -const openAllowlistWarningCache = new Set(); +export type PluginLoadOptions = ExtensionHostPluginLoadOptions; export function clearPluginLoaderCache(): void { - clearExtensionHostRegistryCache(); - openAllowlistWarningCache.clear(); + clearExtensionHostLoaderState(); } -const defaultLogger = () => createSubsystemLogger("plugins"); - export const __testing = { listPluginSdkAliasCandidates, listPluginSdkExportedSubpaths, @@ -68,176 +29,5 @@ export const __testing = { }; export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegistry { - const env = options.env ?? process.env; - // Test env: default-disable plugins unless explicitly configured. - // This keeps unit/gateway suites fast and avoids loading heavyweight plugin deps by accident. - const cfg = applyTestPluginDefaults(options.config ?? {}, env); - const logger = options.logger ?? defaultLogger(); - const validateOnly = options.mode === "validate"; - const normalized = normalizePluginsConfig(cfg.plugins); - const cacheKey = buildExtensionHostRegistryCacheKey({ - workspaceDir: options.workspaceDir, - plugins: normalized, - installs: cfg.plugins?.installs, - env, - }); - const cacheEnabled = options.cache !== false; - if (cacheEnabled) { - const cached = getCachedExtensionHostRegistry(cacheKey); - if (cached) { - activateExtensionHostRegistry(cached, cacheKey); - return cached; - } - } - - // Clear previously registered plugin commands before reloading - clearPluginCommands(); - - // Lazily initialize the runtime so startup paths that discover/skip plugins do - // not eagerly load every channel runtime dependency. - let resolvedRuntime: PluginRuntime | null = null; - const resolveRuntime = (): PluginRuntime => { - resolvedRuntime ??= createPluginRuntime(options.runtimeOptions); - return resolvedRuntime; - }; - const runtime = new Proxy({} as PluginRuntime, { - get(_target, prop, receiver) { - return Reflect.get(resolveRuntime(), prop, receiver); - }, - set(_target, prop, value, receiver) { - return Reflect.set(resolveRuntime(), prop, value, receiver); - }, - has(_target, prop) { - return Reflect.has(resolveRuntime(), prop); - }, - ownKeys() { - return Reflect.ownKeys(resolveRuntime() as object); - }, - getOwnPropertyDescriptor(_target, prop) { - return Reflect.getOwnPropertyDescriptor(resolveRuntime() as object, prop); - }, - defineProperty(_target, prop, attributes) { - return Reflect.defineProperty(resolveRuntime() as object, prop, attributes); - }, - deleteProperty(_target, prop) { - return Reflect.deleteProperty(resolveRuntime() as object, prop); - }, - getPrototypeOf() { - return Reflect.getPrototypeOf(resolveRuntime() as object); - }, - }); - const { registry, createApi } = createPluginRegistry({ - logger, - runtime, - coreGatewayHandlers: options.coreGatewayHandlers as Record, - }); - - const discovery = discoverOpenClawPlugins({ - workspaceDir: options.workspaceDir, - extraPaths: normalized.loadPaths, - cache: options.cache, - env, - }); - const manifestRegistry = loadPluginManifestRegistry({ - config: cfg, - workspaceDir: options.workspaceDir, - cache: options.cache, - env, - candidates: discovery.candidates, - diagnostics: discovery.diagnostics, - }); - pushExtensionHostDiagnostics(registry.diagnostics, manifestRegistry.diagnostics); - warnWhenExtensionAllowlistIsOpen({ - logger, - pluginsEnabled: normalized.enabled, - allow: normalized.allow, - warningCacheKey: cacheKey, - warningCache: openAllowlistWarningCache, - discoverablePlugins: manifestRegistry.plugins.map((plugin) => ({ - id: plugin.id, - source: plugin.source, - origin: plugin.origin, - })), - }); - const provenance = buildExtensionHostProvenanceIndex({ - config: cfg, - normalizedLoadPaths: normalized.loadPaths, - env, - }); - - // Lazy: avoid creating the Jiti loader when all plugins are disabled (common in unit tests). - let jitiLoader: ReturnType | null = null; - const getJiti = () => { - if (jitiLoader) { - return jitiLoader; - } - const pluginSdkAlias = resolvePluginSdkAlias(); - const aliasMap = { - ...(pluginSdkAlias ? { "openclaw/plugin-sdk": pluginSdkAlias } : {}), - ...resolvePluginSdkScopedAliasMap(), - }; - jitiLoader = createJiti(import.meta.url, { - interopDefault: true, - extensions: [".ts", ".tsx", ".mts", ".cts", ".mtsx", ".ctsx", ".js", ".mjs", ".cjs", ".json"], - ...(Object.keys(aliasMap).length > 0 - ? { - alias: aliasMap, - } - : {}), - }); - return jitiLoader; - }; - - const manifestByRoot = new Map( - manifestRegistry.plugins.map((record) => [record.rootDir, record]), - ); - const orderedCandidates = [...discovery.candidates].toSorted((left, right) => { - return compareExtensionHostDuplicateCandidateOrder({ - left, - right, - manifestByRoot, - provenance, - env, - }); - }); - - const seenIds = new Map(); - const memorySlot = normalized.slots.memory; - let selectedMemoryPluginId: string | null = null; - let memorySlotMatched = false; - - for (const candidate of orderedCandidates) { - const manifestRecord = manifestByRoot.get(candidate.rootDir); - if (!manifestRecord) { - continue; - } - const processed = processExtensionHostPluginCandidate({ - candidate, - manifestRecord, - normalizedConfig: normalized, - rootConfig: cfg, - validateOnly, - logger, - registry, - seenIds, - selectedMemoryPluginId, - createApi, - loadModule: (safeSource) => getJiti()(safeSource) as OpenClawPluginModule, - }); - selectedMemoryPluginId = processed.selectedMemoryPluginId; - memorySlotMatched ||= processed.memorySlotMatched; - } - - return finalizeExtensionHostRegistryLoad({ - registry, - memorySlot, - memorySlotMatched, - provenance, - logger, - env, - cacheEnabled, - cacheKey, - setCachedRegistry: setCachedExtensionHostRegistry, - activateRegistry: activateExtensionHostRegistry, - }); + return loadExtensionHostPluginRegistry(options); }