From 5194cf2019f43b9e017f11c8f756ace814581eff Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 28 Mar 2026 16:57:16 +0000 Subject: [PATCH] refactor: load bundled provider catalogs dynamically --- .../models-config.providers.static.test.ts | 25 +++ src/agents/models-config.providers.static.ts | 152 ++++++++++++++---- src/plugins/bundled-plugin-metadata.ts | 39 +++++ 3 files changed, 189 insertions(+), 27 deletions(-) create mode 100644 src/agents/models-config.providers.static.test.ts diff --git a/src/agents/models-config.providers.static.test.ts b/src/agents/models-config.providers.static.test.ts new file mode 100644 index 00000000000..0bbaacc75e6 --- /dev/null +++ b/src/agents/models-config.providers.static.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { + loadBundledProviderCatalogExportMap, + resolveBundledProviderCatalogEntries, +} from "./models-config.providers.static.js"; + +describe("models-config bundled provider catalogs", () => { + it("detects provider catalogs from plugin folders via metadata artifacts", () => { + const entries = resolveBundledProviderCatalogEntries(); + expect(entries.map((entry) => entry.dirName)).toEqual( + expect.arrayContaining(["openrouter", "volcengine"]), + ); + expect(entries.find((entry) => entry.dirName === "volcengine")).toMatchObject({ + dirName: "volcengine", + pluginId: "volcengine", + }); + }); + + it("loads provider catalog exports from detected plugin folders", async () => { + const exports = await loadBundledProviderCatalogExportMap(); + expect(exports.buildOpenrouterProvider).toBeTypeOf("function"); + expect(exports.buildDoubaoProvider).toBeTypeOf("function"); + expect(exports.buildDoubaoCodingProvider).toBeTypeOf("function"); + }); +}); diff --git a/src/agents/models-config.providers.static.ts b/src/agents/models-config.providers.static.ts index 25489f86d80..9d590bff14a 100644 --- a/src/agents/models-config.providers.static.ts +++ b/src/agents/models-config.providers.static.ts @@ -1,27 +1,125 @@ -export { - ANTHROPIC_VERTEX_DEFAULT_MODEL_ID, - MODELSTUDIO_BASE_URL, - MODELSTUDIO_DEFAULT_MODEL_ID, - QIANFAN_BASE_URL, - QIANFAN_DEFAULT_MODEL_ID, - XIAOMI_DEFAULT_MODEL_ID, - buildAnthropicVertexProvider, - buildBytePlusCodingProvider, - buildBytePlusProvider, - buildDeepSeekProvider, - buildDoubaoCodingProvider, - buildDoubaoProvider, - buildKilocodeProvider, - buildKimiCodingProvider, - buildMinimaxPortalProvider, - buildMinimaxProvider, - buildModelStudioProvider, - buildMoonshotProvider, - buildNvidiaProvider, - buildOpenAICodexProvider, - buildOpenrouterProvider, - buildQianfanProvider, - buildSyntheticProvider, - buildTogetherProvider, - buildXiaomiProvider, -} from "../plugin-sdk/provider-catalog.js"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + BUNDLED_PLUGIN_METADATA, + resolveBundledPluginPublicSurfacePath, +} from "../plugins/bundled-plugin-metadata.js"; + +const PROVIDER_CATALOG_ARTIFACT_BASENAME = "provider-catalog.js"; +const DEFAULT_PROVIDER_CATALOG_ROOT = path.resolve(import.meta.dirname, "../.."); + +export type BundledProviderCatalogEntry = { + dirName: string; + pluginId: string; + providers: readonly string[]; + artifactPath: string; +}; + +type ProviderCatalogModule = Record; +type ProviderCatalogExportMap = Record; + +let providerCatalogEntriesCache: ReadonlyArray | null = null; +let providerCatalogModulesPromise: Promise>> | null = + null; +let providerCatalogExportMapPromise: Promise> | null = null; + +export function resolveBundledProviderCatalogEntries(params?: { + rootDir?: string; +}): ReadonlyArray { + const rootDir = params?.rootDir ?? DEFAULT_PROVIDER_CATALOG_ROOT; + if (rootDir === DEFAULT_PROVIDER_CATALOG_ROOT && providerCatalogEntriesCache) { + return providerCatalogEntriesCache; + } + + const entries: BundledProviderCatalogEntry[] = []; + for (const entry of BUNDLED_PLUGIN_METADATA) { + if (!entry.publicSurfaceArtifacts?.includes(PROVIDER_CATALOG_ARTIFACT_BASENAME)) { + continue; + } + const artifactPath = resolveBundledPluginPublicSurfacePath({ + rootDir, + dirName: entry.dirName, + artifactBasename: PROVIDER_CATALOG_ARTIFACT_BASENAME, + }); + if (!artifactPath) { + continue; + } + entries.push({ + dirName: entry.dirName, + pluginId: entry.manifest.id, + providers: entry.manifest.providers ?? [], + artifactPath, + }); + } + entries.sort((left, right) => left.dirName.localeCompare(right.dirName)); + + if (rootDir === DEFAULT_PROVIDER_CATALOG_ROOT) { + providerCatalogEntriesCache = entries; + } + return entries; +} + +export async function loadBundledProviderCatalogModules(params?: { + rootDir?: string; +}): Promise>> { + const rootDir = params?.rootDir ?? DEFAULT_PROVIDER_CATALOG_ROOT; + if (rootDir === DEFAULT_PROVIDER_CATALOG_ROOT && providerCatalogModulesPromise) { + return providerCatalogModulesPromise; + } + + const loadPromise = (async () => { + const entries = resolveBundledProviderCatalogEntries({ rootDir }); + const modules = await Promise.all( + entries.map(async (entry) => { + const module = (await import( + pathToFileURL(entry.artifactPath).href + )) as ProviderCatalogModule; + return [entry.dirName, module] as const; + }), + ); + return Object.freeze(Object.fromEntries(modules)); + })(); + + if (rootDir === DEFAULT_PROVIDER_CATALOG_ROOT) { + providerCatalogModulesPromise = loadPromise; + } + return loadPromise; +} + +export async function loadBundledProviderCatalogExportMap(params?: { + rootDir?: string; +}): Promise> { + const rootDir = params?.rootDir ?? DEFAULT_PROVIDER_CATALOG_ROOT; + if (rootDir === DEFAULT_PROVIDER_CATALOG_ROOT && providerCatalogExportMapPromise) { + return providerCatalogExportMapPromise; + } + + const loadPromise = (async () => { + const modules = await loadBundledProviderCatalogModules({ rootDir }); + const exports: ProviderCatalogExportMap = {}; + const exportOwners = new Map(); + + for (const [dirName, module] of Object.entries(modules)) { + for (const [exportName, exportValue] of Object.entries(module)) { + if (exportName === "default") { + continue; + } + const existingOwner = exportOwners.get(exportName); + if (existingOwner && existingOwner !== dirName) { + throw new Error( + `Duplicate provider catalog export "${exportName}" from folders "${existingOwner}" and "${dirName}"`, + ); + } + exportOwners.set(exportName, dirName); + exports[exportName] = exportValue; + } + } + + return Object.freeze(exports); + })(); + + if (rootDir === DEFAULT_PROVIDER_CATALOG_ROOT) { + providerCatalogExportMapPromise = loadPromise; + } + return loadPromise; +} diff --git a/src/plugins/bundled-plugin-metadata.ts b/src/plugins/bundled-plugin-metadata.ts index 1db292781d8..da083d657ce 100644 --- a/src/plugins/bundled-plugin-metadata.ts +++ b/src/plugins/bundled-plugin-metadata.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { GENERATED_BUNDLED_PLUGIN_METADATA } from "./bundled-plugin-metadata.generated.js"; import type { PluginManifest, OpenClawPackageManifest } from "./manifest.js"; +const PUBLIC_SURFACE_SOURCE_EXTENSIONS = [".ts", ".mts", ".js", ".mjs", ".cts", ".cjs"] as const; + type GeneratedBundledPluginPathPair = { source: string; built: string; @@ -44,3 +46,40 @@ export function resolveBundledPluginGeneratedPath( } return null; } + +export function resolveBundledPluginPublicSurfacePath(params: { + rootDir: string; + dirName: string; + artifactBasename: string; +}): string | null { + const artifactBasename = params.artifactBasename.replace(/^\.\//u, ""); + if (!artifactBasename) { + return null; + } + + const builtCandidate = path.resolve( + params.rootDir, + "dist", + "extensions", + params.dirName, + artifactBasename, + ); + if (fs.existsSync(builtCandidate)) { + return builtCandidate; + } + + const sourceBaseName = artifactBasename.replace(/\.js$/u, ""); + for (const ext of PUBLIC_SURFACE_SOURCE_EXTENSIONS) { + const sourceCandidate = path.resolve( + params.rootDir, + "extensions", + params.dirName, + `${sourceBaseName}${ext}`, + ); + if (fs.existsSync(sourceCandidate)) { + return sourceCandidate; + } + } + + return null; +}