import { describe, it, expect, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { resetLogger, setLoggerOverride } from "../logging/logger.js"; import { buildAllowedModelSet, inferUniqueProviderFromConfiguredModels, parseModelRef, buildModelAliasIndex, normalizeModelSelection, normalizeProviderId, normalizeProviderIdForAuth, modelKey, resolveAllowedModelRef, resolveConfiguredModelRef, resolveThinkingDefault, resolveModelRefFromString, } from "./model-selection.js"; const EXPLICIT_ALLOWLIST_CONFIG = { agents: { defaults: { model: { primary: "openai/gpt-5.2" }, models: { "anthropic/claude-sonnet-4-6": { alias: "sonnet" }, }, }, }, } as OpenClawConfig; const BUNDLED_ALLOWLIST_CATALOG = [ { provider: "anthropic", id: "claude-sonnet-4-5", name: "Claude Sonnet 4.5" }, { provider: "openai", id: "gpt-5.2", name: "gpt-5.2" }, ]; const ANTHROPIC_OPUS_CATALOG = [ { provider: "anthropic", id: "claude-opus-4-6", name: "Claude Opus 4.6", reasoning: true, }, ]; function resolveAnthropicOpusThinking(cfg: OpenClawConfig) { return resolveThinkingDefault({ cfg, provider: "anthropic", model: "claude-opus-4-6", catalog: ANTHROPIC_OPUS_CATALOG, }); } function createAgentFallbackConfig(params: { primary?: string; fallbacks?: string[]; agentFallbacks?: string[]; }) { return { agents: { defaults: { models: { "openai/gpt-4o": {}, }, model: { primary: params.primary ?? "openai/gpt-4o", fallbacks: params.fallbacks ?? [], }, }, ...(params.agentFallbacks ? { list: [ { id: "coder", model: { primary: params.primary ?? "openai/gpt-4o", fallbacks: params.agentFallbacks, }, }, ], } : {}), }, } as OpenClawConfig; } function createProviderWithModelsConfig(provider: string, models: Array>) { return { models: { providers: { [provider]: { baseUrl: `https://${provider}.example.com`, models, }, }, }, } as Partial; } function resolveConfiguredRefForTest(cfg: Partial) { return resolveConfiguredModelRef({ cfg: cfg as OpenClawConfig, defaultProvider: "anthropic", defaultModel: "claude-opus-4-6", }); } describe("model-selection", () => { describe("normalizeProviderId", () => { it("should normalize provider names", () => { expect(normalizeProviderId("Anthropic")).toBe("anthropic"); expect(normalizeProviderId("Z.ai")).toBe("zai"); expect(normalizeProviderId("z-ai")).toBe("zai"); expect(normalizeProviderId("OpenCode-Zen")).toBe("opencode"); expect(normalizeProviderId("qwen")).toBe("qwen-portal"); expect(normalizeProviderId("kimi-code")).toBe("kimi"); expect(normalizeProviderId("kimi-coding")).toBe("kimi"); expect(normalizeProviderId("bedrock")).toBe("amazon-bedrock"); expect(normalizeProviderId("aws-bedrock")).toBe("amazon-bedrock"); expect(normalizeProviderId("amazon-bedrock")).toBe("amazon-bedrock"); }); }); describe("normalizeProviderIdForAuth", () => { it("maps coding-plan variants to base provider for auth lookup", () => { expect(normalizeProviderIdForAuth("volcengine-plan")).toBe("volcengine"); expect(normalizeProviderIdForAuth("byteplus-plan")).toBe("byteplus"); expect(normalizeProviderIdForAuth("openai")).toBe("openai"); }); }); describe("modelKey", () => { it("keeps canonical OpenRouter native ids without duplicating the provider", () => { expect(modelKey("openrouter", "openrouter/hunter-alpha")).toBe("openrouter/hunter-alpha"); }); }); describe("parseModelRef", () => { const expectParsedModelVariants = ( variants: string[], defaultProvider: string, expected: { provider: string; model: string }, ) => { for (const raw of variants) { expect(parseModelRef(raw, defaultProvider), raw).toEqual(expected); } }; it.each([ { name: "parses explicit provider/model refs", variants: ["anthropic/claude-3-5-sonnet"], defaultProvider: "openai", expected: { provider: "anthropic", model: "claude-3-5-sonnet" }, }, { name: "uses the default provider when omitted", variants: ["claude-3-5-sonnet"], defaultProvider: "anthropic", expected: { provider: "anthropic", model: "claude-3-5-sonnet" }, }, { name: "preserves nested model ids after the provider prefix", variants: ["nvidia/moonshotai/kimi-k2.5"], defaultProvider: "anthropic", expected: { provider: "nvidia", model: "moonshotai/kimi-k2.5" }, }, { name: "normalizes anthropic shorthand aliases", variants: ["anthropic/opus-4.6", "opus-4.6", " anthropic / opus-4.6 "], defaultProvider: "anthropic", expected: { provider: "anthropic", model: "claude-opus-4-6" }, }, { name: "normalizes anthropic sonnet aliases", variants: ["anthropic/sonnet-4.6", "sonnet-4.6"], defaultProvider: "anthropic", expected: { provider: "anthropic", model: "claude-sonnet-4-6" }, }, { name: "keeps dated anthropic model ids unchanged", variants: ["anthropic/claude-sonnet-4-20250514", "claude-sonnet-4-20250514"], defaultProvider: "anthropic", expected: { provider: "anthropic", model: "claude-sonnet-4-20250514" }, }, { name: "normalizes deprecated google flash preview ids", variants: ["google/gemini-3.1-flash-preview", "gemini-3.1-flash-preview"], defaultProvider: "google", expected: { provider: "google", model: "gemini-3-flash-preview" }, }, { name: "normalizes gemini 3.1 flash-lite ids", variants: ["google/gemini-3.1-flash-lite", "gemini-3.1-flash-lite"], defaultProvider: "google", expected: { provider: "google", model: "gemini-3.1-flash-lite-preview" }, }, { name: "keeps OpenAI codex refs on the openai provider", variants: ["openai/gpt-5.3-codex", "gpt-5.3-codex"], defaultProvider: "openai", expected: { provider: "openai", model: "gpt-5.3-codex" }, }, { name: "preserves openrouter native model prefixes", variants: ["openrouter/aurora-alpha"], defaultProvider: "openai", expected: { provider: "openrouter", model: "openrouter/aurora-alpha" }, }, { name: "passes through openrouter upstream provider ids", variants: ["openrouter/anthropic/claude-sonnet-4-5"], defaultProvider: "openai", expected: { provider: "openrouter", model: "anthropic/claude-sonnet-4-5" }, }, { name: "normalizes Vercel Claude shorthand to anthropic-prefixed model ids", variants: ["vercel-ai-gateway/claude-opus-4.6"], defaultProvider: "openai", expected: { provider: "vercel-ai-gateway", model: "anthropic/claude-opus-4.6" }, }, { name: "normalizes Vercel Anthropic aliases without double-prefixing", variants: ["vercel-ai-gateway/opus-4.6"], defaultProvider: "openai", expected: { provider: "vercel-ai-gateway", model: "anthropic/claude-opus-4-6" }, }, { name: "keeps already-prefixed Vercel Anthropic models unchanged", variants: ["vercel-ai-gateway/anthropic/claude-opus-4.6"], defaultProvider: "openai", expected: { provider: "vercel-ai-gateway", model: "anthropic/claude-opus-4.6" }, }, { name: "passes through non-Claude Vercel model ids unchanged", variants: ["vercel-ai-gateway/openai/gpt-5.2"], defaultProvider: "openai", expected: { provider: "vercel-ai-gateway", model: "openai/gpt-5.2" }, }, { name: "keeps already-suffixed codex variants unchanged", variants: ["openai/gpt-5.3-codex-codex"], defaultProvider: "anthropic", expected: { provider: "openai", model: "gpt-5.3-codex-codex" }, }, { name: "normalizes gemini 3.1 flash-lite ids for google-vertex", variants: ["google-vertex/gemini-3.1-flash-lite", "gemini-3.1-flash-lite"], defaultProvider: "google-vertex", expected: { provider: "google-vertex", model: "gemini-3.1-flash-lite-preview" }, }, ])("$name", ({ variants, defaultProvider, expected }) => { expectParsedModelVariants(variants, defaultProvider, expected); }); it("round-trips normalized refs through modelKey", () => { const parsed = parseModelRef(" opus-4.6 ", "anthropic"); expect(parsed).toEqual({ provider: "anthropic", model: "claude-opus-4-6" }); expect(modelKey(parsed?.provider ?? "", parsed?.model ?? "")).toBe( "anthropic/claude-opus-4-6", ); }); it.each(["", " ", "/", "anthropic/", "/model"])("returns null for invalid ref %j", (raw) => { expect(parseModelRef(raw, "anthropic")).toBeNull(); }); }); describe("inferUniqueProviderFromConfiguredModels", () => { it("infers provider when configured model match is unique", () => { const cfg = { agents: { defaults: { models: { "anthropic/claude-sonnet-4-6": {}, }, }, }, } as OpenClawConfig; expect( inferUniqueProviderFromConfiguredModels({ cfg, model: "claude-sonnet-4-6", }), ).toBe("anthropic"); }); it("returns undefined when configured matches are ambiguous", () => { const cfg = { agents: { defaults: { models: { "anthropic/claude-sonnet-4-6": {}, "minimax/claude-sonnet-4-6": {}, }, }, }, } as OpenClawConfig; expect( inferUniqueProviderFromConfiguredModels({ cfg, model: "claude-sonnet-4-6", }), ).toBeUndefined(); }); it("returns undefined for provider-prefixed model ids", () => { const cfg = { agents: { defaults: { models: { "anthropic/claude-sonnet-4-6": {}, }, }, }, } as OpenClawConfig; expect( inferUniqueProviderFromConfiguredModels({ cfg, model: "anthropic/claude-sonnet-4-6", }), ).toBeUndefined(); }); it("infers provider for slash-containing model id when allowlist match is unique", () => { const cfg = { agents: { defaults: { models: { "vercel-ai-gateway/anthropic/claude-sonnet-4-6": {}, }, }, }, } as OpenClawConfig; expect( inferUniqueProviderFromConfiguredModels({ cfg, model: "anthropic/claude-sonnet-4-6", }), ).toBe("vercel-ai-gateway"); }); }); describe("buildModelAliasIndex", () => { it("should build alias index from config", () => { const cfg: Partial = { agents: { defaults: { models: { "anthropic/claude-3-5-sonnet": { alias: "fast" }, "openai/gpt-4o": { alias: "smart" }, }, }, }, }; const index = buildModelAliasIndex({ cfg: cfg as OpenClawConfig, defaultProvider: "anthropic", }); expect(index.byAlias.get("fast")?.ref).toEqual({ provider: "anthropic", model: "claude-3-5-sonnet", }); expect(index.byAlias.get("smart")?.ref).toEqual({ provider: "openai", model: "gpt-4o" }); expect(index.byKey.get(modelKey("anthropic", "claude-3-5-sonnet"))).toEqual(["fast"]); }); }); describe("buildAllowedModelSet", () => { it("keeps explicitly allowlisted models even when missing from bundled catalog", () => { const result = buildAllowedModelSet({ cfg: EXPLICIT_ALLOWLIST_CONFIG, catalog: BUNDLED_ALLOWLIST_CATALOG, defaultProvider: "anthropic", }); expect(result.allowAny).toBe(false); expect(result.allowedKeys.has("anthropic/claude-sonnet-4-6")).toBe(true); expect(result.allowedCatalog).toEqual([ { provider: "anthropic", id: "claude-sonnet-4-6", name: "claude-sonnet-4-6" }, ]); }); it("includes fallback models in allowed set", () => { const cfg = createAgentFallbackConfig({ fallbacks: ["anthropic/claude-sonnet-4-6", "google/gemini-3-pro"], }); const result = buildAllowedModelSet({ cfg, catalog: [], defaultProvider: "openai", defaultModel: "gpt-4o", }); expect(result.allowedKeys.has("openai/gpt-4o")).toBe(true); expect(result.allowedKeys.has("anthropic/claude-sonnet-4-6")).toBe(true); expect(result.allowedKeys.has("google/gemini-3-pro-preview")).toBe(true); expect(result.allowAny).toBe(false); }); it("handles empty fallbacks gracefully", () => { const cfg = createAgentFallbackConfig({}); const result = buildAllowedModelSet({ cfg, catalog: [], defaultProvider: "openai", defaultModel: "gpt-4o", }); expect(result.allowedKeys.has("openai/gpt-4o")).toBe(true); expect(result.allowAny).toBe(false); }); it("prefers per-agent fallback overrides when agentId is provided", () => { const cfg = createAgentFallbackConfig({ fallbacks: ["google/gemini-3-pro"], agentFallbacks: ["anthropic/claude-sonnet-4-6"], }); const result = buildAllowedModelSet({ cfg, catalog: [], defaultProvider: "openai", defaultModel: "gpt-4o", agentId: "coder", }); expect(result.allowedKeys.has("openai/gpt-4o")).toBe(true); expect(result.allowedKeys.has("anthropic/claude-sonnet-4-6")).toBe(true); expect(result.allowedKeys.has("google/gemini-3-pro-preview")).toBe(false); expect(result.allowAny).toBe(false); }); }); describe("resolveAllowedModelRef", () => { it("accepts explicit allowlist refs absent from bundled catalog", () => { const result = resolveAllowedModelRef({ cfg: EXPLICIT_ALLOWLIST_CONFIG, catalog: BUNDLED_ALLOWLIST_CATALOG, raw: "anthropic/claude-sonnet-4-6", defaultProvider: "openai", defaultModel: "gpt-5.2", }); expect(result).toEqual({ key: "anthropic/claude-sonnet-4-6", ref: { provider: "anthropic", model: "claude-sonnet-4-6" }, }); }); it("strips trailing auth profile suffix before allowlist matching", () => { const cfg: OpenClawConfig = { agents: { defaults: { models: { "openai/@cf/openai/gpt-oss-20b": {}, }, }, }, } as OpenClawConfig; const result = resolveAllowedModelRef({ cfg, catalog: [], raw: "openai/@cf/openai/gpt-oss-20b@cf:default", defaultProvider: "anthropic", }); expect(result).toEqual({ key: "openai/@cf/openai/gpt-oss-20b", ref: { provider: "openai", model: "@cf/openai/gpt-oss-20b" }, }); }); }); describe("resolveModelRefFromString", () => { it("should resolve from string with alias", () => { const index = { byAlias: new Map([ ["fast", { alias: "fast", ref: { provider: "anthropic", model: "sonnet" } }], ]), byKey: new Map(), }; const resolved = resolveModelRefFromString({ raw: "fast", defaultProvider: "openai", aliasIndex: index, }); expect(resolved?.ref).toEqual({ provider: "anthropic", model: "sonnet" }); expect(resolved?.alias).toBe("fast"); }); it("should resolve direct ref if no alias match", () => { const resolved = resolveModelRefFromString({ raw: "openai/gpt-4", defaultProvider: "anthropic", }); expect(resolved?.ref).toEqual({ provider: "openai", model: "gpt-4" }); }); it("strips trailing profile suffix for simple model refs", () => { const resolved = resolveModelRefFromString({ raw: "gpt-5@myprofile", defaultProvider: "openai", }); expect(resolved?.ref).toEqual({ provider: "openai", model: "gpt-5" }); }); it("strips trailing profile suffix for provider/model refs", () => { const resolved = resolveModelRefFromString({ raw: "google/gemini-flash-latest@google:bevfresh", defaultProvider: "anthropic", }); expect(resolved?.ref).toEqual({ provider: "google", model: "gemini-flash-latest", }); }); it("preserves Cloudflare @cf model segments", () => { const resolved = resolveModelRefFromString({ raw: "openai/@cf/openai/gpt-oss-20b", defaultProvider: "anthropic", }); expect(resolved?.ref).toEqual({ provider: "openai", model: "@cf/openai/gpt-oss-20b", }); }); it("preserves OpenRouter @preset model segments", () => { const resolved = resolveModelRefFromString({ raw: "openrouter/@preset/kimi-2-5", defaultProvider: "anthropic", }); expect(resolved?.ref).toEqual({ provider: "openrouter", model: "@preset/kimi-2-5", }); }); it("splits trailing profile suffix after OpenRouter preset paths", () => { const resolved = resolveModelRefFromString({ raw: "openrouter/@preset/kimi-2-5@work", defaultProvider: "anthropic", }); expect(resolved?.ref).toEqual({ provider: "openrouter", model: "@preset/kimi-2-5", }); }); it("strips profile suffix before alias resolution", () => { const index = { byAlias: new Map([ ["kimi", { alias: "kimi", ref: { provider: "nvidia", model: "moonshotai/kimi-k2.5" } }], ]), byKey: new Map(), }; const resolved = resolveModelRefFromString({ raw: "kimi@nvidia:default", defaultProvider: "openai", aliasIndex: index, }); expect(resolved?.ref).toEqual({ provider: "nvidia", model: "moonshotai/kimi-k2.5", }); expect(resolved?.alias).toBe("kimi"); }); }); describe("resolveConfiguredModelRef", () => { it("should fall back to anthropic and warn if provider is missing for non-alias", () => { setLoggerOverride({ level: "silent", consoleLevel: "warn" }); const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); try { const cfg: Partial = { agents: { defaults: { model: { primary: "claude-3-5-sonnet" }, }, }, }; const result = resolveConfiguredModelRef({ cfg: cfg as OpenClawConfig, defaultProvider: "google", defaultModel: "gemini-pro", }); expect(result).toEqual({ provider: "anthropic", model: "claude-3-5-sonnet" }); expect(warnSpy).toHaveBeenCalledWith( expect.stringContaining('Falling back to "anthropic/claude-3-5-sonnet"'), ); } finally { setLoggerOverride(null); resetLogger(); } }); it("sanitizes control characters in providerless-model warnings", () => { setLoggerOverride({ level: "silent", consoleLevel: "warn" }); const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); try { const cfg: Partial = { agents: { defaults: { model: { primary: "\u001B[31mclaude-3-5-sonnet\nspoof" }, }, }, }; const result = resolveConfiguredModelRef({ cfg: cfg as OpenClawConfig, defaultProvider: "google", defaultModel: "gemini-pro", }); expect(result).toEqual({ provider: "anthropic", model: "\u001B[31mclaude-3-5-sonnet\nspoof", }); const warning = warnSpy.mock.calls[0]?.[0] as string; expect(warning).toContain('Falling back to "anthropic/claude-3-5-sonnet"'); expect(warning).not.toContain("\u001B"); expect(warning).not.toContain("\n"); } finally { warnSpy.mockRestore(); setLoggerOverride(null); resetLogger(); } }); it("should use default provider/model if config is empty", () => { const cfg: Partial = {}; const result = resolveConfiguredModelRef({ cfg: cfg as OpenClawConfig, defaultProvider: "openai", defaultModel: "gpt-4", }); expect(result).toEqual({ provider: "openai", model: "gpt-4" }); }); it("should prefer configured custom provider when default provider is not in models.providers", () => { const cfg = createProviderWithModelsConfig("n1n", [ { id: "gpt-5.4", name: "GPT 5.4", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 128000, maxTokens: 4096, }, ]); const result = resolveConfiguredRefForTest(cfg); expect(result).toEqual({ provider: "n1n", model: "gpt-5.4" }); }); it("should keep default provider when it is in models.providers", () => { const cfg = createProviderWithModelsConfig("anthropic", [ { id: "claude-opus-4-6", name: "Claude Opus 4.6", reasoning: true, input: ["text", "image"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 200000, maxTokens: 4096, }, ]); const result = resolveConfiguredRefForTest(cfg); expect(result).toEqual({ provider: "anthropic", model: "claude-opus-4-6" }); }); it("should fall back to hardcoded default when no custom providers have models", () => { const cfg = createProviderWithModelsConfig("empty-provider", []); const result = resolveConfiguredRefForTest(cfg); expect(result).toEqual({ provider: "anthropic", model: "claude-opus-4-6" }); }); it("should warn when specified model cannot be resolved and falls back to default", () => { setLoggerOverride({ level: "silent", consoleLevel: "warn" }); const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); try { const cfg: Partial = { agents: { defaults: { model: { primary: "openai/" }, }, }, }; const result = resolveConfiguredModelRef({ cfg: cfg as OpenClawConfig, defaultProvider: "anthropic", defaultModel: "claude-opus-4-6", }); expect(result).toEqual({ provider: "anthropic", model: "claude-opus-4-6" }); expect(warnSpy).toHaveBeenCalledWith( expect.stringContaining('Falling back to default "anthropic/claude-opus-4-6"'), ); } finally { warnSpy.mockRestore(); setLoggerOverride(null); resetLogger(); } }); }); describe("resolveThinkingDefault", () => { it("prefers per-model params.thinking over global thinkingDefault", () => { const cfg = { agents: { defaults: { thinkingDefault: "low", models: { "anthropic/claude-opus-4-6": { params: { thinking: "high" }, }, }, }, }, } as OpenClawConfig; expect(resolveAnthropicOpusThinking(cfg)).toBe("high"); }); it("accepts legacy duplicated OpenRouter keys for per-model thinking", () => { const cfg = { agents: { defaults: { models: { "openrouter/openrouter/hunter-alpha": { params: { thinking: "high" }, }, }, }, }, } as OpenClawConfig; expect( resolveThinkingDefault({ cfg, provider: "openrouter", model: "openrouter/hunter-alpha", }), ).toBe("high"); }); it("accepts per-model params.thinking=adaptive", () => { const cfg = { agents: { defaults: { models: { "anthropic/claude-opus-4-6": { params: { thinking: "adaptive" }, }, }, }, }, } as OpenClawConfig; expect(resolveAnthropicOpusThinking(cfg)).toBe("adaptive"); }); it("defaults Anthropic Claude 4.6 models to adaptive", () => { const cfg = {} as OpenClawConfig; expect(resolveAnthropicOpusThinking(cfg)).toBe("adaptive"); expect( resolveThinkingDefault({ cfg, provider: "amazon-bedrock", model: "us.anthropic.claude-sonnet-4-6-v1:0", catalog: [ { provider: "amazon-bedrock", id: "us.anthropic.claude-sonnet-4-6-v1:0", name: "Claude Sonnet 4.6", reasoning: true, }, ], }), ).toBe("adaptive"); }); }); }); describe("normalizeModelSelection", () => { it("returns trimmed string for string input", () => { expect(normalizeModelSelection("ollama/llama3.2:3b")).toBe("ollama/llama3.2:3b"); }); it("returns undefined for empty/whitespace string", () => { expect(normalizeModelSelection("")).toBeUndefined(); expect(normalizeModelSelection(" ")).toBeUndefined(); }); it("extracts primary from object", () => { expect(normalizeModelSelection({ primary: "google/gemini-2.5-flash" })).toBe( "google/gemini-2.5-flash", ); }); it("returns undefined for object without primary", () => { expect(normalizeModelSelection({ fallbacks: ["a"] })).toBeUndefined(); expect(normalizeModelSelection({})).toBeUndefined(); }); it("returns undefined for null/undefined/number", () => { expect(normalizeModelSelection(undefined)).toBeUndefined(); expect(normalizeModelSelection(null)).toBeUndefined(); expect(normalizeModelSelection(42)).toBeUndefined(); }); });