diff --git a/src/agents/models.profiles.live.test.ts b/src/agents/models.profiles.live.test.ts index 40b44a78857..c7ec61e0a65 100644 --- a/src/agents/models.profiles.live.test.ts +++ b/src/agents/models.profiles.live.test.ts @@ -192,6 +192,15 @@ function isProviderUnavailableErrorMessage(raw: string): boolean { ); } +function isOllamaUnavailableErrorMessage(raw: string): boolean { + const msg = raw.toLowerCase(); + return ( + msg.includes("ollama could not be reached") || + (msg.includes("127.0.0.1:11434") && msg.includes("econnrefused")) || + (msg.includes("localhost:11434") && msg.includes("econnrefused")) + ); +} + function toInt(value: string | undefined, fallback: number): number { const trimmed = value?.trim(); if (!trimmed) { @@ -766,6 +775,15 @@ describeLive("live models (profile keys)", () => { logProgress(`${progressLabel}: skip (provider unavailable)`); break; } + if ( + allowNotFoundSkip && + model.provider === "ollama" && + isOllamaUnavailableErrorMessage(message) + ) { + skipped.push({ model: id, reason: message }); + logProgress(`${progressLabel}: skip (ollama unavailable)`); + break; + } logProgress(`${progressLabel}: failed`); failures.push({ model: id, error: message }); break; diff --git a/src/gateway/gateway-models.profiles.live.test.ts b/src/gateway/gateway-models.profiles.live.test.ts index 3b636d5a3b7..c1bde246dc4 100644 --- a/src/gateway/gateway-models.profiles.live.test.ts +++ b/src/gateway/gateway-models.profiles.live.test.ts @@ -465,6 +465,15 @@ function isProviderUnavailableErrorMessage(raw: string): boolean { ); } +function isOllamaUnavailableErrorMessage(raw: string): boolean { + const msg = raw.toLowerCase(); + return ( + msg.includes("ollama could not be reached") || + (msg.includes("127.0.0.1:11434") && msg.includes("econnrefused")) || + (msg.includes("localhost:11434") && msg.includes("econnrefused")) + ); +} + function isInstructionsRequiredError(error: string): boolean { return /instructions are required/i.test(error); } @@ -1584,10 +1593,12 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) { } // OpenAI Codex refresh tokens can become single-use; skip instead of failing all live tests. if (model.provider === "openai-codex" && isRefreshTokenReused(message)) { + skippedCount += 1; logProgress(`${progressLabel}: skip (codex refresh token reused)`); break; } if (model.provider === "openai-codex" && isChatGPTUsageLimitErrorMessage(message)) { + skippedCount += 1; logProgress(`${progressLabel}: skip (chatgpt usage limit)`); break; } @@ -1630,6 +1641,11 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) { logProgress(`${progressLabel}: skip (missing auth profile)`); break; } + if (model.provider === "ollama" && isOllamaUnavailableErrorMessage(message)) { + skippedCount += 1; + logProgress(`${progressLabel}: skip (ollama unavailable)`); + break; + } if (params.label.startsWith("minimax-")) { skippedCount += 1; logProgress(`${progressLabel}: skip (minimax endpoint error)`);