From 87c447ed46c355bb8c54c41324a5b5a63c0a61aa Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 13 Mar 2026 17:51:36 +0000 Subject: [PATCH] test: tighten failover classifier coverage --- ...dded-helpers.isbillingerrormessage.test.ts | 265 ++++++++++-------- 1 file changed, 143 insertions(+), 122 deletions(-) diff --git a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts index 3cbefadbce8..e8578c7feb2 100644 --- a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts +++ b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts @@ -45,98 +45,117 @@ const GROQ_TOO_MANY_REQUESTS_MESSAGE = const GROQ_SERVICE_UNAVAILABLE_MESSAGE = "503 Service Unavailable: The server is temporarily unable to handle the request due to overloading or maintenance."; // pragma: allowlist secret +function expectMessageMatches( + matcher: (message: string) => boolean, + samples: readonly string[], + expected: boolean, +) { + for (const sample of samples) { + expect(matcher(sample), sample).toBe(expected); + } +} + describe("isAuthPermanentErrorMessage", () => { - it("matches permanent auth failure patterns", () => { - const samples = [ - "invalid_api_key", - "api key revoked", - "api key deactivated", - "key has been disabled", - "key has been revoked", - "account has been deactivated", - "could not authenticate api key", - "could not validate credentials", - "API_KEY_REVOKED", - "api_key_deleted", - ]; - for (const sample of samples) { - expect(isAuthPermanentErrorMessage(sample)).toBe(true); - } - }); - it("does not match transient auth errors", () => { - const samples = [ - "unauthorized", - "invalid token", - "authentication failed", - "forbidden", - "access denied", - "token has expired", - ]; - for (const sample of samples) { - expect(isAuthPermanentErrorMessage(sample)).toBe(false); - } + it.each([ + { + name: "matches permanent auth failure patterns", + samples: [ + "invalid_api_key", + "api key revoked", + "api key deactivated", + "key has been disabled", + "key has been revoked", + "account has been deactivated", + "could not authenticate api key", + "could not validate credentials", + "API_KEY_REVOKED", + "api_key_deleted", + ], + expected: true, + }, + { + name: "does not match transient auth errors", + samples: [ + "unauthorized", + "invalid token", + "authentication failed", + "forbidden", + "access denied", + "token has expired", + ], + expected: false, + }, + ])("$name", ({ samples, expected }) => { + expectMessageMatches(isAuthPermanentErrorMessage, samples, expected); }); }); describe("isAuthErrorMessage", () => { - it("matches credential validation errors", () => { - const samples = [ - 'No credentials found for profile "anthropic:default".', - "No API key found for profile openai.", - ]; - for (const sample of samples) { - expect(isAuthErrorMessage(sample)).toBe(true); - } - }); - it("matches OAuth refresh failures", () => { - const samples = [ - "OAuth token refresh failed for anthropic: Failed to refresh OAuth token for anthropic. Please try again or re-authenticate.", - "Please re-authenticate to continue.", - ]; - for (const sample of samples) { - expect(isAuthErrorMessage(sample)).toBe(true); - } + it.each([ + 'No credentials found for profile "anthropic:default".', + "No API key found for profile openai.", + "OAuth token refresh failed for anthropic: Failed to refresh OAuth token for anthropic. Please try again or re-authenticate.", + "Please re-authenticate to continue.", + ])("matches auth errors for %j", (sample) => { + expect(isAuthErrorMessage(sample)).toBe(true); }); }); describe("isBillingErrorMessage", () => { - it("matches credit / payment failures", () => { - const samples = [ - "Your credit balance is too low to access the Anthropic API.", - "insufficient credits", - "Payment Required", - "HTTP 402 Payment Required", - "plans & billing", - // Venice returns "Insufficient USD or Diem balance" which has extra words - // between "insufficient" and "balance" - "Insufficient USD or Diem balance to complete request. Visit https://venice.ai/settings/api to add credits.", - // OpenRouter returns "requires more credits" for underfunded accounts - "This model requires more credits to use", - "This endpoint require more credits", - ]; - for (const sample of samples) { - expect(isBillingErrorMessage(sample)).toBe(true); - } - }); - it("does not false-positive on issue IDs or text containing 402", () => { - const falsePositives = [ - "Fixed issue CHE-402 in the latest release", - "See ticket #402 for details", - "ISSUE-402 has been resolved", - "Room 402 is available", - "Error code 403 was returned, not 402-related", - "The building at 402 Main Street", - "processed 402 records", - "402 items found in the database", - "port 402 is open", - "Use a 402 stainless bolt", - "Book a 402 room", - "There is a 402 near me", - ]; - for (const sample of falsePositives) { - expect(isBillingErrorMessage(sample)).toBe(false); - } + it.each([ + { + name: "matches credit and payment failures", + samples: [ + "Your credit balance is too low to access the Anthropic API.", + "insufficient credits", + "Payment Required", + "HTTP 402 Payment Required", + "plans & billing", + "Insufficient USD or Diem balance to complete request. Visit https://venice.ai/settings/api to add credits.", + "This model requires more credits to use", + "This endpoint require more credits", + ], + expected: true, + }, + { + name: "does not false-positive on issue ids and numeric references", + samples: [ + "Fixed issue CHE-402 in the latest release", + "See ticket #402 for details", + "ISSUE-402 has been resolved", + "Room 402 is available", + "Error code 403 was returned, not 402-related", + "The building at 402 Main Street", + "processed 402 records", + "402 items found in the database", + "port 402 is open", + "Use a 402 stainless bolt", + "Book a 402 room", + "There is a 402 near me", + ], + expected: false, + }, + { + name: "still matches real HTTP 402 billing errors", + samples: [ + "HTTP 402 Payment Required", + "status: 402", + "error code 402", + "http 402", + "status=402 payment required", + "got a 402 from the API", + "returned 402", + "received a 402 response", + '{"status":402,"type":"error"}', + '{"code":402,"message":"payment required"}', + '{"error":{"code":402,"message":"billing hard limit reached"}}', + ], + expected: true, + }, + ])("$name", ({ samples, expected }) => { + expectMessageMatches(isBillingErrorMessage, samples, expected); }); + it("does not false-positive on long assistant responses mentioning billing keywords", () => { // Simulate a multi-paragraph assistant response that mentions billing terms const longResponse = @@ -176,37 +195,27 @@ describe("isBillingErrorMessage", () => { expect(longNonError.length).toBeGreaterThan(512); expect(isBillingErrorMessage(longNonError)).toBe(false); }); - it("still matches real HTTP 402 billing errors", () => { - const realErrors = [ - "HTTP 402 Payment Required", - "status: 402", - "error code 402", - "http 402", - "status=402 payment required", - "got a 402 from the API", - "returned 402", - "received a 402 response", - '{"status":402,"type":"error"}', - '{"code":402,"message":"payment required"}', - '{"error":{"code":402,"message":"billing hard limit reached"}}', - ]; - for (const sample of realErrors) { - expect(isBillingErrorMessage(sample)).toBe(true); - } + + it("prefers billing when API-key and 402 hints both appear", () => { + const sample = + "402 Payment Required: The account associated with this API key has reached its maximum allowed monthly spending limit."; + expect(isBillingErrorMessage(sample)).toBe(true); + expect(classifyFailoverReason(sample)).toBe("billing"); }); }); describe("isCloudCodeAssistFormatError", () => { it("matches format errors", () => { - const samples = [ - "INVALID_REQUEST_ERROR: string should match pattern", - "messages.1.content.1.tool_use.id", - "tool_use.id should match pattern", - "invalid request format", - ]; - for (const sample of samples) { - expect(isCloudCodeAssistFormatError(sample)).toBe(true); - } + expectMessageMatches( + isCloudCodeAssistFormatError, + [ + "INVALID_REQUEST_ERROR: string should match pattern", + "messages.1.content.1.tool_use.id", + "tool_use.id should match pattern", + "invalid request format", + ], + true, + ); }); }); @@ -238,20 +247,24 @@ describe("isCloudflareOrHtmlErrorPage", () => { }); describe("isCompactionFailureError", () => { - it("matches compaction overflow failures", () => { - const samples = [ - 'Context overflow: Summarization failed: 400 {"message":"prompt is too long"}', - "auto-compaction failed due to context overflow", - "Compaction failed: prompt is too long", - "Summarization failed: context window exceeded for this request", - ]; - for (const sample of samples) { - expect(isCompactionFailureError(sample)).toBe(true); - } - }); - it("ignores non-compaction overflow errors", () => { - expect(isCompactionFailureError("Context overflow: prompt too large")).toBe(false); - expect(isCompactionFailureError("rate limit exceeded")).toBe(false); + it.each([ + { + name: "matches compaction overflow failures", + samples: [ + 'Context overflow: Summarization failed: 400 {"message":"prompt is too long"}', + "auto-compaction failed due to context overflow", + "Compaction failed: prompt is too long", + "Summarization failed: context window exceeded for this request", + ], + expected: true, + }, + { + name: "ignores non-compaction overflow errors", + samples: ["Context overflow: prompt too large", "rate limit exceeded"], + expected: false, + }, + ])("$name", ({ samples, expected }) => { + expectMessageMatches(isCompactionFailureError, samples, expected); }); }); @@ -506,6 +519,10 @@ describe("isTransientHttpError", () => { }); describe("classifyFailoverReasonFromHttpStatus", () => { + it("treats HTTP 401 permanent auth failures as auth_permanent", () => { + expect(classifyFailoverReasonFromHttpStatus(401, "invalid_api_key")).toBe("auth_permanent"); + }); + it("treats HTTP 422 as format error", () => { expect(classifyFailoverReasonFromHttpStatus(422)).toBe("format"); expect(classifyFailoverReasonFromHttpStatus(422, "check open ai req parameter error")).toBe( @@ -518,6 +535,10 @@ describe("classifyFailoverReasonFromHttpStatus", () => { expect(classifyFailoverReasonFromHttpStatus(422, "insufficient credits")).toBe("billing"); }); + it("treats HTTP 400 insufficient-quota payloads as billing instead of format", () => { + expect(classifyFailoverReasonFromHttpStatus(400, INSUFFICIENT_QUOTA_PAYLOAD)).toBe("billing"); + }); + it("treats HTTP 499 as transient for structured errors", () => { expect(classifyFailoverReasonFromHttpStatus(499)).toBe("timeout"); expect(classifyFailoverReasonFromHttpStatus(499, "499 Client Closed Request")).toBe("timeout");