mirror of https://github.com/openclaw/openclaw.git
test: tighten failover classifier coverage
This commit is contained in:
Peter Steinberger
parent
f5b006f6a1
commit
87c447ed46
|
|
@ -45,98 +45,117 @@ const GROQ_TOO_MANY_REQUESTS_MESSAGE =
|
|||
const GROQ_SERVICE_UNAVAILABLE_MESSAGE =
|
||||
"503 Service Unavailable: The server is temporarily unable to handle the request due to overloading or maintenance."; // pragma: allowlist secret
|
||||
|
||||
function expectMessageMatches(
|
||||
matcher: (message: string) => boolean,
|
||||
samples: readonly string[],
|
||||
expected: boolean,
|
||||
) {
|
||||
for (const sample of samples) {
|
||||
expect(matcher(sample), sample).toBe(expected);
|
||||
}
|
||||
}
|
||||
|
||||
describe("isAuthPermanentErrorMessage", () => {
|
||||
it("matches permanent auth failure patterns", () => {
|
||||
const samples = [
|
||||
"invalid_api_key",
|
||||
"api key revoked",
|
||||
"api key deactivated",
|
||||
"key has been disabled",
|
||||
"key has been revoked",
|
||||
"account has been deactivated",
|
||||
"could not authenticate api key",
|
||||
"could not validate credentials",
|
||||
"API_KEY_REVOKED",
|
||||
"api_key_deleted",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isAuthPermanentErrorMessage(sample)).toBe(true);
|
||||
}
|
||||
});
|
||||
it("does not match transient auth errors", () => {
|
||||
const samples = [
|
||||
"unauthorized",
|
||||
"invalid token",
|
||||
"authentication failed",
|
||||
"forbidden",
|
||||
"access denied",
|
||||
"token has expired",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isAuthPermanentErrorMessage(sample)).toBe(false);
|
||||
}
|
||||
it.each([
|
||||
{
|
||||
name: "matches permanent auth failure patterns",
|
||||
samples: [
|
||||
"invalid_api_key",
|
||||
"api key revoked",
|
||||
"api key deactivated",
|
||||
"key has been disabled",
|
||||
"key has been revoked",
|
||||
"account has been deactivated",
|
||||
"could not authenticate api key",
|
||||
"could not validate credentials",
|
||||
"API_KEY_REVOKED",
|
||||
"api_key_deleted",
|
||||
],
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "does not match transient auth errors",
|
||||
samples: [
|
||||
"unauthorized",
|
||||
"invalid token",
|
||||
"authentication failed",
|
||||
"forbidden",
|
||||
"access denied",
|
||||
"token has expired",
|
||||
],
|
||||
expected: false,
|
||||
},
|
||||
])("$name", ({ samples, expected }) => {
|
||||
expectMessageMatches(isAuthPermanentErrorMessage, samples, expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isAuthErrorMessage", () => {
|
||||
it("matches credential validation errors", () => {
|
||||
const samples = [
|
||||
'No credentials found for profile "anthropic:default".',
|
||||
"No API key found for profile openai.",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isAuthErrorMessage(sample)).toBe(true);
|
||||
}
|
||||
});
|
||||
it("matches OAuth refresh failures", () => {
|
||||
const samples = [
|
||||
"OAuth token refresh failed for anthropic: Failed to refresh OAuth token for anthropic. Please try again or re-authenticate.",
|
||||
"Please re-authenticate to continue.",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isAuthErrorMessage(sample)).toBe(true);
|
||||
}
|
||||
it.each([
|
||||
'No credentials found for profile "anthropic:default".',
|
||||
"No API key found for profile openai.",
|
||||
"OAuth token refresh failed for anthropic: Failed to refresh OAuth token for anthropic. Please try again or re-authenticate.",
|
||||
"Please re-authenticate to continue.",
|
||||
])("matches auth errors for %j", (sample) => {
|
||||
expect(isAuthErrorMessage(sample)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isBillingErrorMessage", () => {
|
||||
it("matches credit / payment failures", () => {
|
||||
const samples = [
|
||||
"Your credit balance is too low to access the Anthropic API.",
|
||||
"insufficient credits",
|
||||
"Payment Required",
|
||||
"HTTP 402 Payment Required",
|
||||
"plans & billing",
|
||||
// Venice returns "Insufficient USD or Diem balance" which has extra words
|
||||
// between "insufficient" and "balance"
|
||||
"Insufficient USD or Diem balance to complete request. Visit https://venice.ai/settings/api to add credits.",
|
||||
// OpenRouter returns "requires more credits" for underfunded accounts
|
||||
"This model requires more credits to use",
|
||||
"This endpoint require more credits",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isBillingErrorMessage(sample)).toBe(true);
|
||||
}
|
||||
});
|
||||
it("does not false-positive on issue IDs or text containing 402", () => {
|
||||
const falsePositives = [
|
||||
"Fixed issue CHE-402 in the latest release",
|
||||
"See ticket #402 for details",
|
||||
"ISSUE-402 has been resolved",
|
||||
"Room 402 is available",
|
||||
"Error code 403 was returned, not 402-related",
|
||||
"The building at 402 Main Street",
|
||||
"processed 402 records",
|
||||
"402 items found in the database",
|
||||
"port 402 is open",
|
||||
"Use a 402 stainless bolt",
|
||||
"Book a 402 room",
|
||||
"There is a 402 near me",
|
||||
];
|
||||
for (const sample of falsePositives) {
|
||||
expect(isBillingErrorMessage(sample)).toBe(false);
|
||||
}
|
||||
it.each([
|
||||
{
|
||||
name: "matches credit and payment failures",
|
||||
samples: [
|
||||
"Your credit balance is too low to access the Anthropic API.",
|
||||
"insufficient credits",
|
||||
"Payment Required",
|
||||
"HTTP 402 Payment Required",
|
||||
"plans & billing",
|
||||
"Insufficient USD or Diem balance to complete request. Visit https://venice.ai/settings/api to add credits.",
|
||||
"This model requires more credits to use",
|
||||
"This endpoint require more credits",
|
||||
],
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "does not false-positive on issue ids and numeric references",
|
||||
samples: [
|
||||
"Fixed issue CHE-402 in the latest release",
|
||||
"See ticket #402 for details",
|
||||
"ISSUE-402 has been resolved",
|
||||
"Room 402 is available",
|
||||
"Error code 403 was returned, not 402-related",
|
||||
"The building at 402 Main Street",
|
||||
"processed 402 records",
|
||||
"402 items found in the database",
|
||||
"port 402 is open",
|
||||
"Use a 402 stainless bolt",
|
||||
"Book a 402 room",
|
||||
"There is a 402 near me",
|
||||
],
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "still matches real HTTP 402 billing errors",
|
||||
samples: [
|
||||
"HTTP 402 Payment Required",
|
||||
"status: 402",
|
||||
"error code 402",
|
||||
"http 402",
|
||||
"status=402 payment required",
|
||||
"got a 402 from the API",
|
||||
"returned 402",
|
||||
"received a 402 response",
|
||||
'{"status":402,"type":"error"}',
|
||||
'{"code":402,"message":"payment required"}',
|
||||
'{"error":{"code":402,"message":"billing hard limit reached"}}',
|
||||
],
|
||||
expected: true,
|
||||
},
|
||||
])("$name", ({ samples, expected }) => {
|
||||
expectMessageMatches(isBillingErrorMessage, samples, expected);
|
||||
});
|
||||
|
||||
it("does not false-positive on long assistant responses mentioning billing keywords", () => {
|
||||
// Simulate a multi-paragraph assistant response that mentions billing terms
|
||||
const longResponse =
|
||||
|
|
@ -176,37 +195,27 @@ describe("isBillingErrorMessage", () => {
|
|||
expect(longNonError.length).toBeGreaterThan(512);
|
||||
expect(isBillingErrorMessage(longNonError)).toBe(false);
|
||||
});
|
||||
it("still matches real HTTP 402 billing errors", () => {
|
||||
const realErrors = [
|
||||
"HTTP 402 Payment Required",
|
||||
"status: 402",
|
||||
"error code 402",
|
||||
"http 402",
|
||||
"status=402 payment required",
|
||||
"got a 402 from the API",
|
||||
"returned 402",
|
||||
"received a 402 response",
|
||||
'{"status":402,"type":"error"}',
|
||||
'{"code":402,"message":"payment required"}',
|
||||
'{"error":{"code":402,"message":"billing hard limit reached"}}',
|
||||
];
|
||||
for (const sample of realErrors) {
|
||||
expect(isBillingErrorMessage(sample)).toBe(true);
|
||||
}
|
||||
|
||||
it("prefers billing when API-key and 402 hints both appear", () => {
|
||||
const sample =
|
||||
"402 Payment Required: The account associated with this API key has reached its maximum allowed monthly spending limit.";
|
||||
expect(isBillingErrorMessage(sample)).toBe(true);
|
||||
expect(classifyFailoverReason(sample)).toBe("billing");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isCloudCodeAssistFormatError", () => {
|
||||
it("matches format errors", () => {
|
||||
const samples = [
|
||||
"INVALID_REQUEST_ERROR: string should match pattern",
|
||||
"messages.1.content.1.tool_use.id",
|
||||
"tool_use.id should match pattern",
|
||||
"invalid request format",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isCloudCodeAssistFormatError(sample)).toBe(true);
|
||||
}
|
||||
expectMessageMatches(
|
||||
isCloudCodeAssistFormatError,
|
||||
[
|
||||
"INVALID_REQUEST_ERROR: string should match pattern",
|
||||
"messages.1.content.1.tool_use.id",
|
||||
"tool_use.id should match pattern",
|
||||
"invalid request format",
|
||||
],
|
||||
true,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -238,20 +247,24 @@ describe("isCloudflareOrHtmlErrorPage", () => {
|
|||
});
|
||||
|
||||
describe("isCompactionFailureError", () => {
|
||||
it("matches compaction overflow failures", () => {
|
||||
const samples = [
|
||||
'Context overflow: Summarization failed: 400 {"message":"prompt is too long"}',
|
||||
"auto-compaction failed due to context overflow",
|
||||
"Compaction failed: prompt is too long",
|
||||
"Summarization failed: context window exceeded for this request",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isCompactionFailureError(sample)).toBe(true);
|
||||
}
|
||||
});
|
||||
it("ignores non-compaction overflow errors", () => {
|
||||
expect(isCompactionFailureError("Context overflow: prompt too large")).toBe(false);
|
||||
expect(isCompactionFailureError("rate limit exceeded")).toBe(false);
|
||||
it.each([
|
||||
{
|
||||
name: "matches compaction overflow failures",
|
||||
samples: [
|
||||
'Context overflow: Summarization failed: 400 {"message":"prompt is too long"}',
|
||||
"auto-compaction failed due to context overflow",
|
||||
"Compaction failed: prompt is too long",
|
||||
"Summarization failed: context window exceeded for this request",
|
||||
],
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "ignores non-compaction overflow errors",
|
||||
samples: ["Context overflow: prompt too large", "rate limit exceeded"],
|
||||
expected: false,
|
||||
},
|
||||
])("$name", ({ samples, expected }) => {
|
||||
expectMessageMatches(isCompactionFailureError, samples, expected);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -506,6 +519,10 @@ describe("isTransientHttpError", () => {
|
|||
});
|
||||
|
||||
describe("classifyFailoverReasonFromHttpStatus", () => {
|
||||
it("treats HTTP 401 permanent auth failures as auth_permanent", () => {
|
||||
expect(classifyFailoverReasonFromHttpStatus(401, "invalid_api_key")).toBe("auth_permanent");
|
||||
});
|
||||
|
||||
it("treats HTTP 422 as format error", () => {
|
||||
expect(classifyFailoverReasonFromHttpStatus(422)).toBe("format");
|
||||
expect(classifyFailoverReasonFromHttpStatus(422, "check open ai req parameter error")).toBe(
|
||||
|
|
@ -518,6 +535,10 @@ describe("classifyFailoverReasonFromHttpStatus", () => {
|
|||
expect(classifyFailoverReasonFromHttpStatus(422, "insufficient credits")).toBe("billing");
|
||||
});
|
||||
|
||||
it("treats HTTP 400 insufficient-quota payloads as billing instead of format", () => {
|
||||
expect(classifyFailoverReasonFromHttpStatus(400, INSUFFICIENT_QUOTA_PAYLOAD)).toBe("billing");
|
||||
});
|
||||
|
||||
it("treats HTTP 499 as transient for structured errors", () => {
|
||||
expect(classifyFailoverReasonFromHttpStatus(499)).toBe("timeout");
|
||||
expect(classifyFailoverReasonFromHttpStatus(499, "499 Client Closed Request")).toBe("timeout");
|
||||
|
|
|
|||
Loading…
Reference in New Issue