From a65d70f84b4e332d9e83234be3fdb994a80a6444 Mon Sep 17 00:00:00 2001 From: zhouhe-xydt Date: Fri, 6 Mar 2026 17:04:09 +0800 Subject: [PATCH] Fix failover for zhipuai 1310 Weekly/Monthly Limit Exhausted (#33813) Merged via squash. Prepared head SHA: 3dc441e58de48913720cf7b6137fa761758d8344 Co-authored-by: zhouhe-xydt <265407618+zhouhe-xydt@users.noreply.github.com> Co-authored-by: altaywtf <9790196+altaywtf@users.noreply.github.com> Reviewed-by: @altaywtf --- CHANGELOG.md | 1 + src/agents/failover-error.test.ts | 25 +++++++++++++++++++ ...dded-helpers.isbillingerrormessage.test.ts | 19 ++++++++++++++ src/agents/pi-embedded-helpers/errors.ts | 4 +++ .../pi-embedded-helpers/failover-matches.ts | 7 ++++++ 5 files changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c8f46958b..741e32f8055 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -194,6 +194,7 @@ Docs: https://docs.openclaw.ai - Memory/flush default prompt: ban timestamped variant filenames during default memory flush runs so durable notes stay in the canonical daily `memory/YYYY-MM-DD.md` file. (#34951) thanks @zerone0x. - Agents/reply delivery timing: flush embedded Pi block replies before waiting on compaction retries so already-generated assistant replies reach channels before compaction wait completes. (#35489) thanks @Sid-Qin. - Agents/gateway config guidance: stop exposing `config.schema` through the agent `gateway` tool, remove prompt/docs guidance that told agents to call it, and keep agents on `config.get` plus `config.patch`/`config.apply` for config changes. (#7382) thanks @kakuteki. +- Agents/failover: classify periodic provider limit exhaustion text (for example `Weekly/Monthly Limit Exhausted`) as `rate_limit` while keeping explicit `402 Payment Required` variants in billing, so failover continues without misclassifying billing-wrapped quota errors. (#33813) thanks @zhouhe-xydt. ## 2026.3.2 diff --git a/src/agents/failover-error.test.ts b/src/agents/failover-error.test.ts index 4e4379bf5da..6d0b6202f04 100644 --- a/src/agents/failover-error.test.ts +++ b/src/agents/failover-error.test.ts @@ -22,6 +22,10 @@ const OPENROUTER_CREDITS_MESSAGE = "Payment Required: insufficient credits"; // https://github.com/openclaw/openclaw/issues/23440 const INSUFFICIENT_QUOTA_PAYLOAD = '{"type":"error","error":{"type":"insufficient_quota","message":"Your account has insufficient quota balance to run this request."}}'; +// Issue-backed ZhipuAI/GLM quota-exhausted log from #33785: +// https://github.com/openclaw/openclaw/issues/33785 +const ZHIPUAI_WEEKLY_MONTHLY_LIMIT_EXHAUSTED_MESSAGE = + "LLM error 1310: Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-03-06 22:19:54 (request_id: 20260303141547610b7f574d1b44cb)"; // AWS Bedrock 429 ThrottlingException / 503 ServiceUnavailable: // https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html const BEDROCK_THROTTLING_EXCEPTION_MESSAGE = @@ -113,6 +117,27 @@ describe("failover-error", () => { ).toBe("billing"); }); + it("treats zhipuai weekly/monthly limit exhausted as rate_limit", () => { + expect( + resolveFailoverReasonFromError({ + message: ZHIPUAI_WEEKLY_MONTHLY_LIMIT_EXHAUSTED_MESSAGE, + }), + ).toBe("rate_limit"); + expect( + resolveFailoverReasonFromError({ + message: "LLM error: monthly limit reached", + }), + ).toBe("rate_limit"); + }); + + it("keeps raw-text 402 weekly/monthly limit errors in billing", () => { + expect( + resolveFailoverReasonFromError({ + message: "402 Payment Required: Weekly/Monthly Limit Exhausted", + }), + ).toBe("billing"); + }); + it("infers format errors from error messages", () => { expect( resolveFailoverReasonFromError({ diff --git a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts index dd8a38d2814..9eb2657158b 100644 --- a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts +++ b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts @@ -535,6 +535,14 @@ describe("classifyFailoverReason", () => { ).toBe("rate_limit"); expect(classifyFailoverReason("all credentials for model x are cooling down")).toBeNull(); expect(classifyFailoverReason("invalid request format")).toBe("format"); + expect(classifyFailoverReason("credit balance too low")).toBe("billing"); + // Billing with "limit exhausted" must stay billing, not rate_limit (avoids key-disable regression) + expect( + classifyFailoverReason("HTTP 402 payment required. Your limit exhausted for this plan."), + ).toBe("billing"); + expect(classifyFailoverReason("402 Payment Required: Weekly/Monthly Limit Exhausted")).toBe( + "billing", + ); expect(classifyFailoverReason(INSUFFICIENT_QUOTA_PAYLOAD)).toBe("billing"); expect(classifyFailoverReason("deadline exceeded")).toBe("timeout"); expect(classifyFailoverReason("request ended without sending any chunks")).toBe("timeout"); @@ -584,6 +592,17 @@ describe("classifyFailoverReason", () => { // but it should not be treated as provider overload / rate limit. expect(classifyFailoverReason("LLM error: service unavailable")).toBe("timeout"); }); + it("classifies zhipuai Weekly/Monthly Limit Exhausted as rate_limit (#33785)", () => { + expect( + classifyFailoverReason( + "LLM error 1310: Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-03-06 22:19:54 (request_id: 20260303141547610b7f574d1b44cb)", + ), + ).toBe("rate_limit"); + // Independent coverage for broader periodic limit patterns. + expect(classifyFailoverReason("LLM error: weekly/monthly limit reached")).toBe("rate_limit"); + expect(classifyFailoverReason("LLM error: monthly limit reached")).toBe("rate_limit"); + expect(classifyFailoverReason("LLM error: daily limit exceeded")).toBe("rate_limit"); + }); it("classifies permanent auth errors as auth_permanent", () => { expect(classifyFailoverReason("invalid_api_key")).toBe("auth_permanent"); expect(classifyFailoverReason("Your api key has been revoked")).toBe("auth_permanent"); diff --git a/src/agents/pi-embedded-helpers/errors.ts b/src/agents/pi-embedded-helpers/errors.ts index e4944b0731c..0f602ce66d7 100644 --- a/src/agents/pi-embedded-helpers/errors.ts +++ b/src/agents/pi-embedded-helpers/errors.ts @@ -8,6 +8,7 @@ import { isAuthPermanentErrorMessage, isBillingErrorMessage, isOverloadedErrorMessage, + isPeriodicUsageLimitErrorMessage, isRateLimitErrorMessage, isTimeoutErrorMessage, matchesFormatErrorPattern, @@ -842,6 +843,9 @@ export function classifyFailoverReason(raw: string): FailoverReason | null { if (isJsonApiInternalServerError(raw)) { return "timeout"; } + if (isPeriodicUsageLimitErrorMessage(raw)) { + return isBillingErrorMessage(raw) ? "billing" : "rate_limit"; + } if (isRateLimitErrorMessage(raw)) { return "rate_limit"; } diff --git a/src/agents/pi-embedded-helpers/failover-matches.ts b/src/agents/pi-embedded-helpers/failover-matches.ts index abbd6e769fa..6a7ce9d51d3 100644 --- a/src/agents/pi-embedded-helpers/failover-matches.ts +++ b/src/agents/pi-embedded-helpers/failover-matches.ts @@ -1,5 +1,8 @@ type ErrorPattern = RegExp | string; +const PERIODIC_USAGE_LIMIT_RE = + /\b(?:daily|weekly|monthly)(?:\/(?:daily|weekly|monthly))* (?:usage )?limit(?:s)?(?: (?:exhausted|reached|exceeded))?\b/i; + const ERROR_PATTERNS = { rateLimit: [ /rate[_ ]limit|too many requests|429/, @@ -117,6 +120,10 @@ export function isTimeoutErrorMessage(raw: string): boolean { return matchesErrorPatterns(raw, ERROR_PATTERNS.timeout); } +export function isPeriodicUsageLimitErrorMessage(raw: string): boolean { + return PERIODIC_USAGE_LIMIT_RE.test(raw); +} + export function isBillingErrorMessage(raw: string): boolean { const value = raw.toLowerCase(); if (!value) {