Fix failover for zhipuai 1310 Weekly/Monthly Limit Exhausted (#33813)

Merged via squash.

Prepared head SHA: 3dc441e58d
Co-authored-by: zhouhe-xydt <265407618+zhouhe-xydt@users.noreply.github.com>
Co-authored-by: altaywtf <9790196+altaywtf@users.noreply.github.com>
Reviewed-by: @altaywtf
This commit is contained in:
zhouhe-xydt 2026-03-06 17:04:09 +08:00 committed by GitHub
parent ee6f7b1bf0
commit a65d70f84b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 56 additions and 0 deletions

View File

@ -194,6 +194,7 @@ Docs: https://docs.openclaw.ai
- Memory/flush default prompt: ban timestamped variant filenames during default memory flush runs so durable notes stay in the canonical daily `memory/YYYY-MM-DD.md` file. (#34951) thanks @zerone0x.
- Agents/reply delivery timing: flush embedded Pi block replies before waiting on compaction retries so already-generated assistant replies reach channels before compaction wait completes. (#35489) thanks @Sid-Qin.
- Agents/gateway config guidance: stop exposing `config.schema` through the agent `gateway` tool, remove prompt/docs guidance that told agents to call it, and keep agents on `config.get` plus `config.patch`/`config.apply` for config changes. (#7382) thanks @kakuteki.
- Agents/failover: classify periodic provider limit exhaustion text (for example `Weekly/Monthly Limit Exhausted`) as `rate_limit` while keeping explicit `402 Payment Required` variants in billing, so failover continues without misclassifying billing-wrapped quota errors. (#33813) thanks @zhouhe-xydt.
## 2026.3.2

View File

@ -22,6 +22,10 @@ const OPENROUTER_CREDITS_MESSAGE = "Payment Required: insufficient credits";
// https://github.com/openclaw/openclaw/issues/23440
const INSUFFICIENT_QUOTA_PAYLOAD =
'{"type":"error","error":{"type":"insufficient_quota","message":"Your account has insufficient quota balance to run this request."}}';
// Issue-backed ZhipuAI/GLM quota-exhausted log from #33785:
// https://github.com/openclaw/openclaw/issues/33785
const ZHIPUAI_WEEKLY_MONTHLY_LIMIT_EXHAUSTED_MESSAGE =
"LLM error 1310: Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-03-06 22:19:54 (request_id: 20260303141547610b7f574d1b44cb)";
// AWS Bedrock 429 ThrottlingException / 503 ServiceUnavailable:
// https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html
const BEDROCK_THROTTLING_EXCEPTION_MESSAGE =
@ -113,6 +117,27 @@ describe("failover-error", () => {
).toBe("billing");
});
it("treats zhipuai weekly/monthly limit exhausted as rate_limit", () => {
expect(
resolveFailoverReasonFromError({
message: ZHIPUAI_WEEKLY_MONTHLY_LIMIT_EXHAUSTED_MESSAGE,
}),
).toBe("rate_limit");
expect(
resolveFailoverReasonFromError({
message: "LLM error: monthly limit reached",
}),
).toBe("rate_limit");
});
it("keeps raw-text 402 weekly/monthly limit errors in billing", () => {
expect(
resolveFailoverReasonFromError({
message: "402 Payment Required: Weekly/Monthly Limit Exhausted",
}),
).toBe("billing");
});
it("infers format errors from error messages", () => {
expect(
resolveFailoverReasonFromError({

View File

@ -535,6 +535,14 @@ describe("classifyFailoverReason", () => {
).toBe("rate_limit");
expect(classifyFailoverReason("all credentials for model x are cooling down")).toBeNull();
expect(classifyFailoverReason("invalid request format")).toBe("format");
expect(classifyFailoverReason("credit balance too low")).toBe("billing");
// Billing with "limit exhausted" must stay billing, not rate_limit (avoids key-disable regression)
expect(
classifyFailoverReason("HTTP 402 payment required. Your limit exhausted for this plan."),
).toBe("billing");
expect(classifyFailoverReason("402 Payment Required: Weekly/Monthly Limit Exhausted")).toBe(
"billing",
);
expect(classifyFailoverReason(INSUFFICIENT_QUOTA_PAYLOAD)).toBe("billing");
expect(classifyFailoverReason("deadline exceeded")).toBe("timeout");
expect(classifyFailoverReason("request ended without sending any chunks")).toBe("timeout");
@ -584,6 +592,17 @@ describe("classifyFailoverReason", () => {
// but it should not be treated as provider overload / rate limit.
expect(classifyFailoverReason("LLM error: service unavailable")).toBe("timeout");
});
it("classifies zhipuai Weekly/Monthly Limit Exhausted as rate_limit (#33785)", () => {
expect(
classifyFailoverReason(
"LLM error 1310: Weekly/Monthly Limit Exhausted. Your limit will reset at 2026-03-06 22:19:54 (request_id: 20260303141547610b7f574d1b44cb)",
),
).toBe("rate_limit");
// Independent coverage for broader periodic limit patterns.
expect(classifyFailoverReason("LLM error: weekly/monthly limit reached")).toBe("rate_limit");
expect(classifyFailoverReason("LLM error: monthly limit reached")).toBe("rate_limit");
expect(classifyFailoverReason("LLM error: daily limit exceeded")).toBe("rate_limit");
});
it("classifies permanent auth errors as auth_permanent", () => {
expect(classifyFailoverReason("invalid_api_key")).toBe("auth_permanent");
expect(classifyFailoverReason("Your api key has been revoked")).toBe("auth_permanent");

View File

@ -8,6 +8,7 @@ import {
isAuthPermanentErrorMessage,
isBillingErrorMessage,
isOverloadedErrorMessage,
isPeriodicUsageLimitErrorMessage,
isRateLimitErrorMessage,
isTimeoutErrorMessage,
matchesFormatErrorPattern,
@ -842,6 +843,9 @@ export function classifyFailoverReason(raw: string): FailoverReason | null {
if (isJsonApiInternalServerError(raw)) {
return "timeout";
}
if (isPeriodicUsageLimitErrorMessage(raw)) {
return isBillingErrorMessage(raw) ? "billing" : "rate_limit";
}
if (isRateLimitErrorMessage(raw)) {
return "rate_limit";
}

View File

@ -1,5 +1,8 @@
type ErrorPattern = RegExp | string;
const PERIODIC_USAGE_LIMIT_RE =
/\b(?:daily|weekly|monthly)(?:\/(?:daily|weekly|monthly))* (?:usage )?limit(?:s)?(?: (?:exhausted|reached|exceeded))?\b/i;
const ERROR_PATTERNS = {
rateLimit: [
/rate[_ ]limit|too many requests|429/,
@ -117,6 +120,10 @@ export function isTimeoutErrorMessage(raw: string): boolean {
return matchesErrorPatterns(raw, ERROR_PATTERNS.timeout);
}
export function isPeriodicUsageLimitErrorMessage(raw: string): boolean {
return PERIODIC_USAGE_LIMIT_RE.test(raw);
}
export function isBillingErrorMessage(raw: string): boolean {
const value = raw.toLowerCase();
if (!value) {