From d2a1b24b83f951a4e8171f2605cbea42b818216f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 23 Mar 2026 06:41:23 +0000 Subject: [PATCH] test: honor env auth in gateway live probes --- .../gateway-models.profiles.live.test.ts | 38 +++++++-------- src/gateway/live-tool-probe-utils.test.ts | 46 +++++++++++++++++++ src/gateway/live-tool-probe-utils.ts | 7 +++ 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/src/gateway/gateway-models.profiles.live.test.ts b/src/gateway/gateway-models.profiles.live.test.ts index e742b32e6f0..fa97673f3bc 100644 --- a/src/gateway/gateway-models.profiles.live.test.ts +++ b/src/gateway/gateway-models.profiles.live.test.ts @@ -10,7 +10,6 @@ import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import { type AuthProfileStore, ensureAuthProfileStore, - resolveAuthProfileOrder, saveAuthProfileStore, } from "../agents/auth-profiles.js"; import { @@ -43,6 +42,7 @@ import { loadSessionEntry, readSessionMessages } from "./session-utils.js"; const LIVE = isTruthyEnvValue(process.env.LIVE) || isTruthyEnvValue(process.env.OPENCLAW_LIVE_TEST); const GATEWAY_LIVE = isTruthyEnvValue(process.env.OPENCLAW_LIVE_GATEWAY); const ZAI_FALLBACK = isTruthyEnvValue(process.env.OPENCLAW_LIVE_GATEWAY_ZAI_FALLBACK); +const REQUIRE_PROFILE_KEYS = isTruthyEnvValue(process.env.OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS); const PROVIDERS = parseFilter(process.env.OPENCLAW_LIVE_GATEWAY_PROVIDERS); const THINKING_LEVEL = "high"; const THINKING_TAG_RE = /<\s*\/?\s*(?:think(?:ing)?|thought|antthinking)\s*>/i; @@ -1383,9 +1383,6 @@ describeLive("gateway live (dev agent, profile keys)", () => { await ensureOpenClawModelsJson(cfg); const agentDir = resolveOpenClawAgentDir(); - const authStore = ensureAuthProfileStore(agentDir, { - allowKeychainPrompt: false, - }); const authStorage = discoverAuthStorage(agentDir); const modelRegistry = discoverModels(authStorage, agentDir); const all = modelRegistry.getAll(); @@ -1399,8 +1396,8 @@ describeLive("gateway live (dev agent, profile keys)", () => { ? all.filter((m) => filter.has(`${m.provider}/${m.id}`)) : all.filter((m) => isModernModelRef({ provider: m.provider, id: m.id })); - const providerProfileCache = new Map(); const candidates: Array> = []; + const skipped: Array<{ model: string; error: string }> = []; for (const model of wanted) { if (shouldSuppressBuiltInModel({ provider: model.provider, id: model.id })) { continue; @@ -1408,23 +1405,28 @@ describeLive("gateway live (dev agent, profile keys)", () => { if (PROVIDERS && !PROVIDERS.has(model.provider)) { continue; } - let hasProfile = providerProfileCache.get(model.provider); - if (hasProfile === undefined) { - const order = resolveAuthProfileOrder({ - cfg, - store: authStore, - provider: model.provider, - }); - hasProfile = order.some((profileId) => Boolean(authStore.profiles[profileId])); - providerProfileCache.set(model.provider, hasProfile); + const modelRef = `${model.provider}/${model.id}`; + try { + const apiKeyInfo = await getApiKeyForModel({ model, cfg }); + if (REQUIRE_PROFILE_KEYS && !apiKeyInfo.source.startsWith("profile:")) { + skipped.push({ + model: modelRef, + error: `non-profile credential source: ${apiKeyInfo.source}`, + }); + continue; + } + candidates.push(model); + } catch (error) { + skipped.push({ model: modelRef, error: String(error) }); } - if (!hasProfile) { - continue; - } - candidates.push(model); } if (candidates.length === 0) { + if (skipped.length > 0) { + logProgress( + `[all-models] auth lookup skipped candidates:\n${formatFailurePreview(skipped, 8)}`, + ); + } logProgress("[all-models] no API keys found; skipping"); return; } diff --git a/src/gateway/live-tool-probe-utils.test.ts b/src/gateway/live-tool-probe-utils.test.ts index 75f27c08036..def908d52ea 100644 --- a/src/gateway/live-tool-probe-utils.test.ts +++ b/src/gateway/live-tool-probe-utils.test.ts @@ -136,6 +136,30 @@ describe("live tool probe utils", () => { }, expected: true, }, + { + name: "retries conversational try-again output", + params: { + text: "Let me try reading the file again:", + nonceA: "nonce-a", + nonceB: "nonce-b", + provider: "zai", + attempt: 0, + maxAttempts: 3, + }, + expected: true, + }, + { + name: "does not retry generic conversational text without tool-retry context", + params: { + text: "Let me try a different approach.", + nonceA: "nonce-a", + nonceB: "nonce-b", + provider: "zai", + attempt: 0, + maxAttempts: 3, + }, + expected: false, + }, { name: "retries mistral nonce marker echoes without parsed values", params: { @@ -234,6 +258,28 @@ describe("live tool probe utils", () => { }, expected: true, }, + { + name: "retries conversational try-again exec output", + params: { + text: "Let me try reading the file again:", + nonce: "nonce-c", + provider: "zai", + attempt: 0, + maxAttempts: 3, + }, + expected: true, + }, + { + name: "does not retry generic exec conversational text without tool-retry context", + params: { + text: "Let me try a different approach.", + nonce: "nonce-c", + provider: "zai", + attempt: 0, + maxAttempts: 3, + }, + expected: false, + }, { name: "does not special-case anthropic refusals for other providers", params: { diff --git a/src/gateway/live-tool-probe-utils.ts b/src/gateway/live-tool-probe-utils.ts index 62b618fe24d..a9eed34017d 100644 --- a/src/gateway/live-tool-probe-utils.ts +++ b/src/gateway/live-tool-probe-utils.ts @@ -53,6 +53,13 @@ function hasMalformedToolOutput(text: string): boolean { if (trimmed.includes("[object Object]")) { return true; } + if ( + lower.includes("try reading the file again") || + lower.includes("trying to read the file again") || + lower.includes("try the read tool again") + ) { + return true; + } if (/\bread\s*\[/.test(lower) || /\btool\b/.test(lower) || /\bfunction\b/.test(lower)) { return true; }