mirror of https://github.com/openclaw/openclaw.git
342 lines
11 KiB
TypeScript
342 lines
11 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
import { LiveSessionModelSwitchError } from "../../agents/live-model-switch.js";
|
|
import type { TemplateContext } from "../templating.js";
|
|
import type { GetReplyOptions } from "../types.js";
|
|
import type { FollowupRun } from "./queue.js";
|
|
import type { TypingSignaler } from "./typing-mode.js";
|
|
|
|
const state = vi.hoisted(() => ({
|
|
runEmbeddedPiAgentMock: vi.fn(),
|
|
runWithModelFallbackMock: vi.fn(),
|
|
}));
|
|
|
|
vi.mock("../../agents/pi-embedded.js", () => ({
|
|
runEmbeddedPiAgent: (params: unknown) => state.runEmbeddedPiAgentMock(params),
|
|
}));
|
|
|
|
vi.mock("../../agents/model-fallback.js", () => ({
|
|
runWithModelFallback: (params: unknown) => state.runWithModelFallbackMock(params),
|
|
isFallbackSummaryError: (err: unknown) =>
|
|
err instanceof Error &&
|
|
err.name === "FallbackSummaryError" &&
|
|
Array.isArray((err as { attempts?: unknown[] }).attempts),
|
|
}));
|
|
|
|
vi.mock("../../agents/model-selection.js", () => ({
|
|
isCliProvider: () => false,
|
|
}));
|
|
|
|
vi.mock("../../agents/cli-runner.js", () => ({
|
|
runCliAgent: vi.fn(),
|
|
}));
|
|
|
|
vi.mock("../../agents/cli-session.js", () => ({
|
|
getCliSessionId: vi.fn(),
|
|
}));
|
|
|
|
vi.mock("../../agents/bootstrap-budget.js", () => ({
|
|
resolveBootstrapWarningSignaturesSeen: () => [],
|
|
}));
|
|
|
|
vi.mock("../../agents/pi-embedded-helpers.js", () => ({
|
|
BILLING_ERROR_USER_MESSAGE: "billing",
|
|
isCompactionFailureError: () => false,
|
|
isContextOverflowError: () => false,
|
|
isBillingErrorMessage: () => false,
|
|
isLikelyContextOverflowError: () => false,
|
|
isRateLimitErrorMessage: () => false,
|
|
isTransientHttpError: () => false,
|
|
sanitizeUserFacingText: (text?: string) => text ?? "",
|
|
}));
|
|
|
|
vi.mock("../../config/sessions.js", () => ({
|
|
resolveGroupSessionKey: vi.fn(() => null),
|
|
resolveSessionTranscriptPath: vi.fn(),
|
|
updateSessionStore: vi.fn(),
|
|
}));
|
|
|
|
vi.mock("../../globals.js", () => ({
|
|
logVerbose: vi.fn(),
|
|
}));
|
|
|
|
vi.mock("../../infra/agent-events.js", () => ({
|
|
emitAgentEvent: vi.fn(),
|
|
registerAgentRunContext: vi.fn(),
|
|
}));
|
|
|
|
vi.mock("../../runtime.js", () => ({
|
|
defaultRuntime: {
|
|
error: vi.fn(),
|
|
},
|
|
}));
|
|
|
|
vi.mock("../../utils/message-channel.js", () => ({
|
|
isMarkdownCapableMessageChannel: () => true,
|
|
resolveMessageChannel: () => "whatsapp",
|
|
isInternalMessageChannel: () => false,
|
|
}));
|
|
|
|
vi.mock("../heartbeat.js", () => ({
|
|
stripHeartbeatToken: (text: string) => ({
|
|
text,
|
|
didStrip: false,
|
|
shouldSkip: false,
|
|
}),
|
|
}));
|
|
|
|
vi.mock("./agent-runner-utils.js", () => ({
|
|
buildEmbeddedRunExecutionParams: (params: { provider: string; model: string }) => ({
|
|
embeddedContext: {},
|
|
senderContext: {},
|
|
runBaseParams: {
|
|
provider: params.provider,
|
|
model: params.model,
|
|
},
|
|
}),
|
|
resolveModelFallbackOptions: vi.fn(() => ({})),
|
|
}));
|
|
|
|
vi.mock("./reply-delivery.js", () => ({
|
|
createBlockReplyDeliveryHandler: vi.fn(),
|
|
}));
|
|
|
|
vi.mock("./reply-media-paths.runtime.js", () => ({
|
|
createReplyMediaPathNormalizer: () => (payload: unknown) => payload,
|
|
}));
|
|
|
|
async function getRunAgentTurnWithFallback() {
|
|
return (await import("./agent-runner-execution.js")).runAgentTurnWithFallback;
|
|
}
|
|
|
|
type FallbackRunnerParams = {
|
|
run: (provider: string, model: string) => Promise<unknown>;
|
|
};
|
|
|
|
type EmbeddedAgentParams = {
|
|
onToolResult?: (payload: { text?: string; mediaUrls?: string[] }) => Promise<void> | void;
|
|
};
|
|
|
|
function createMockTypingSignaler(): TypingSignaler {
|
|
return {
|
|
mode: "message",
|
|
shouldStartImmediately: false,
|
|
shouldStartOnMessageStart: true,
|
|
shouldStartOnText: true,
|
|
shouldStartOnReasoning: false,
|
|
signalRunStart: vi.fn(async () => {}),
|
|
signalMessageStart: vi.fn(async () => {}),
|
|
signalTextDelta: vi.fn(async () => {}),
|
|
signalReasoningDelta: vi.fn(async () => {}),
|
|
signalToolStart: vi.fn(async () => {}),
|
|
};
|
|
}
|
|
|
|
function createFollowupRun(): FollowupRun {
|
|
return {
|
|
prompt: "hello",
|
|
summaryLine: "hello",
|
|
enqueuedAt: Date.now(),
|
|
run: {
|
|
agentId: "agent",
|
|
agentDir: "/tmp/agent",
|
|
sessionId: "session",
|
|
sessionKey: "main",
|
|
messageProvider: "whatsapp",
|
|
sessionFile: "/tmp/session.jsonl",
|
|
workspaceDir: "/tmp",
|
|
config: {},
|
|
skillsSnapshot: {},
|
|
provider: "anthropic",
|
|
model: "claude",
|
|
thinkLevel: "low",
|
|
verboseLevel: "off",
|
|
elevatedLevel: "off",
|
|
bashElevated: {
|
|
enabled: false,
|
|
allowed: false,
|
|
defaultLevel: "off",
|
|
},
|
|
timeoutMs: 1_000,
|
|
blockReplyBreak: "message_end",
|
|
},
|
|
} as unknown as FollowupRun;
|
|
}
|
|
|
|
describe("runAgentTurnWithFallback", () => {
|
|
beforeEach(() => {
|
|
state.runEmbeddedPiAgentMock.mockReset();
|
|
state.runWithModelFallbackMock.mockReset();
|
|
state.runWithModelFallbackMock.mockImplementation(async (params: FallbackRunnerParams) => ({
|
|
result: await params.run("anthropic", "claude"),
|
|
provider: "anthropic",
|
|
model: "claude",
|
|
attempts: [],
|
|
}));
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.clearAllMocks();
|
|
});
|
|
|
|
it("forwards media-only tool results without typing text", async () => {
|
|
const onToolResult = vi.fn();
|
|
state.runEmbeddedPiAgentMock.mockImplementationOnce(async (params: EmbeddedAgentParams) => {
|
|
await params.onToolResult?.({ mediaUrls: ["/tmp/generated.png"] });
|
|
return { payloads: [{ text: "final" }], meta: {} };
|
|
});
|
|
|
|
const runAgentTurnWithFallback = await getRunAgentTurnWithFallback();
|
|
const pendingToolTasks = new Set<Promise<void>>();
|
|
const typingSignals = createMockTypingSignaler();
|
|
const result = await runAgentTurnWithFallback({
|
|
commandBody: "hello",
|
|
followupRun: createFollowupRun(),
|
|
sessionCtx: {
|
|
Provider: "whatsapp",
|
|
MessageSid: "msg",
|
|
} as unknown as TemplateContext,
|
|
opts: {
|
|
onToolResult,
|
|
} satisfies GetReplyOptions,
|
|
typingSignals,
|
|
blockReplyPipeline: null,
|
|
blockStreamingEnabled: false,
|
|
resolvedBlockStreamingBreak: "message_end",
|
|
applyReplyToMode: (payload) => payload,
|
|
shouldEmitToolResult: () => true,
|
|
shouldEmitToolOutput: () => false,
|
|
pendingToolTasks,
|
|
resetSessionAfterCompactionFailure: async () => false,
|
|
resetSessionAfterRoleOrderingConflict: async () => false,
|
|
isHeartbeat: false,
|
|
sessionKey: "main",
|
|
getActiveSessionEntry: () => undefined,
|
|
resolvedVerboseLevel: "off",
|
|
});
|
|
|
|
await Promise.all(pendingToolTasks);
|
|
|
|
expect(result.kind).toBe("success");
|
|
expect(typingSignals.signalTextDelta).not.toHaveBeenCalled();
|
|
expect(onToolResult).toHaveBeenCalledTimes(1);
|
|
expect(onToolResult.mock.calls[0]?.[0]).toMatchObject({
|
|
mediaUrls: ["/tmp/generated.png"],
|
|
});
|
|
expect(onToolResult.mock.calls[0]?.[0]?.text).toBeUndefined();
|
|
});
|
|
|
|
it("does not show a rate-limit countdown for mixed-cause fallback exhaustion", async () => {
|
|
state.runWithModelFallbackMock.mockRejectedValueOnce(
|
|
Object.assign(
|
|
new Error(
|
|
"All models failed (2): anthropic/claude: 429 (rate_limit) | openai/gpt-5.2: 402 (billing)",
|
|
),
|
|
{
|
|
name: "FallbackSummaryError",
|
|
attempts: [
|
|
{ provider: "anthropic", model: "claude", error: "429", reason: "rate_limit" },
|
|
{ provider: "openai", model: "gpt-5.2", error: "402", reason: "billing" },
|
|
],
|
|
soonestCooldownExpiry: Date.now() + 60_000,
|
|
},
|
|
),
|
|
);
|
|
|
|
const runAgentTurnWithFallback = await getRunAgentTurnWithFallback();
|
|
const result = await runAgentTurnWithFallback({
|
|
commandBody: "hello",
|
|
followupRun: createFollowupRun(),
|
|
sessionCtx: {
|
|
Provider: "whatsapp",
|
|
MessageSid: "msg",
|
|
} as unknown as TemplateContext,
|
|
opts: {},
|
|
typingSignals: createMockTypingSignaler(),
|
|
blockReplyPipeline: null,
|
|
blockStreamingEnabled: false,
|
|
resolvedBlockStreamingBreak: "message_end",
|
|
applyReplyToMode: (payload) => payload,
|
|
shouldEmitToolResult: () => true,
|
|
shouldEmitToolOutput: () => false,
|
|
pendingToolTasks: new Set(),
|
|
resetSessionAfterCompactionFailure: async () => false,
|
|
resetSessionAfterRoleOrderingConflict: async () => false,
|
|
isHeartbeat: false,
|
|
sessionKey: "main",
|
|
getActiveSessionEntry: () => undefined,
|
|
resolvedVerboseLevel: "off",
|
|
});
|
|
|
|
expect(result.kind).toBe("final");
|
|
if (result.kind === "final") {
|
|
expect(result.payload.text).toContain("Agent failed before reply");
|
|
expect(result.payload.text).not.toContain("Rate-limited");
|
|
}
|
|
});
|
|
|
|
it("restarts the active prompt when a live model switch is requested", async () => {
|
|
let fallbackInvocation = 0;
|
|
state.runWithModelFallbackMock.mockImplementation(
|
|
async (params: { run: (provider: string, model: string) => Promise<unknown> }) => ({
|
|
result: await params.run(
|
|
fallbackInvocation === 0 ? "anthropic" : "openai",
|
|
fallbackInvocation === 0 ? "claude" : "gpt-5.4",
|
|
),
|
|
provider: fallbackInvocation === 0 ? "anthropic" : "openai",
|
|
model: fallbackInvocation++ === 0 ? "claude" : "gpt-5.4",
|
|
attempts: [],
|
|
}),
|
|
);
|
|
state.runEmbeddedPiAgentMock
|
|
.mockImplementationOnce(async () => {
|
|
throw new LiveSessionModelSwitchError({
|
|
provider: "openai",
|
|
model: "gpt-5.4",
|
|
});
|
|
})
|
|
.mockImplementationOnce(async () => {
|
|
return {
|
|
payloads: [{ text: "switched" }],
|
|
meta: {
|
|
agentMeta: {
|
|
sessionId: "session",
|
|
provider: "openai",
|
|
model: "gpt-5.4",
|
|
},
|
|
},
|
|
};
|
|
});
|
|
|
|
const runAgentTurnWithFallback = await getRunAgentTurnWithFallback();
|
|
const followupRun = createFollowupRun();
|
|
const result = await runAgentTurnWithFallback({
|
|
commandBody: "hello",
|
|
followupRun,
|
|
sessionCtx: {
|
|
Provider: "whatsapp",
|
|
MessageSid: "msg",
|
|
} as unknown as TemplateContext,
|
|
opts: {},
|
|
typingSignals: createMockTypingSignaler(),
|
|
blockReplyPipeline: null,
|
|
blockStreamingEnabled: false,
|
|
resolvedBlockStreamingBreak: "message_end",
|
|
applyReplyToMode: (payload) => payload,
|
|
shouldEmitToolResult: () => true,
|
|
shouldEmitToolOutput: () => false,
|
|
pendingToolTasks: new Set(),
|
|
resetSessionAfterCompactionFailure: async () => false,
|
|
resetSessionAfterRoleOrderingConflict: async () => false,
|
|
isHeartbeat: false,
|
|
sessionKey: "main",
|
|
getActiveSessionEntry: () => undefined,
|
|
resolvedVerboseLevel: "off",
|
|
});
|
|
|
|
expect(result.kind).toBe("success");
|
|
expect(state.runEmbeddedPiAgentMock).toHaveBeenCalledTimes(2);
|
|
expect(followupRun.run.provider).toBe("openai");
|
|
expect(followupRun.run.model).toBe("gpt-5.4");
|
|
});
|
|
});
|