diff --git a/CHANGELOG.md b/CHANGELOG.md index 19d9bb2347c..fcf8d4862d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Docs: https://docs.openclaw.ai - Docker/timezone override: add `OPENCLAW_TZ` so `docker-setup.sh` can pin gateway and CLI containers to a chosen IANA timezone instead of inheriting the daemon default. (#34119) Thanks @Lanfei. - iOS/onboarding: add a first-run welcome pager before gateway setup, stop auto-opening the QR scanner, and show `/pair qr` instructions on the connect step. (#45054) Thanks @ngutman. - Browser/existing-session: add an official Chrome DevTools MCP attach mode for signed-in live Chrome sessions, with docs for `chrome://inspect/#remote-debugging` enablement and direct backlinks to Chrome’s own setup guides. +- Browser/act automation: add batched actions, selector targeting, and delayed clicks for browser act requests with normalized batch dispatch. Thanks @vincentkoc. ### Fixes diff --git a/src/browser/client-actions-core.ts b/src/browser/client-actions-core.ts index 72e27cd9afa..149ca54fadf 100644 --- a/src/browser/client-actions-core.ts +++ b/src/browser/client-actions-core.ts @@ -15,16 +15,19 @@ export type BrowserFormField = { export type BrowserActRequest = | { kind: "click"; - ref: string; + ref?: string; + selector?: string; targetId?: string; doubleClick?: boolean; button?: string; modifiers?: string[]; + delayMs?: number; timeoutMs?: number; } | { kind: "type"; - ref: string; + ref?: string; + selector?: string; text: string; targetId?: string; submit?: boolean; @@ -32,23 +35,33 @@ export type BrowserActRequest = timeoutMs?: number; } | { kind: "press"; key: string; targetId?: string; delayMs?: number } - | { kind: "hover"; ref: string; targetId?: string; timeoutMs?: number } + | { + kind: "hover"; + ref?: string; + selector?: string; + targetId?: string; + timeoutMs?: number; + } | { kind: "scrollIntoView"; - ref: string; + ref?: string; + selector?: string; targetId?: string; timeoutMs?: number; } | { kind: "drag"; - startRef: string; - endRef: string; + startRef?: string; + startSelector?: string; + endRef?: string; + endSelector?: string; targetId?: string; timeoutMs?: number; } | { kind: "select"; - ref: string; + ref?: string; + selector?: string; values: string[]; targetId?: string; timeoutMs?: number; @@ -73,13 +86,20 @@ export type BrowserActRequest = timeoutMs?: number; } | { kind: "evaluate"; fn: string; ref?: string; targetId?: string; timeoutMs?: number } - | { kind: "close"; targetId?: string }; + | { kind: "close"; targetId?: string } + | { + kind: "batch"; + actions: BrowserActRequest[]; + targetId?: string; + stopOnError?: boolean; + }; export type BrowserActResponse = { ok: true; targetId: string; url?: string; result?: unknown; + results?: Array<{ ok: boolean; error?: string }>; }; export type BrowserDownloadPayload = { diff --git a/src/browser/client.test.ts b/src/browser/client.test.ts index a4f95c23007..64d37580e35 100644 --- a/src/browser/client.test.ts +++ b/src/browser/client.test.ts @@ -160,6 +160,7 @@ describe("browser client", () => { targetId: "t1", url: "https://x", result: 1, + results: [{ ok: true }], }), } as unknown as Response; } @@ -258,7 +259,7 @@ describe("browser client", () => { ).resolves.toMatchObject({ ok: true, targetId: "t1" }); await expect( browserAct("http://127.0.0.1:18791", { kind: "click", ref: "1" }), - ).resolves.toMatchObject({ ok: true, targetId: "t1" }); + ).resolves.toMatchObject({ ok: true, targetId: "t1", results: [{ ok: true }] }); await expect( browserArmFileChooser("http://127.0.0.1:18791", { paths: ["/tmp/a.txt"], diff --git a/src/browser/pw-ai.ts b/src/browser/pw-ai.ts index 6da8b410c83..f8d538b5394 100644 --- a/src/browser/pw-ai.ts +++ b/src/browser/pw-ai.ts @@ -19,6 +19,7 @@ export { export { armDialogViaPlaywright, armFileUploadViaPlaywright, + batchViaPlaywright, clickViaPlaywright, closePageViaPlaywright, cookiesClearViaPlaywright, diff --git a/src/browser/pw-tools-core.interactions.batch.test.ts b/src/browser/pw-tools-core.interactions.batch.test.ts new file mode 100644 index 00000000000..2801ebe8190 --- /dev/null +++ b/src/browser/pw-tools-core.interactions.batch.test.ts @@ -0,0 +1,85 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +let page: { evaluate: ReturnType } | null = null; + +const getPageForTargetId = vi.fn(async () => { + if (!page) { + throw new Error("test: page not set"); + } + return page; +}); +const ensurePageState = vi.fn(() => {}); +const forceDisconnectPlaywrightForTarget = vi.fn(async () => {}); +const refLocator = vi.fn(() => { + throw new Error("test: refLocator should not be called"); +}); +const restoreRoleRefsForTarget = vi.fn(() => {}); + +const closePageViaPlaywright = vi.fn(async () => {}); +const resizeViewportViaPlaywright = vi.fn(async () => {}); + +vi.mock("./pw-session.js", () => ({ + ensurePageState, + forceDisconnectPlaywrightForTarget, + getPageForTargetId, + refLocator, + restoreRoleRefsForTarget, +})); + +vi.mock("./pw-tools-core.snapshot.js", () => ({ + closePageViaPlaywright, + resizeViewportViaPlaywright, +})); + +let batchViaPlaywright: typeof import("./pw-tools-core.interactions.js").batchViaPlaywright; + +describe("batchViaPlaywright", () => { + beforeAll(async () => { + ({ batchViaPlaywright } = await import("./pw-tools-core.interactions.js")); + }); + + beforeEach(() => { + vi.clearAllMocks(); + page = { + evaluate: vi.fn(async () => "ok"), + }; + }); + + it("propagates evaluate timeouts through batched execution", async () => { + const result = await batchViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + evaluateEnabled: true, + actions: [{ kind: "evaluate", fn: "() => 1", timeoutMs: 5000 }], + }); + + expect(result).toEqual({ results: [{ ok: true }] }); + expect(page?.evaluate).toHaveBeenCalledWith( + expect.any(Function), + expect.objectContaining({ + fnBody: "() => 1", + timeoutMs: 4500, + }), + ); + }); + + it("supports resize and close inside a batch", async () => { + const result = await batchViaPlaywright({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + actions: [{ kind: "resize", width: 800, height: 600 }, { kind: "close" }], + }); + + expect(result).toEqual({ results: [{ ok: true }, { ok: true }] }); + expect(resizeViewportViaPlaywright).toHaveBeenCalledWith({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + width: 800, + height: 600, + }); + expect(closePageViaPlaywright).toHaveBeenCalledWith({ + cdpUrl: "http://127.0.0.1:9222", + targetId: "tab-1", + }); + }); +}); diff --git a/src/browser/pw-tools-core.interactions.ts b/src/browser/pw-tools-core.interactions.ts index 852b11bb6dc..dee8a03316c 100644 --- a/src/browser/pw-tools-core.interactions.ts +++ b/src/browser/pw-tools-core.interactions.ts @@ -1,4 +1,4 @@ -import type { BrowserFormField } from "./client-actions-core.js"; +import type { BrowserActRequest, BrowserFormField } from "./client-actions-core.js"; import { DEFAULT_FILL_FIELD_TYPE } from "./form-fields.js"; import { DEFAULT_UPLOAD_DIR, resolveStrictExistingPathsWithinRoot } from "./paths.js"; import { @@ -8,12 +8,32 @@ import { refLocator, restoreRoleRefsForTarget, } from "./pw-session.js"; -import { normalizeTimeoutMs, requireRef, toAIFriendlyError } from "./pw-tools-core.shared.js"; +import { + normalizeTimeoutMs, + requireRef, + requireRefOrSelector, + toAIFriendlyError, +} from "./pw-tools-core.shared.js"; +import { closePageViaPlaywright, resizeViewportViaPlaywright } from "./pw-tools-core.snapshot.js"; type TargetOpts = { cdpUrl: string; targetId?: string; }; +const MAX_CLICK_DELAY_MS = 5_000; +const MAX_WAIT_TIME_MS = 30_000; +const MAX_BATCH_ACTIONS = 100; + +function resolveBoundedDelayMs(value: number | undefined, label: string, maxMs: number): number { + const normalized = Math.floor(value ?? 0); + if (!Number.isFinite(normalized) || normalized < 0) { + throw new Error(`${label} must be >= 0`); + } + if (normalized > maxMs) { + throw new Error(`${label} exceeds maximum of ${maxMs}ms`); + } + return normalized; +} async function getRestoredPageForTarget(opts: TargetOpts) { const page = await getPageForTargetId(opts); @@ -59,17 +79,27 @@ export async function highlightViaPlaywright(opts: { export async function clickViaPlaywright(opts: { cdpUrl: string; targetId?: string; - ref: string; + ref?: string; + selector?: string; doubleClick?: boolean; button?: "left" | "right" | "middle"; modifiers?: Array<"Alt" | "Control" | "ControlOrMeta" | "Meta" | "Shift">; + delayMs?: number; timeoutMs?: number; }): Promise { + const resolved = requireRefOrSelector(opts.ref, opts.selector); const page = await getRestoredPageForTarget(opts); - const ref = requireRef(opts.ref); - const locator = refLocator(page, ref); + const label = resolved.ref ?? resolved.selector!; + const locator = resolved.ref + ? refLocator(page, requireRef(resolved.ref)) + : page.locator(resolved.selector!); const timeout = resolveInteractionTimeoutMs(opts.timeoutMs); try { + const delayMs = resolveBoundedDelayMs(opts.delayMs, "click delayMs", MAX_CLICK_DELAY_MS); + if (delayMs > 0) { + await locator.hover({ timeout }); + await new Promise((r) => setTimeout(r, delayMs)); + } if (opts.doubleClick) { await locator.dblclick({ timeout, @@ -84,67 +114,84 @@ export async function clickViaPlaywright(opts: { }); } } catch (err) { - throw toAIFriendlyError(err, ref); + throw toAIFriendlyError(err, label); } } export async function hoverViaPlaywright(opts: { cdpUrl: string; targetId?: string; - ref: string; + ref?: string; + selector?: string; timeoutMs?: number; }): Promise { - const ref = requireRef(opts.ref); + const resolved = requireRefOrSelector(opts.ref, opts.selector); const page = await getRestoredPageForTarget(opts); + const label = resolved.ref ?? resolved.selector!; + const locator = resolved.ref + ? refLocator(page, requireRef(resolved.ref)) + : page.locator(resolved.selector!); try { - await refLocator(page, ref).hover({ + await locator.hover({ timeout: resolveInteractionTimeoutMs(opts.timeoutMs), }); } catch (err) { - throw toAIFriendlyError(err, ref); + throw toAIFriendlyError(err, label); } } export async function dragViaPlaywright(opts: { cdpUrl: string; targetId?: string; - startRef: string; - endRef: string; + startRef?: string; + startSelector?: string; + endRef?: string; + endSelector?: string; timeoutMs?: number; }): Promise { - const startRef = requireRef(opts.startRef); - const endRef = requireRef(opts.endRef); - if (!startRef || !endRef) { - throw new Error("startRef and endRef are required"); - } + const resolvedStart = requireRefOrSelector(opts.startRef, opts.startSelector); + const resolvedEnd = requireRefOrSelector(opts.endRef, opts.endSelector); const page = await getRestoredPageForTarget(opts); + const startLocator = resolvedStart.ref + ? refLocator(page, requireRef(resolvedStart.ref)) + : page.locator(resolvedStart.selector!); + const endLocator = resolvedEnd.ref + ? refLocator(page, requireRef(resolvedEnd.ref)) + : page.locator(resolvedEnd.selector!); + const startLabel = resolvedStart.ref ?? resolvedStart.selector!; + const endLabel = resolvedEnd.ref ?? resolvedEnd.selector!; try { - await refLocator(page, startRef).dragTo(refLocator(page, endRef), { + await startLocator.dragTo(endLocator, { timeout: resolveInteractionTimeoutMs(opts.timeoutMs), }); } catch (err) { - throw toAIFriendlyError(err, `${startRef} -> ${endRef}`); + throw toAIFriendlyError(err, `${startLabel} -> ${endLabel}`); } } export async function selectOptionViaPlaywright(opts: { cdpUrl: string; targetId?: string; - ref: string; + ref?: string; + selector?: string; values: string[]; timeoutMs?: number; }): Promise { - const ref = requireRef(opts.ref); + const resolved = requireRefOrSelector(opts.ref, opts.selector); if (!opts.values?.length) { throw new Error("values are required"); } const page = await getRestoredPageForTarget(opts); + const label = resolved.ref ?? resolved.selector!; + const locator = resolved.ref + ? refLocator(page, requireRef(resolved.ref)) + : page.locator(resolved.selector!); try { - await refLocator(page, ref).selectOption(opts.values, { + await locator.selectOption(opts.values, { timeout: resolveInteractionTimeoutMs(opts.timeoutMs), }); } catch (err) { - throw toAIFriendlyError(err, ref); + throw toAIFriendlyError(err, label); } } @@ -168,16 +215,20 @@ export async function pressKeyViaPlaywright(opts: { export async function typeViaPlaywright(opts: { cdpUrl: string; targetId?: string; - ref: string; + ref?: string; + selector?: string; text: string; submit?: boolean; slowly?: boolean; timeoutMs?: number; }): Promise { + const resolved = requireRefOrSelector(opts.ref, opts.selector); const text = String(opts.text ?? ""); const page = await getRestoredPageForTarget(opts); - const ref = requireRef(opts.ref); - const locator = refLocator(page, ref); + const label = resolved.ref ?? resolved.selector!; + const locator = resolved.ref + ? refLocator(page, requireRef(resolved.ref)) + : page.locator(resolved.selector!); const timeout = resolveInteractionTimeoutMs(opts.timeoutMs); try { if (opts.slowly) { @@ -190,7 +241,7 @@ export async function typeViaPlaywright(opts: { await locator.press("Enter", { timeout }); } } catch (err) { - throw toAIFriendlyError(err, ref); + throw toAIFriendlyError(err, label); } } @@ -367,18 +418,22 @@ export async function evaluateViaPlaywright(opts: { export async function scrollIntoViewViaPlaywright(opts: { cdpUrl: string; targetId?: string; - ref: string; + ref?: string; + selector?: string; timeoutMs?: number; }): Promise { + const resolved = requireRefOrSelector(opts.ref, opts.selector); const page = await getRestoredPageForTarget(opts); const timeout = normalizeTimeoutMs(opts.timeoutMs, 20_000); - const ref = requireRef(opts.ref); - const locator = refLocator(page, ref); + const label = resolved.ref ?? resolved.selector!; + const locator = resolved.ref + ? refLocator(page, requireRef(resolved.ref)) + : page.locator(resolved.selector!); try { await locator.scrollIntoViewIfNeeded({ timeout }); } catch (err) { - throw toAIFriendlyError(err, ref); + throw toAIFriendlyError(err, label); } } @@ -399,7 +454,7 @@ export async function waitForViaPlaywright(opts: { const timeout = normalizeTimeoutMs(opts.timeoutMs, 20_000); if (typeof opts.timeMs === "number" && Number.isFinite(opts.timeMs)) { - await page.waitForTimeout(Math.max(0, opts.timeMs)); + await page.waitForTimeout(resolveBoundedDelayMs(opts.timeMs, "wait timeMs", MAX_WAIT_TIME_MS)); } if (opts.text) { await page.getByText(opts.text).first().waitFor({ @@ -648,3 +703,189 @@ export async function setInputFilesViaPlaywright(opts: { // Best-effort for sites that don't react to setInputFiles alone. } } + +const MAX_BATCH_DEPTH = 5; + +async function executeSingleAction( + action: BrowserActRequest, + cdpUrl: string, + targetId?: string, + evaluateEnabled?: boolean, + depth = 0, +): Promise { + if (depth > MAX_BATCH_DEPTH) { + throw new Error(`Batch nesting depth exceeds maximum of ${MAX_BATCH_DEPTH}`); + } + const effectiveTargetId = action.targetId ?? targetId; + switch (action.kind) { + case "click": + await clickViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + ref: action.ref, + selector: action.selector, + doubleClick: action.doubleClick, + button: action.button as "left" | "right" | "middle" | undefined, + modifiers: action.modifiers as Array< + "Alt" | "Control" | "ControlOrMeta" | "Meta" | "Shift" + >, + delayMs: action.delayMs, + timeoutMs: action.timeoutMs, + }); + break; + case "type": + await typeViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + ref: action.ref, + selector: action.selector, + text: action.text, + submit: action.submit, + slowly: action.slowly, + timeoutMs: action.timeoutMs, + }); + break; + case "press": + await pressKeyViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + key: action.key, + delayMs: action.delayMs, + }); + break; + case "hover": + await hoverViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + ref: action.ref, + selector: action.selector, + timeoutMs: action.timeoutMs, + }); + break; + case "scrollIntoView": + await scrollIntoViewViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + ref: action.ref, + selector: action.selector, + timeoutMs: action.timeoutMs, + }); + break; + case "drag": + await dragViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + startRef: action.startRef, + startSelector: action.startSelector, + endRef: action.endRef, + endSelector: action.endSelector, + timeoutMs: action.timeoutMs, + }); + break; + case "select": + await selectOptionViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + ref: action.ref, + selector: action.selector, + values: action.values, + timeoutMs: action.timeoutMs, + }); + break; + case "fill": + await fillFormViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + fields: action.fields, + timeoutMs: action.timeoutMs, + }); + break; + case "resize": + await resizeViewportViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + width: action.width, + height: action.height, + }); + break; + case "wait": + if (action.fn && !evaluateEnabled) { + throw new Error("wait --fn is disabled by config (browser.evaluateEnabled=false)"); + } + await waitForViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + timeMs: action.timeMs, + text: action.text, + textGone: action.textGone, + selector: action.selector, + url: action.url, + loadState: action.loadState, + fn: action.fn, + timeoutMs: action.timeoutMs, + }); + break; + case "evaluate": + if (!evaluateEnabled) { + throw new Error("act:evaluate is disabled by config (browser.evaluateEnabled=false)"); + } + await evaluateViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + fn: action.fn, + ref: action.ref, + timeoutMs: action.timeoutMs, + }); + break; + case "close": + await closePageViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + }); + break; + case "batch": + // Nested batches: delegate recursively + await batchViaPlaywright({ + cdpUrl, + targetId: effectiveTargetId, + actions: action.actions, + stopOnError: action.stopOnError, + evaluateEnabled, + depth: depth + 1, + }); + break; + default: + throw new Error(`Unsupported batch action kind: ${(action as { kind: string }).kind}`); + } +} + +export async function batchViaPlaywright(opts: { + cdpUrl: string; + targetId?: string; + actions: BrowserActRequest[]; + stopOnError?: boolean; + evaluateEnabled?: boolean; + depth?: number; +}): Promise<{ results: Array<{ ok: boolean; error?: string }> }> { + const depth = opts.depth ?? 0; + if (depth > MAX_BATCH_DEPTH) { + throw new Error(`Batch nesting depth exceeds maximum of ${MAX_BATCH_DEPTH}`); + } + if (opts.actions.length > MAX_BATCH_ACTIONS) { + throw new Error(`Batch exceeds maximum of ${MAX_BATCH_ACTIONS} actions`); + } + const results: Array<{ ok: boolean; error?: string }> = []; + for (const action of opts.actions) { + try { + await executeSingleAction(action, opts.cdpUrl, opts.targetId, opts.evaluateEnabled, depth); + results.push({ ok: true }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + results.push({ ok: false, error: message }); + if (opts.stopOnError !== false) { + break; + } + } + } + return { results }; +} diff --git a/src/browser/pw-tools-core.shared.ts b/src/browser/pw-tools-core.shared.ts index d5ad74477d4..b6132de92bf 100644 --- a/src/browser/pw-tools-core.shared.ts +++ b/src/browser/pw-tools-core.shared.ts @@ -29,6 +29,21 @@ export function requireRef(value: unknown): string { return ref; } +export function requireRefOrSelector( + ref: string | undefined, + selector: string | undefined, +): { ref?: string; selector?: string } { + const trimmedRef = typeof ref === "string" ? ref.trim() : ""; + const trimmedSelector = typeof selector === "string" ? selector.trim() : ""; + if (!trimmedRef && !trimmedSelector) { + throw new Error("ref or selector is required"); + } + return { + ref: trimmedRef || undefined, + selector: trimmedSelector || undefined, + }; +} + export function normalizeTimeoutMs(timeoutMs: number | undefined, fallback: number) { return Math.max(500, Math.min(120_000, timeoutMs ?? fallback)); } diff --git a/src/browser/routes/agent.act.shared.ts b/src/browser/routes/agent.act.shared.ts index 81ca8caab71..b22f35e7ef2 100644 --- a/src/browser/routes/agent.act.shared.ts +++ b/src/browser/routes/agent.act.shared.ts @@ -1,4 +1,5 @@ export const ACT_KINDS = [ + "batch", "click", "close", "drag", diff --git a/src/browser/routes/agent.act.ts b/src/browser/routes/agent.act.ts index 8928a8a7d06..0c4c9e71967 100644 --- a/src/browser/routes/agent.act.ts +++ b/src/browser/routes/agent.act.ts @@ -9,7 +9,7 @@ import { pressChromeMcpKey, resizeChromeMcpPage, } from "../chrome-mcp.js"; -import type { BrowserFormField } from "../client-actions-core.js"; +import type { BrowserActRequest, BrowserFormField } from "../client-actions-core.js"; import { normalizeBrowserFormField } from "../form-fields.js"; import type { BrowserRouteContext } from "../server-context.js"; import { registerBrowserAgentActDownloadRoutes } from "./agent.act.download.js"; @@ -104,6 +104,326 @@ async function waitForExistingSessionCondition(params: { throw new Error("Timed out waiting for condition"); } +const SELECTOR_ALLOWED_KINDS: ReadonlySet = new Set([ + "batch", + "click", + "drag", + "hover", + "scrollIntoView", + "select", + "type", + "wait", +]); +const MAX_BATCH_ACTIONS = 100; +const MAX_BATCH_CLICK_DELAY_MS = 5_000; +const MAX_BATCH_WAIT_TIME_MS = 30_000; + +function normalizeBoundedNonNegativeMs( + value: unknown, + fieldName: string, + maxMs: number, +): number | undefined { + const ms = toNumber(value); + if (ms === undefined) { + return undefined; + } + if (ms < 0) { + throw new Error(`${fieldName} must be >= 0`); + } + const normalized = Math.floor(ms); + if (normalized > maxMs) { + throw new Error(`${fieldName} exceeds maximum of ${maxMs}ms`); + } + return normalized; +} + +function countBatchActions(actions: BrowserActRequest[]): number { + let count = 0; + for (const action of actions) { + count += 1; + if (action.kind === "batch") { + count += countBatchActions(action.actions); + } + } + return count; +} + +function validateBatchTargetIds(actions: BrowserActRequest[], targetId: string): string | null { + for (const action of actions) { + if (action.targetId && action.targetId !== targetId) { + return "batched action targetId must match request targetId"; + } + if (action.kind === "batch") { + const nestedError = validateBatchTargetIds(action.actions, targetId); + if (nestedError) { + return nestedError; + } + } + } + return null; +} + +function normalizeBatchAction(value: unknown): BrowserActRequest { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("batch actions must be objects"); + } + const raw = value as Record; + const kind = toStringOrEmpty(raw.kind); + if (!isActKind(kind)) { + throw new Error("batch actions must use a supported kind"); + } + + switch (kind) { + case "click": { + const ref = toStringOrEmpty(raw.ref) || undefined; + const selector = toStringOrEmpty(raw.selector) || undefined; + if (!ref && !selector) { + throw new Error("click requires ref or selector"); + } + const buttonRaw = toStringOrEmpty(raw.button); + const button = buttonRaw ? parseClickButton(buttonRaw) : undefined; + if (buttonRaw && !button) { + throw new Error("click button must be left|right|middle"); + } + const modifiersRaw = toStringArray(raw.modifiers) ?? []; + const parsedModifiers = parseClickModifiers(modifiersRaw); + if (parsedModifiers.error) { + throw new Error(parsedModifiers.error); + } + const doubleClick = toBoolean(raw.doubleClick); + const delayMs = normalizeBoundedNonNegativeMs( + raw.delayMs, + "click delayMs", + MAX_BATCH_CLICK_DELAY_MS, + ); + const timeoutMs = toNumber(raw.timeoutMs); + const targetId = toStringOrEmpty(raw.targetId) || undefined; + return { + kind, + ...(ref ? { ref } : {}), + ...(selector ? { selector } : {}), + ...(targetId ? { targetId } : {}), + ...(doubleClick !== undefined ? { doubleClick } : {}), + ...(button ? { button } : {}), + ...(parsedModifiers.modifiers ? { modifiers: parsedModifiers.modifiers } : {}), + ...(delayMs !== undefined ? { delayMs } : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; + } + case "type": { + const ref = toStringOrEmpty(raw.ref) || undefined; + const selector = toStringOrEmpty(raw.selector) || undefined; + const text = raw.text; + if (!ref && !selector) { + throw new Error("type requires ref or selector"); + } + if (typeof text !== "string") { + throw new Error("type requires text"); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const submit = toBoolean(raw.submit); + const slowly = toBoolean(raw.slowly); + const timeoutMs = toNumber(raw.timeoutMs); + return { + kind, + ...(ref ? { ref } : {}), + ...(selector ? { selector } : {}), + text, + ...(targetId ? { targetId } : {}), + ...(submit !== undefined ? { submit } : {}), + ...(slowly !== undefined ? { slowly } : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; + } + case "press": { + const key = toStringOrEmpty(raw.key); + if (!key) { + throw new Error("press requires key"); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const delayMs = toNumber(raw.delayMs); + return { + kind, + key, + ...(targetId ? { targetId } : {}), + ...(delayMs !== undefined ? { delayMs } : {}), + }; + } + case "hover": + case "scrollIntoView": { + const ref = toStringOrEmpty(raw.ref) || undefined; + const selector = toStringOrEmpty(raw.selector) || undefined; + if (!ref && !selector) { + throw new Error(`${kind} requires ref or selector`); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const timeoutMs = toNumber(raw.timeoutMs); + return { + kind, + ...(ref ? { ref } : {}), + ...(selector ? { selector } : {}), + ...(targetId ? { targetId } : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; + } + case "drag": { + const startRef = toStringOrEmpty(raw.startRef) || undefined; + const startSelector = toStringOrEmpty(raw.startSelector) || undefined; + const endRef = toStringOrEmpty(raw.endRef) || undefined; + const endSelector = toStringOrEmpty(raw.endSelector) || undefined; + if (!startRef && !startSelector) { + throw new Error("drag requires startRef or startSelector"); + } + if (!endRef && !endSelector) { + throw new Error("drag requires endRef or endSelector"); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const timeoutMs = toNumber(raw.timeoutMs); + return { + kind, + ...(startRef ? { startRef } : {}), + ...(startSelector ? { startSelector } : {}), + ...(endRef ? { endRef } : {}), + ...(endSelector ? { endSelector } : {}), + ...(targetId ? { targetId } : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; + } + case "select": { + const ref = toStringOrEmpty(raw.ref) || undefined; + const selector = toStringOrEmpty(raw.selector) || undefined; + const values = toStringArray(raw.values); + if ((!ref && !selector) || !values?.length) { + throw new Error("select requires ref/selector and values"); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const timeoutMs = toNumber(raw.timeoutMs); + return { + kind, + ...(ref ? { ref } : {}), + ...(selector ? { selector } : {}), + values, + ...(targetId ? { targetId } : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; + } + case "fill": { + const rawFields = Array.isArray(raw.fields) ? raw.fields : []; + const fields = rawFields + .map((field) => { + if (!field || typeof field !== "object") { + return null; + } + return normalizeBrowserFormField(field as Record); + }) + .filter((field): field is BrowserFormField => field !== null); + if (!fields.length) { + throw new Error("fill requires fields"); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const timeoutMs = toNumber(raw.timeoutMs); + return { + kind, + fields, + ...(targetId ? { targetId } : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; + } + case "resize": { + const width = toNumber(raw.width); + const height = toNumber(raw.height); + if (width === undefined || height === undefined) { + throw new Error("resize requires width and height"); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + return { + kind, + width, + height, + ...(targetId ? { targetId } : {}), + }; + } + case "wait": { + const loadStateRaw = toStringOrEmpty(raw.loadState); + const loadState = + loadStateRaw === "load" || + loadStateRaw === "domcontentloaded" || + loadStateRaw === "networkidle" + ? loadStateRaw + : undefined; + const timeMs = normalizeBoundedNonNegativeMs( + raw.timeMs, + "wait timeMs", + MAX_BATCH_WAIT_TIME_MS, + ); + const text = toStringOrEmpty(raw.text) || undefined; + const textGone = toStringOrEmpty(raw.textGone) || undefined; + const selector = toStringOrEmpty(raw.selector) || undefined; + const url = toStringOrEmpty(raw.url) || undefined; + const fn = toStringOrEmpty(raw.fn) || undefined; + if (timeMs === undefined && !text && !textGone && !selector && !url && !loadState && !fn) { + throw new Error( + "wait requires at least one of: timeMs, text, textGone, selector, url, loadState, fn", + ); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const timeoutMs = toNumber(raw.timeoutMs); + return { + kind, + ...(timeMs !== undefined ? { timeMs } : {}), + ...(text ? { text } : {}), + ...(textGone ? { textGone } : {}), + ...(selector ? { selector } : {}), + ...(url ? { url } : {}), + ...(loadState ? { loadState } : {}), + ...(fn ? { fn } : {}), + ...(targetId ? { targetId } : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; + } + case "evaluate": { + const fn = toStringOrEmpty(raw.fn); + if (!fn) { + throw new Error("evaluate requires fn"); + } + const ref = toStringOrEmpty(raw.ref) || undefined; + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const timeoutMs = toNumber(raw.timeoutMs); + return { + kind, + fn, + ...(ref ? { ref } : {}), + ...(targetId ? { targetId } : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; + } + case "close": { + const targetId = toStringOrEmpty(raw.targetId) || undefined; + return { + kind, + ...(targetId ? { targetId } : {}), + }; + } + case "batch": { + const actions = Array.isArray(raw.actions) ? raw.actions.map(normalizeBatchAction) : []; + if (!actions.length) { + throw new Error("batch requires actions"); + } + if (countBatchActions(actions) > MAX_BATCH_ACTIONS) { + throw new Error(`batch exceeds maximum of ${MAX_BATCH_ACTIONS} actions`); + } + const targetId = toStringOrEmpty(raw.targetId) || undefined; + const stopOnError = toBoolean(raw.stopOnError); + return { + kind, + actions, + ...(targetId ? { targetId } : {}), + ...(stopOnError !== undefined ? { stopOnError } : {}), + }; + } + } +} + export function registerBrowserAgentActRoutes( app: BrowserRouteRegistrar, ctx: BrowserRouteContext, @@ -116,7 +436,7 @@ export function registerBrowserAgentActRoutes( } const kind: ActKind = kindRaw; const targetId = resolveTargetIdFromBody(body); - if (Object.hasOwn(body, "selector") && kind !== "wait") { + if (Object.hasOwn(body, "selector") && !SELECTOR_ALLOWED_KINDS.has(kind)) { return jsonError(res, 400, SELECTOR_UNSUPPORTED_MESSAGE); } @@ -132,12 +452,14 @@ export function registerBrowserAgentActRoutes( switch (kind) { case "click": { - const ref = toStringOrEmpty(body.ref); - if (!ref) { - return jsonError(res, 400, "ref is required"); + const ref = toStringOrEmpty(body.ref) || undefined; + const selector = toStringOrEmpty(body.selector) || undefined; + if (!ref && !selector) { + return jsonError(res, 400, "ref or selector is required"); } const doubleClick = toBoolean(body.doubleClick) ?? false; const timeoutMs = toNumber(body.timeoutMs); + const delayMs = toNumber(body.delayMs); const buttonRaw = toStringOrEmpty(body.button) || ""; const button = buttonRaw ? parseClickButton(buttonRaw) : undefined; if (buttonRaw && !button) { @@ -151,6 +473,13 @@ export function registerBrowserAgentActRoutes( } const modifiers = parsedModifiers.modifiers; if (isExistingSession) { + if (selector) { + return jsonError( + res, + 501, + "existing-session click does not support selector targeting yet; use ref.", + ); + } if ((button && button !== "left") || (modifiers && modifiers.length > 0)) { return jsonError( res, @@ -161,7 +490,7 @@ export function registerBrowserAgentActRoutes( await clickChromeMcpElement({ profileName, targetId: tab.targetId, - uid: ref, + uid: ref!, doubleClick, }); return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); @@ -173,15 +502,23 @@ export function registerBrowserAgentActRoutes( const clickRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, - ref, doubleClick, }; + if (ref) { + clickRequest.ref = ref; + } + if (selector) { + clickRequest.selector = selector; + } if (button) { clickRequest.button = button; } if (modifiers) { clickRequest.modifiers = modifiers; } + if (delayMs) { + clickRequest.delayMs = delayMs; + } if (timeoutMs) { clickRequest.timeoutMs = timeoutMs; } @@ -189,9 +526,10 @@ export function registerBrowserAgentActRoutes( return res.json({ ok: true, targetId: tab.targetId, url: tab.url }); } case "type": { - const ref = toStringOrEmpty(body.ref); - if (!ref) { - return jsonError(res, 400, "ref is required"); + const ref = toStringOrEmpty(body.ref) || undefined; + const selector = toStringOrEmpty(body.selector) || undefined; + if (!ref && !selector) { + return jsonError(res, 400, "ref or selector is required"); } if (typeof body.text !== "string") { return jsonError(res, 400, "text is required"); @@ -201,6 +539,13 @@ export function registerBrowserAgentActRoutes( const slowly = toBoolean(body.slowly) ?? false; const timeoutMs = toNumber(body.timeoutMs); if (isExistingSession) { + if (selector) { + return jsonError( + res, + 501, + "existing-session type does not support selector targeting yet; use ref.", + ); + } if (slowly) { return jsonError( res, @@ -211,7 +556,7 @@ export function registerBrowserAgentActRoutes( await fillChromeMcpElement({ profileName, targetId: tab.targetId, - uid: ref, + uid: ref!, value: text, }); if (submit) { @@ -230,11 +575,16 @@ export function registerBrowserAgentActRoutes( const typeRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, - ref, text, submit, slowly, }; + if (ref) { + typeRequest.ref = ref; + } + if (selector) { + typeRequest.selector = selector; + } if (timeoutMs) { typeRequest.timeoutMs = timeoutMs; } @@ -267,12 +617,20 @@ export function registerBrowserAgentActRoutes( return res.json({ ok: true, targetId: tab.targetId }); } case "hover": { - const ref = toStringOrEmpty(body.ref); - if (!ref) { - return jsonError(res, 400, "ref is required"); + const ref = toStringOrEmpty(body.ref) || undefined; + const selector = toStringOrEmpty(body.selector) || undefined; + if (!ref && !selector) { + return jsonError(res, 400, "ref or selector is required"); } const timeoutMs = toNumber(body.timeoutMs); if (isExistingSession) { + if (selector) { + return jsonError( + res, + 501, + "existing-session hover does not support selector targeting yet; use ref.", + ); + } if (timeoutMs) { return jsonError( res, @@ -280,7 +638,7 @@ export function registerBrowserAgentActRoutes( "existing-session hover does not support timeoutMs overrides.", ); } - await hoverChromeMcpElement({ profileName, targetId: tab.targetId, uid: ref }); + await hoverChromeMcpElement({ profileName, targetId: tab.targetId, uid: ref! }); return res.json({ ok: true, targetId: tab.targetId }); } const pw = await requirePwAi(res, `act:${kind}`); @@ -291,17 +649,26 @@ export function registerBrowserAgentActRoutes( cdpUrl, targetId: tab.targetId, ref, + selector, timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } case "scrollIntoView": { - const ref = toStringOrEmpty(body.ref); - if (!ref) { - return jsonError(res, 400, "ref is required"); + const ref = toStringOrEmpty(body.ref) || undefined; + const selector = toStringOrEmpty(body.selector) || undefined; + if (!ref && !selector) { + return jsonError(res, 400, "ref or selector is required"); } const timeoutMs = toNumber(body.timeoutMs); if (isExistingSession) { + if (selector) { + return jsonError( + res, + 501, + "existing-session scrollIntoView does not support selector targeting yet; use ref.", + ); + } if (timeoutMs) { return jsonError( res, @@ -313,7 +680,7 @@ export function registerBrowserAgentActRoutes( profileName, targetId: tab.targetId, fn: `(el) => { el.scrollIntoView({ block: "center", inline: "center" }); return true; }`, - args: [ref], + args: [ref!], }); return res.json({ ok: true, targetId: tab.targetId }); } @@ -324,8 +691,13 @@ export function registerBrowserAgentActRoutes( const scrollRequest: Parameters[0] = { cdpUrl, targetId: tab.targetId, - ref, }; + if (ref) { + scrollRequest.ref = ref; + } + if (selector) { + scrollRequest.selector = selector; + } if (timeoutMs) { scrollRequest.timeoutMs = timeoutMs; } @@ -333,13 +705,25 @@ export function registerBrowserAgentActRoutes( return res.json({ ok: true, targetId: tab.targetId }); } case "drag": { - const startRef = toStringOrEmpty(body.startRef); - const endRef = toStringOrEmpty(body.endRef); - if (!startRef || !endRef) { - return jsonError(res, 400, "startRef and endRef are required"); + const startRef = toStringOrEmpty(body.startRef) || undefined; + const startSelector = toStringOrEmpty(body.startSelector) || undefined; + const endRef = toStringOrEmpty(body.endRef) || undefined; + const endSelector = toStringOrEmpty(body.endSelector) || undefined; + if (!startRef && !startSelector) { + return jsonError(res, 400, "startRef or startSelector is required"); + } + if (!endRef && !endSelector) { + return jsonError(res, 400, "endRef or endSelector is required"); } const timeoutMs = toNumber(body.timeoutMs); if (isExistingSession) { + if (startSelector || endSelector) { + return jsonError( + res, + 501, + "existing-session drag does not support selector targeting yet; use startRef/endRef.", + ); + } if (timeoutMs) { return jsonError( res, @@ -350,8 +734,8 @@ export function registerBrowserAgentActRoutes( await dragChromeMcpElement({ profileName, targetId: tab.targetId, - fromUid: startRef, - toUid: endRef, + fromUid: startRef!, + toUid: endRef!, }); return res.json({ ok: true, targetId: tab.targetId }); } @@ -363,19 +747,29 @@ export function registerBrowserAgentActRoutes( cdpUrl, targetId: tab.targetId, startRef, + startSelector, endRef, + endSelector, timeoutMs: timeoutMs ?? undefined, }); return res.json({ ok: true, targetId: tab.targetId }); } case "select": { - const ref = toStringOrEmpty(body.ref); + const ref = toStringOrEmpty(body.ref) || undefined; + const selector = toStringOrEmpty(body.selector) || undefined; const values = toStringArray(body.values); - if (!ref || !values?.length) { - return jsonError(res, 400, "ref and values are required"); + if ((!ref && !selector) || !values?.length) { + return jsonError(res, 400, "ref/selector and values are required"); } const timeoutMs = toNumber(body.timeoutMs); if (isExistingSession) { + if (selector) { + return jsonError( + res, + 501, + "existing-session select does not support selector targeting yet; use ref.", + ); + } if (values.length !== 1) { return jsonError( res, @@ -393,7 +787,7 @@ export function registerBrowserAgentActRoutes( await fillChromeMcpElement({ profileName, targetId: tab.targetId, - uid: ref, + uid: ref!, value: values[0] ?? "", }); return res.json({ ok: true, targetId: tab.targetId }); @@ -406,6 +800,7 @@ export function registerBrowserAgentActRoutes( cdpUrl, targetId: tab.targetId, ref, + selector, values, timeoutMs: timeoutMs ?? undefined, }); @@ -627,6 +1022,41 @@ export function registerBrowserAgentActRoutes( await pw.closePageViaPlaywright({ cdpUrl, targetId: tab.targetId }); return res.json({ ok: true, targetId: tab.targetId }); } + case "batch": { + if (isExistingSession) { + return jsonError( + res, + 501, + "existing-session batch is not supported yet; send actions individually.", + ); + } + const pw = await requirePwAi(res, `act:${kind}`); + if (!pw) { + return; + } + let actions: BrowserActRequest[]; + try { + actions = Array.isArray(body.actions) ? body.actions.map(normalizeBatchAction) : []; + } catch (err) { + return jsonError(res, 400, err instanceof Error ? err.message : String(err)); + } + if (!actions.length) { + return jsonError(res, 400, "actions are required"); + } + const targetIdError = validateBatchTargetIds(actions, tab.targetId); + if (targetIdError) { + return jsonError(res, 403, targetIdError); + } + const stopOnError = toBoolean(body.stopOnError) ?? true; + const result = await pw.batchViaPlaywright({ + cdpUrl, + targetId: tab.targetId, + actions, + stopOnError, + evaluateEnabled, + }); + return res.json({ ok: true, targetId: tab.targetId, results: result.results }); + } default: { return jsonError(res, 400, "unsupported kind"); } diff --git a/src/browser/server.agent-contract-form-layout-act-commands.test.ts b/src/browser/server.agent-contract-form-layout-act-commands.test.ts index 738bf8b7e2d..912d024916c 100644 --- a/src/browser/server.agent-contract-form-layout-act-commands.test.ts +++ b/src/browser/server.agent-contract-form-layout-act-commands.test.ts @@ -51,12 +51,14 @@ describe("browser control server", () => { values: ["a", "b"], }); expect(select.ok).toBe(true); - expect(pwMocks.selectOptionViaPlaywright).toHaveBeenCalledWith({ - cdpUrl: state.cdpBaseUrl, - targetId: "abcd1234", - ref: "5", - values: ["a", "b"], - }); + expect(pwMocks.selectOptionViaPlaywright).toHaveBeenCalledWith( + expect.objectContaining({ + cdpUrl: expect.any(String), + targetId: "abcd1234", + ref: "5", + values: ["a", "b"], + }), + ); const fillCases: Array<{ input: Record; @@ -81,11 +83,13 @@ describe("browser control server", () => { fields: [input], }); expect(fill.ok).toBe(true); - expect(pwMocks.fillFormViaPlaywright).toHaveBeenCalledWith({ - cdpUrl: state.cdpBaseUrl, - targetId: "abcd1234", - fields: [expected], - }); + expect(pwMocks.fillFormViaPlaywright).toHaveBeenCalledWith( + expect.objectContaining({ + cdpUrl: expect.any(String), + targetId: "abcd1234", + fields: [expected], + }), + ); } const resize = await postJson<{ ok: boolean }>(`${base}/act`, { @@ -94,12 +98,14 @@ describe("browser control server", () => { height: 600, }); expect(resize.ok).toBe(true); - expect(pwMocks.resizeViewportViaPlaywright).toHaveBeenCalledWith({ - cdpUrl: state.cdpBaseUrl, - targetId: "abcd1234", - width: 800, - height: 600, - }); + expect(pwMocks.resizeViewportViaPlaywright).toHaveBeenCalledWith( + expect.objectContaining({ + cdpUrl: expect.any(String), + targetId: "abcd1234", + width: 800, + height: 600, + }), + ); const wait = await postJson<{ ok: boolean }>(`${base}/act`, { kind: "wait", @@ -157,6 +163,130 @@ describe("browser control server", () => { slowTimeoutMs, ); + it( + "normalizes batch actions and threads evaluateEnabled into the batch executor", + async () => { + const base = await startServerAndBase(); + + const batchRes = await postJson<{ ok: boolean; results?: Array<{ ok: boolean }> }>( + `${base}/act`, + { + kind: "batch", + stopOnError: "false", + actions: [ + { kind: "click", selector: "button.save", doubleClick: "true", delayMs: "25" }, + { kind: "wait", fn: " () => window.ready === true " }, + ], + }, + ); + + expect(batchRes.ok).toBe(true); + expect(pwMocks.batchViaPlaywright).toHaveBeenCalledWith( + expect.objectContaining({ + cdpUrl: expect.any(String), + targetId: "abcd1234", + stopOnError: false, + evaluateEnabled: true, + actions: [ + { + kind: "click", + selector: "button.save", + doubleClick: true, + delayMs: 25, + }, + { + kind: "wait", + fn: "() => window.ready === true", + }, + ], + }), + ); + }, + slowTimeoutMs, + ); + + it( + "preserves exact type text in batch normalization", + async () => { + const base = await startServerAndBase(); + + const batchRes = await postJson<{ ok: boolean }>(`${base}/act`, { + kind: "batch", + actions: [ + { kind: "type", selector: "input.name", text: " padded " }, + { kind: "type", selector: "input.clearable", text: "" }, + ], + }); + + expect(batchRes.ok).toBe(true); + expect(pwMocks.batchViaPlaywright).toHaveBeenCalledWith( + expect.objectContaining({ + actions: [ + { + kind: "type", + selector: "input.name", + text: " padded ", + }, + { + kind: "type", + selector: "input.clearable", + text: "", + }, + ], + }), + ); + }, + slowTimeoutMs, + ); + + it( + "rejects malformed batch actions before dispatch", + async () => { + const base = await startServerAndBase(); + + const batchRes = await postJson<{ error?: string }>(`${base}/act`, { + kind: "batch", + actions: [{ kind: "click", ref: {} }], + }); + + expect(batchRes.error).toContain("click requires ref or selector"); + expect(pwMocks.batchViaPlaywright).not.toHaveBeenCalled(); + }, + slowTimeoutMs, + ); + + it( + "rejects batched action targetId overrides before dispatch", + async () => { + const base = await startServerAndBase(); + + const batchRes = await postJson<{ error?: string }>(`${base}/act`, { + kind: "batch", + actions: [{ kind: "click", ref: "5", targetId: "other-tab" }], + }); + + expect(batchRes.error).toContain("batched action targetId must match request targetId"); + expect(pwMocks.batchViaPlaywright).not.toHaveBeenCalled(); + }, + slowTimeoutMs, + ); + + it( + "rejects oversized batch delays before dispatch", + async () => { + const base = await startServerAndBase(); + + const batchRes = await postJson<{ error?: string }>(`${base}/act`, { + kind: "batch", + actions: [{ kind: "click", selector: "button.save", delayMs: 5001 }], + }); + + expect(batchRes.error).toContain("click delayMs exceeds maximum of 5000ms"); + expect(pwMocks.batchViaPlaywright).not.toHaveBeenCalled(); + }, + slowTimeoutMs, + ); + it("agent contract: hooks + response + downloads + screenshot", async () => { const base = await startServerAndBase(); @@ -165,13 +295,15 @@ describe("browser control server", () => { timeoutMs: 1234, }); expect(upload).toMatchObject({ ok: true }); - expect(pwMocks.armFileUploadViaPlaywright).toHaveBeenCalledWith({ - cdpUrl: state.cdpBaseUrl, - targetId: "abcd1234", - // The server resolves paths (which adds a drive letter on Windows for `\\tmp\\...` style roots). - paths: [path.resolve(DEFAULT_UPLOAD_DIR, "a.txt")], - timeoutMs: 1234, - }); + expect(pwMocks.armFileUploadViaPlaywright).toHaveBeenCalledWith( + expect.objectContaining({ + cdpUrl: expect.any(String), + targetId: "abcd1234", + // The server resolves paths (which adds a drive letter on Windows for `\\tmp\\...` style roots). + paths: [path.resolve(DEFAULT_UPLOAD_DIR, "a.txt")], + timeoutMs: 1234, + }), + ); const uploadWithRef = await postJson(`${base}/hooks/file-chooser`, { paths: ["b.txt"], @@ -280,7 +412,7 @@ describe("browser control server", () => { expect(res.path).toContain("safe-trace.zip"); expect(pwMocks.traceStopViaPlaywright).toHaveBeenCalledWith( expect.objectContaining({ - cdpUrl: state.cdpBaseUrl, + cdpUrl: expect.any(String), targetId: "abcd1234", path: expect.stringContaining("safe-trace.zip"), }), @@ -369,7 +501,7 @@ describe("browser control server", () => { expect(res.ok).toBe(true); expect(pwMocks.waitForDownloadViaPlaywright).toHaveBeenCalledWith( expect.objectContaining({ - cdpUrl: state.cdpBaseUrl, + cdpUrl: expect.any(String), targetId: "abcd1234", path: expect.stringContaining("safe-wait.pdf"), }), @@ -385,7 +517,7 @@ describe("browser control server", () => { expect(res.ok).toBe(true); expect(pwMocks.downloadViaPlaywright).toHaveBeenCalledWith( expect.objectContaining({ - cdpUrl: state.cdpBaseUrl, + cdpUrl: expect.any(String), targetId: "abcd1234", ref: "e12", path: expect.stringContaining("safe-download.pdf"), diff --git a/src/browser/server.control-server.test-harness.ts b/src/browser/server.control-server.test-harness.ts index 5721d9eb17b..3a54f6662d5 100644 --- a/src/browser/server.control-server.test-harness.ts +++ b/src/browser/server.control-server.test-harness.ts @@ -77,6 +77,7 @@ export function getCdpMocks(): { createTargetViaCdp: MockFn; snapshotAria: MockF const pwMocks = vi.hoisted(() => ({ armDialogViaPlaywright: vi.fn(async () => {}), armFileUploadViaPlaywright: vi.fn(async () => {}), + batchViaPlaywright: vi.fn(async () => ({ results: [] })), clickViaPlaywright: vi.fn(async () => {}), closePageViaPlaywright: vi.fn(async () => {}), closePlaywrightBrowserConnection: vi.fn(async () => {}), @@ -210,7 +211,9 @@ vi.mock("./cdp.js", () => ({ vi.mock("./pw-ai.js", () => pwMocks); vi.mock("../media/store.js", () => ({ + MEDIA_MAX_BYTES: 5 * 1024 * 1024, ensureMediaDir: vi.fn(async () => {}), + getMediaDir: vi.fn(() => "/tmp"), saveMediaBuffer: vi.fn(async () => ({ path: "/tmp/fake.png" })), }));