diff --git a/CHANGELOG.md b/CHANGELOG.md index 27e80e5c39b..bac756acc1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Browser/existing-session: harden driver validation and session lifecycle so transport errors trigger reconnects while tool-level errors preserve the session, and extract shared ARIA role sets to deduplicate Playwright and Chrome MCP snapshot paths. (#45682) Thanks @odysseus0. - Dashboard/chat UI: stop reloading full chat history on every live tool result in dashboard v2 so tool-heavy runs no longer trigger UI freeze/re-render storms while the final event still refreshes persisted history. (#45541) Thanks @BunsDev. - Ollama/reasoning visibility: stop promoting native `thinking` and `reasoning` fields into final assistant text so local reasoning models no longer leak internal thoughts in normal replies. (#45330) Thanks @xi7ang. - Android/onboarding QR scan: switch setup QR scanning to Google Code Scanner so onboarding uses a more reliable scanner instead of the legacy embedded ZXing flow. (#45021) Thanks @obviyus. diff --git a/src/browser/chrome-mcp.snapshot.ts b/src/browser/chrome-mcp.snapshot.ts index e92709df6f2..f0a1413736a 100644 --- a/src/browser/chrome-mcp.snapshot.ts +++ b/src/browser/chrome-mcp.snapshot.ts @@ -4,6 +4,7 @@ import { type RoleRefMap, type RoleSnapshotOptions, } from "./pw-role-snapshot.js"; +import { CONTENT_ROLES, INTERACTIVE_ROLES, STRUCTURAL_ROLES } from "./snapshot-roles.js"; export type ChromeMcpSnapshotNode = { id?: string; @@ -14,60 +15,6 @@ export type ChromeMcpSnapshotNode = { children?: ChromeMcpSnapshotNode[]; }; -const INTERACTIVE_ROLES = new Set([ - "button", - "checkbox", - "combobox", - "link", - "listbox", - "menuitem", - "menuitemcheckbox", - "menuitemradio", - "option", - "radio", - "searchbox", - "slider", - "spinbutton", - "switch", - "tab", - "textbox", - "treeitem", -]); - -const CONTENT_ROLES = new Set([ - "article", - "cell", - "columnheader", - "gridcell", - "heading", - "listitem", - "main", - "navigation", - "region", - "rowheader", -]); - -const STRUCTURAL_ROLES = new Set([ - "application", - "directory", - "document", - "generic", - "group", - "ignored", - "list", - "menu", - "menubar", - "none", - "presentation", - "row", - "rowgroup", - "tablist", - "table", - "toolbar", - "tree", - "treegrid", -]); - function normalizeRole(node: ChromeMcpSnapshotNode): string { const role = typeof node.role === "string" ? node.role.trim().toLowerCase() : ""; return role || "generic"; diff --git a/src/browser/chrome-mcp.test.ts b/src/browser/chrome-mcp.test.ts index b6fe0a22f12..a77149d7a72 100644 --- a/src/browser/chrome-mcp.test.ts +++ b/src/browser/chrome-mcp.test.ts @@ -190,6 +190,66 @@ describe("chrome MCP page parsing", () => { expect(result).toBe(123); }); + it("preserves session after tool-level errors (isError)", async () => { + let factoryCalls = 0; + const factory: ChromeMcpSessionFactory = async () => { + factoryCalls += 1; + const session = createFakeSession(); + const callTool = vi.fn(async ({ name }: ToolCall) => { + if (name === "evaluate_script") { + return { + content: [{ type: "text", text: "element not found" }], + isError: true, + }; + } + if (name === "list_pages") { + return { + content: [{ type: "text", text: "## Pages\n1: https://example.com [selected]" }], + }; + } + throw new Error(`unexpected tool ${name}`); + }); + session.client.callTool = callTool as typeof session.client.callTool; + return session; + }; + setChromeMcpSessionFactoryForTest(factory); + + // First call: tool error (isError: true) — should NOT destroy session + await expect( + evaluateChromeMcpScript({ profileName: "chrome-live", targetId: "1", fn: "() => null" }), + ).rejects.toThrow(/element not found/); + + // Second call: should reuse the same session (factory called only once) + const tabs = await listChromeMcpTabs("chrome-live"); + expect(factoryCalls).toBe(1); + expect(tabs).toHaveLength(1); + }); + + it("destroys session on transport errors so next call reconnects", async () => { + let factoryCalls = 0; + const factory: ChromeMcpSessionFactory = async () => { + factoryCalls += 1; + const session = createFakeSession(); + if (factoryCalls === 1) { + // First session: transport error (callTool throws) + const callTool = vi.fn(async () => { + throw new Error("connection reset"); + }); + session.client.callTool = callTool as typeof session.client.callTool; + } + return session; + }; + setChromeMcpSessionFactoryForTest(factory); + + // First call: transport error — should destroy session + await expect(listChromeMcpTabs("chrome-live")).rejects.toThrow(/connection reset/); + + // Second call: should create a new session (factory called twice) + const tabs = await listChromeMcpTabs("chrome-live"); + expect(factoryCalls).toBe(2); + expect(tabs).toHaveLength(2); + }); + it("clears failed pending sessions so the next call can retry", async () => { let factoryCalls = 0; const factory: ChromeMcpSessionFactory = async () => { diff --git a/src/browser/chrome-mcp.ts b/src/browser/chrome-mcp.ts index e410cf886e9..25ae39b2293 100644 --- a/src/browser/chrome-mcp.ts +++ b/src/browser/chrome-mcp.ts @@ -248,20 +248,24 @@ async function callTool( args: Record = {}, ): Promise { const session = await getSession(profileName); + let result: ChromeMcpToolResult; try { - const result = (await session.client.callTool({ + result = (await session.client.callTool({ name, arguments: args, })) as ChromeMcpToolResult; - if (result.isError) { - throw new Error(extractToolErrorMessage(result, name)); - } - return result; } catch (err) { + // Transport/connection error — tear down session so it reconnects on next call sessions.delete(profileName); await session.client.close().catch(() => {}); throw err; } + // Tool-level errors (element not found, script error, etc.) don't indicate a + // broken connection — don't tear down the session for these. + if (result.isError) { + throw new Error(extractToolErrorMessage(result, name)); + } + return result; } async function withTempFile(fn: (filePath: string) => Promise): Promise { diff --git a/src/browser/config.test.ts b/src/browser/config.test.ts index d2643a6784b..ddaee1bb365 100644 --- a/src/browser/config.test.ts +++ b/src/browser/config.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { withEnv } from "../test-utils/env.js"; import { resolveBrowserConfig, resolveProfile, shouldStartLocalBrowserServer } from "./config.js"; +import { getBrowserProfileCapabilities } from "./profile-capabilities.js"; describe("browser config", () => { it("defaults to enabled with loopback defaults and lobster-orange color", () => { @@ -278,6 +279,47 @@ describe("browser config", () => { expect(resolved.ssrfPolicy).toEqual({}); }); + it("resolves existing-session profiles without cdpPort or cdpUrl", () => { + const resolved = resolveBrowserConfig({ + profiles: { + "chrome-live": { + driver: "existing-session", + attachOnly: true, + color: "#00AA00", + }, + }, + }); + const profile = resolveProfile(resolved, "chrome-live"); + expect(profile).not.toBeNull(); + expect(profile?.driver).toBe("existing-session"); + expect(profile?.attachOnly).toBe(true); + expect(profile?.cdpPort).toBe(0); + expect(profile?.cdpUrl).toBe(""); + expect(profile?.cdpIsLoopback).toBe(true); + expect(profile?.color).toBe("#00AA00"); + }); + + it("sets usesChromeMcp only for existing-session profiles", () => { + const resolved = resolveBrowserConfig({ + profiles: { + "chrome-live": { driver: "existing-session", attachOnly: true, color: "#00AA00" }, + work: { cdpPort: 18801, color: "#0066CC" }, + }, + }); + + const existingSession = resolveProfile(resolved, "chrome-live")!; + expect(getBrowserProfileCapabilities(existingSession).usesChromeMcp).toBe(true); + + const managed = resolveProfile(resolved, "openclaw")!; + expect(getBrowserProfileCapabilities(managed).usesChromeMcp).toBe(false); + + const extension = resolveProfile(resolved, "chrome")!; + expect(getBrowserProfileCapabilities(extension).usesChromeMcp).toBe(false); + + const work = resolveProfile(resolved, "work")!; + expect(getBrowserProfileCapabilities(work).usesChromeMcp).toBe(false); + }); + describe("default profile preference", () => { it("defaults to openclaw profile when defaultProfile is not configured", () => { const resolved = resolveBrowserConfig({ diff --git a/src/browser/config.ts b/src/browser/config.ts index 529ee791c40..898980de681 100644 --- a/src/browser/config.ts +++ b/src/browser/config.ts @@ -342,6 +342,20 @@ export function resolveProfile( ? "existing-session" : "openclaw"; + if (driver === "existing-session") { + // existing-session uses Chrome MCP auto-connect; no CDP port/URL needed + return { + name: profileName, + cdpPort: 0, + cdpUrl: "", + cdpHost: "", + cdpIsLoopback: true, + color: profile.color, + driver, + attachOnly: true, + }; + } + if (rawProfileUrl) { const parsed = parseHttpUrl(rawProfileUrl, `browser.profiles.${profileName}.cdpUrl`); cdpHost = parsed.parsed.hostname; @@ -361,7 +375,7 @@ export function resolveProfile( cdpIsLoopback: isLoopbackHost(cdpHost), color: profile.color, driver, - attachOnly: driver === "existing-session" ? true : (profile.attachOnly ?? resolved.attachOnly), + attachOnly: profile.attachOnly ?? resolved.attachOnly, }; } diff --git a/src/browser/profile-capabilities.ts b/src/browser/profile-capabilities.ts index 2bcf4f8fe9e..b736a77d943 100644 --- a/src/browser/profile-capabilities.ts +++ b/src/browser/profile-capabilities.ts @@ -9,6 +9,8 @@ export type BrowserProfileMode = export type BrowserProfileCapabilities = { mode: BrowserProfileMode; isRemote: boolean; + /** Profile uses the Chrome DevTools MCP server (existing-session driver). */ + usesChromeMcp: boolean; requiresRelay: boolean; requiresAttachedTab: boolean; usesPersistentPlaywright: boolean; @@ -25,6 +27,7 @@ export function getBrowserProfileCapabilities( return { mode: "local-extension-relay", isRemote: false, + usesChromeMcp: false, requiresRelay: true, requiresAttachedTab: true, usesPersistentPlaywright: false, @@ -39,6 +42,7 @@ export function getBrowserProfileCapabilities( return { mode: "local-existing-session", isRemote: false, + usesChromeMcp: true, requiresRelay: false, requiresAttachedTab: false, usesPersistentPlaywright: false, @@ -53,6 +57,7 @@ export function getBrowserProfileCapabilities( return { mode: "remote-cdp", isRemote: true, + usesChromeMcp: false, requiresRelay: false, requiresAttachedTab: false, usesPersistentPlaywright: true, @@ -66,6 +71,7 @@ export function getBrowserProfileCapabilities( return { mode: "local-managed", isRemote: false, + usesChromeMcp: false, requiresRelay: false, requiresAttachedTab: false, usesPersistentPlaywright: false, diff --git a/src/browser/profiles-service.test.ts b/src/browser/profiles-service.test.ts index f70e23ddb67..e3892347984 100644 --- a/src/browser/profiles-service.test.ts +++ b/src/browser/profiles-service.test.ts @@ -178,10 +178,9 @@ describe("BrowserProfilesService", () => { driver: "existing-session", }); - expect(result.cdpPort).toBe(18801); + expect(result.cdpPort).toBe(0); expect(result.isRemote).toBe(false); expect(state.resolved.profiles["chrome-live"]).toEqual({ - cdpPort: 18801, driver: "existing-session", attachOnly: true, color: expect.any(String), @@ -191,7 +190,6 @@ describe("BrowserProfilesService", () => { browser: expect.objectContaining({ profiles: expect.objectContaining({ "chrome-live": expect.objectContaining({ - cdpPort: 18801, driver: "existing-session", attachOnly: true, }), diff --git a/src/browser/profiles-service.ts b/src/browser/profiles-service.ts index 25c0461f795..0f6d4041f19 100644 --- a/src/browser/profiles-service.ts +++ b/src/browser/profiles-service.ts @@ -141,18 +141,26 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) { if (driver === "extension") { throw new BrowserValidationError("driver=extension requires an explicit loopback cdpUrl"); } - const usedPorts = getUsedPorts(resolvedProfiles); - const range = cdpPortRange(state.resolved); - const cdpPort = allocateCdpPort(usedPorts, range); - if (cdpPort === null) { - throw new BrowserResourceExhaustedError("no available CDP ports in range"); + if (driver === "existing-session") { + // existing-session uses Chrome MCP auto-connect; no CDP port needed + profileConfig = { + driver, + attachOnly: true, + color: profileColor, + }; + } else { + const usedPorts = getUsedPorts(resolvedProfiles); + const range = cdpPortRange(state.resolved); + const cdpPort = allocateCdpPort(usedPorts, range); + if (cdpPort === null) { + throw new BrowserResourceExhaustedError("no available CDP ports in range"); + } + profileConfig = { + cdpPort, + ...(driver ? { driver } : {}), + color: profileColor, + }; } - profileConfig = { - cdpPort, - ...(driver ? { driver } : {}), - ...(driver === "existing-session" ? { attachOnly: true } : {}), - color: profileColor, - }; } const nextConfig: OpenClawConfig = { diff --git a/src/browser/pw-role-snapshot.ts b/src/browser/pw-role-snapshot.ts index 7a0b0ae70fe..312abcf872f 100644 --- a/src/browser/pw-role-snapshot.ts +++ b/src/browser/pw-role-snapshot.ts @@ -1,3 +1,5 @@ +import { CONTENT_ROLES, INTERACTIVE_ROLES, STRUCTURAL_ROLES } from "./snapshot-roles.js"; + export type RoleRef = { role: string; name?: string; @@ -23,60 +25,6 @@ export type RoleSnapshotOptions = { compact?: boolean; }; -const INTERACTIVE_ROLES = new Set([ - "button", - "link", - "textbox", - "checkbox", - "radio", - "combobox", - "listbox", - "menuitem", - "menuitemcheckbox", - "menuitemradio", - "option", - "searchbox", - "slider", - "spinbutton", - "switch", - "tab", - "treeitem", -]); - -const CONTENT_ROLES = new Set([ - "heading", - "cell", - "gridcell", - "columnheader", - "rowheader", - "listitem", - "article", - "region", - "main", - "navigation", -]); - -const STRUCTURAL_ROLES = new Set([ - "generic", - "group", - "list", - "table", - "row", - "rowgroup", - "grid", - "treegrid", - "menu", - "menubar", - "toolbar", - "tablist", - "tree", - "directory", - "document", - "application", - "presentation", - "none", -]); - export function getRoleSnapshotStats(snapshot: string, refs: RoleRefMap): RoleSnapshotStats { const interactive = Object.values(refs).filter((r) => INTERACTIVE_ROLES.has(r.role)).length; return { diff --git a/src/browser/routes/agent.act.download.ts b/src/browser/routes/agent.act.download.ts index 9ed04469c26..cfdf1362797 100644 --- a/src/browser/routes/agent.act.download.ts +++ b/src/browser/routes/agent.act.download.ts @@ -1,3 +1,4 @@ +import { getBrowserProfileCapabilities } from "../profile-capabilities.js"; import type { BrowserRouteContext } from "../server-context.js"; import { readBody, @@ -34,7 +35,7 @@ export function registerBrowserAgentActDownloadRoutes( ctx, targetId, run: async ({ profileCtx, cdpUrl, tab }) => { - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { return jsonError( res, 501, @@ -88,7 +89,7 @@ export function registerBrowserAgentActDownloadRoutes( ctx, targetId, run: async ({ profileCtx, cdpUrl, tab }) => { - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { return jsonError( res, 501, diff --git a/src/browser/routes/agent.act.hooks.ts b/src/browser/routes/agent.act.hooks.ts index bb1f03b7a7c..a141a9cbe5a 100644 --- a/src/browser/routes/agent.act.hooks.ts +++ b/src/browser/routes/agent.act.hooks.ts @@ -1,4 +1,5 @@ import { evaluateChromeMcpScript, uploadChromeMcpFile } from "../chrome-mcp.js"; +import { getBrowserProfileCapabilities } from "../profile-capabilities.js"; import type { BrowserRouteContext } from "../server-context.js"; import { readBody, @@ -43,7 +44,7 @@ export function registerBrowserAgentActHookRoutes( } const resolvedPaths = uploadPathsResult.paths; - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { if (element) { return jsonError( res, @@ -123,7 +124,7 @@ export function registerBrowserAgentActHookRoutes( ctx, targetId, run: async ({ profileCtx, cdpUrl, tab }) => { - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { if (timeoutMs) { return jsonError( res, diff --git a/src/browser/routes/agent.act.ts b/src/browser/routes/agent.act.ts index ae18c044265..1b444d1b963 100644 --- a/src/browser/routes/agent.act.ts +++ b/src/browser/routes/agent.act.ts @@ -11,6 +11,7 @@ import { } from "../chrome-mcp.js"; import type { BrowserActRequest, BrowserFormField } from "../client-actions-core.js"; import { normalizeBrowserFormField } from "../form-fields.js"; +import { getBrowserProfileCapabilities } from "../profile-capabilities.js"; import type { BrowserRouteContext } from "../server-context.js"; import { matchBrowserUrlPattern } from "../url-pattern.js"; import { registerBrowserAgentActDownloadRoutes } from "./agent.act.download.js"; @@ -477,7 +478,7 @@ export function registerBrowserAgentActRoutes( targetId, run: async ({ profileCtx, cdpUrl, tab }) => { const evaluateEnabled = ctx.state().resolved.evaluateEnabled; - const isExistingSession = profileCtx.profile.driver === "existing-session"; + const isExistingSession = getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp; const profileName = profileCtx.profile.name; switch (kind) { @@ -1110,7 +1111,7 @@ export function registerBrowserAgentActRoutes( ctx, targetId, run: async ({ profileCtx, cdpUrl, tab }) => { - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { return jsonError( res, 501, @@ -1147,7 +1148,7 @@ export function registerBrowserAgentActRoutes( ctx, targetId, run: async ({ profileCtx, cdpUrl, tab }) => { - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { await evaluateChromeMcpScript({ profileName: profileCtx.profile.name, targetId: tab.targetId, diff --git a/src/browser/routes/agent.snapshot.ts b/src/browser/routes/agent.snapshot.ts index acddef9e5d7..80c11693a11 100644 --- a/src/browser/routes/agent.snapshot.ts +++ b/src/browser/routes/agent.snapshot.ts @@ -16,6 +16,7 @@ import { assertBrowserNavigationResultAllowed, } from "../navigation-guard.js"; import { withBrowserNavigationPolicy } from "../navigation-guard.js"; +import { getBrowserProfileCapabilities } from "../profile-capabilities.js"; import { DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES, DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE, @@ -225,7 +226,7 @@ export function registerBrowserAgentSnapshotRoutes( ctx, targetId, run: async ({ profileCtx, tab, cdpUrl }) => { - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { const ssrfPolicyOpts = withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy); await assertBrowserNavigationAllowed({ url, ...ssrfPolicyOpts }); const result = await navigateChromeMcpPage({ @@ -263,7 +264,7 @@ export function registerBrowserAgentSnapshotRoutes( if (!profileCtx) { return; } - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { return jsonError( res, 501, @@ -311,7 +312,7 @@ export function registerBrowserAgentSnapshotRoutes( ctx, targetId, run: async ({ profileCtx, tab, cdpUrl }) => { - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { if (element) { return jsonError( res, @@ -395,7 +396,7 @@ export function registerBrowserAgentSnapshotRoutes( if ((plan.labels || plan.mode === "efficient") && plan.format === "aria") { return jsonError(res, 400, "labels/mode=efficient require format=ai"); } - if (profileCtx.profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp) { if (plan.selectorValue || plan.frameSelectorValue) { return jsonError( res, diff --git a/src/browser/routes/basic.ts b/src/browser/routes/basic.ts index ff32decb681..63d0cb92b5c 100644 --- a/src/browser/routes/basic.ts +++ b/src/browser/routes/basic.ts @@ -1,6 +1,7 @@ import { getChromeMcpPid } from "../chrome-mcp.js"; import { resolveBrowserExecutableForPlatform } from "../chrome.executables.js"; import { toBrowserErrorResponse } from "../errors.js"; +import { getBrowserProfileCapabilities } from "../profile-capabilities.js"; import { createBrowserProfilesService } from "../profiles-service.js"; import type { BrowserRouteContext, ProfileContext } from "../server-context.js"; import { resolveProfileContext } from "./agent.shared.js"; @@ -100,10 +101,9 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow running: cdpReady, cdpReady, cdpHttp, - pid: - profileCtx.profile.driver === "existing-session" - ? getChromeMcpPid(profileCtx.profile.name) - : (profileState?.running?.pid ?? null), + pid: getBrowserProfileCapabilities(profileCtx.profile).usesChromeMcp + ? getChromeMcpPid(profileCtx.profile.name) + : (profileState?.running?.pid ?? null), cdpPort: profileCtx.profile.cdpPort, cdpUrl: profileCtx.profile.cdpUrl, chosenBrowser: profileState?.running?.exe.kind ?? null, diff --git a/src/browser/server-context.availability.ts b/src/browser/server-context.availability.ts index d2d9944d964..3b991bbbdfe 100644 --- a/src/browser/server-context.availability.ts +++ b/src/browser/server-context.availability.ts @@ -65,8 +65,8 @@ export function createProfileAvailability({ }); const isReachable = async (timeoutMs?: number) => { - if (profile.driver === "existing-session") { - await ensureChromeMcpAvailable(profile.name); + if (capabilities.usesChromeMcp) { + // listChromeMcpTabs creates the session if needed — no separate ensureChromeMcpAvailable call required await listChromeMcpTabs(profile.name); return true; } @@ -75,7 +75,7 @@ export function createProfileAvailability({ }; const isHttpReachable = async (timeoutMs?: number) => { - if (profile.driver === "existing-session") { + if (capabilities.usesChromeMcp) { return await isReachable(timeoutMs); } const { httpTimeoutMs } = resolveTimeouts(timeoutMs); @@ -122,7 +122,7 @@ export function createProfileAvailability({ if (previousProfile.driver === "extension") { await stopChromeExtensionRelayServer({ cdpUrl: previousProfile.cdpUrl }).catch(() => false); } - if (previousProfile.driver === "existing-session") { + if (getBrowserProfileCapabilities(previousProfile).usesChromeMcp) { await closeChromeMcpSession(previousProfile.name).catch(() => false); } await closePlaywrightBrowserConnectionForProfile(previousProfile.cdpUrl); @@ -154,7 +154,7 @@ export function createProfileAvailability({ const ensureBrowserAvailable = async (): Promise => { await reconcileProfileRuntime(); - if (profile.driver === "existing-session") { + if (capabilities.usesChromeMcp) { await ensureChromeMcpAvailable(profile.name); return; } @@ -258,7 +258,7 @@ export function createProfileAvailability({ const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => { await reconcileProfileRuntime(); - if (profile.driver === "existing-session") { + if (capabilities.usesChromeMcp) { const stopped = await closeChromeMcpSession(profile.name); return { stopped }; } diff --git a/src/browser/server-context.selection.ts b/src/browser/server-context.selection.ts index 9e1fb728b2a..f0ce3e25e06 100644 --- a/src/browser/server-context.selection.ts +++ b/src/browser/server-context.selection.ts @@ -112,7 +112,7 @@ export function createProfileSelectionOps({ const focusTab = async (targetId: string): Promise => { const resolvedTargetId = await resolveTargetIdOrThrow(targetId); - if (profile.driver === "existing-session") { + if (capabilities.usesChromeMcp) { await focusChromeMcpTab(profile.name, resolvedTargetId); const profileState = getProfileState(); profileState.lastTargetId = resolvedTargetId; @@ -142,7 +142,7 @@ export function createProfileSelectionOps({ const closeTab = async (targetId: string): Promise => { const resolvedTargetId = await resolveTargetIdOrThrow(targetId); - if (profile.driver === "existing-session") { + if (capabilities.usesChromeMcp) { await closeChromeMcpTab(profile.name, resolvedTargetId); return; } diff --git a/src/browser/server-context.tab-ops.ts b/src/browser/server-context.tab-ops.ts index 067536fd017..66a134564c6 100644 --- a/src/browser/server-context.tab-ops.ts +++ b/src/browser/server-context.tab-ops.ts @@ -66,7 +66,7 @@ export function createProfileTabOps({ const capabilities = getBrowserProfileCapabilities(profile); const listTabs = async (): Promise => { - if (profile.driver === "existing-session") { + if (capabilities.usesChromeMcp) { return await listChromeMcpTabs(profile.name); } @@ -139,7 +139,7 @@ export function createProfileTabOps({ const openTab = async (url: string): Promise => { const ssrfPolicyOpts = withBrowserNavigationPolicy(state().resolved.ssrfPolicy); - if (profile.driver === "existing-session") { + if (capabilities.usesChromeMcp) { await assertBrowserNavigationAllowed({ url, ...ssrfPolicyOpts }); const page = await openChromeMcpTab(profile.name, url); const profileState = getProfileState(); diff --git a/src/browser/server-context.ts b/src/browser/server-context.ts index 37e182f1e69..fe3810a855c 100644 --- a/src/browser/server-context.ts +++ b/src/browser/server-context.ts @@ -4,6 +4,7 @@ import type { ResolvedBrowserProfile } from "./config.js"; import { resolveProfile } from "./config.js"; import { BrowserProfileNotFoundError, toBrowserErrorResponse } from "./errors.js"; import { InvalidBrowserNavigationUrlError } from "./navigation-guard.js"; +import { getBrowserProfileCapabilities } from "./profile-capabilities.js"; import { refreshResolvedBrowserConfigFromDisk, resolveBrowserProfileWithHotReload, @@ -164,7 +165,7 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon let running = false; const profileCtx = createProfileContext(opts, profile); - if (profile.driver === "existing-session") { + if (getBrowserProfileCapabilities(profile).usesChromeMcp) { try { running = await profileCtx.isReachable(300); if (running) { diff --git a/src/browser/snapshot-roles.ts b/src/browser/snapshot-roles.ts new file mode 100644 index 00000000000..8e5d873e557 --- /dev/null +++ b/src/browser/snapshot-roles.ts @@ -0,0 +1,63 @@ +/** + * Shared ARIA role classification sets used by both the Playwright and Chrome MCP + * snapshot paths. Keep these in sync — divergence causes the two drivers to produce + * different snapshot output for the same page. + */ + +/** Roles that represent user-interactive elements and always get a ref. */ +export const INTERACTIVE_ROLES = new Set([ + "button", + "checkbox", + "combobox", + "link", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "searchbox", + "slider", + "spinbutton", + "switch", + "tab", + "textbox", + "treeitem", +]); + +/** Roles that carry meaningful content and get a ref when named. */ +export const CONTENT_ROLES = new Set([ + "article", + "cell", + "columnheader", + "gridcell", + "heading", + "listitem", + "main", + "navigation", + "region", + "rowheader", +]); + +/** Structural/container roles — typically skipped in compact mode. */ +export const STRUCTURAL_ROLES = new Set([ + "application", + "directory", + "document", + "generic", + "grid", + "group", + "ignored", + "list", + "menu", + "menubar", + "none", + "presentation", + "row", + "rowgroup", + "table", + "tablist", + "toolbar", + "tree", + "treegrid", +]); diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index 741b4bcc0c9..8c78d049d0e 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -371,9 +371,12 @@ export const OpenClawSchema = z color: HexColorSchema, }) .strict() - .refine((value) => value.cdpPort || value.cdpUrl, { - message: "Profile must set cdpPort or cdpUrl", - }), + .refine( + (value) => value.driver === "existing-session" || value.cdpPort || value.cdpUrl, + { + message: "Profile must set cdpPort or cdpUrl", + }, + ), ) .optional(), extraArgs: z.array(z.string()).optional(),