mirror of https://github.com/openclaw/openclaw.git
feat(browser): add chrome MCP existing-session support
This commit is contained in:
parent
9c52e1b7de
commit
593964560b
|
|
@ -9,9 +9,11 @@ Docs: https://docs.openclaw.ai
|
||||||
- Android/chat settings: redesign the chat settings sheet with grouped device and media sections, refresh the Connect and Voice tabs, and tighten the chat composer/session header for a denser mobile layout. (#44894) Thanks @obviyus.
|
- Android/chat settings: redesign the chat settings sheet with grouped device and media sections, refresh the Connect and Voice tabs, and tighten the chat composer/session header for a denser mobile layout. (#44894) Thanks @obviyus.
|
||||||
- Docker/timezone override: add `OPENCLAW_TZ` so `docker-setup.sh` can pin gateway and CLI containers to a chosen IANA timezone instead of inheriting the daemon default. (#34119) Thanks @Lanfei.
|
- Docker/timezone override: add `OPENCLAW_TZ` so `docker-setup.sh` can pin gateway and CLI containers to a chosen IANA timezone instead of inheriting the daemon default. (#34119) Thanks @Lanfei.
|
||||||
- iOS/onboarding: add a first-run welcome pager before gateway setup, stop auto-opening the QR scanner, and show `/pair qr` instructions on the connect step. (#45054) Thanks @ngutman.
|
- iOS/onboarding: add a first-run welcome pager before gateway setup, stop auto-opening the QR scanner, and show `/pair qr` instructions on the connect step. (#45054) Thanks @ngutman.
|
||||||
|
- Browser/existing-session: add an official Chrome DevTools MCP attach mode for signed-in live Chrome sessions, with docs for `chrome://inspect/#remote-debugging` enablement and direct backlinks to Chrome’s own setup guides.
|
||||||
|
|
||||||
### Fixes
|
### Fixes
|
||||||
|
|
||||||
|
- Browser/existing-session: accept text-only `list_pages` and `new_page` responses from Chrome DevTools MCP so live-session tab discovery and new-tab open flows keep working when the server omits structured page metadata.
|
||||||
- Ollama/reasoning visibility: stop promoting native `thinking` and `reasoning` fields into final assistant text so local reasoning models no longer leak internal thoughts in normal replies. (#45330) Thanks @xi7ang.
|
- Ollama/reasoning visibility: stop promoting native `thinking` and `reasoning` fields into final assistant text so local reasoning models no longer leak internal thoughts in normal replies. (#45330) Thanks @xi7ang.
|
||||||
- Windows/gateway install: bound `schtasks` calls and fall back to the Startup-folder login item when task creation hangs, so native `openclaw gateway install` fails fast instead of wedging forever on broken Scheduled Task setups.
|
- Windows/gateway install: bound `schtasks` calls and fall back to the Startup-folder login item when task creation hangs, so native `openclaw gateway install` fails fast instead of wedging forever on broken Scheduled Task setups.
|
||||||
- Windows/gateway auth: stop attaching device identity on local loopback shared-token and password gateway calls, so native Windows agent replies no longer log stale `device signature expired` fallback noise before succeeding.
|
- Windows/gateway auth: stop attaching device identity on local loopback shared-token and password gateway calls, so native Windows agent replies no longer log stale `device signature expired` fallback noise before succeeding.
|
||||||
|
|
|
||||||
|
|
@ -48,6 +48,8 @@ Gateway.
|
||||||
- `openclaw`: managed, isolated browser (no extension required).
|
- `openclaw`: managed, isolated browser (no extension required).
|
||||||
- `chrome`: extension relay to your **system browser** (requires the OpenClaw
|
- `chrome`: extension relay to your **system browser** (requires the OpenClaw
|
||||||
extension to be attached to a tab).
|
extension to be attached to a tab).
|
||||||
|
- `existing-session`: official Chrome MCP attach flow for a running Chrome
|
||||||
|
profile.
|
||||||
|
|
||||||
Set `browser.defaultProfile: "openclaw"` if you want managed mode by default.
|
Set `browser.defaultProfile: "openclaw"` if you want managed mode by default.
|
||||||
|
|
||||||
|
|
@ -77,6 +79,12 @@ Browser settings live in `~/.openclaw/openclaw.json`.
|
||||||
profiles: {
|
profiles: {
|
||||||
openclaw: { cdpPort: 18800, color: "#FF4500" },
|
openclaw: { cdpPort: 18800, color: "#FF4500" },
|
||||||
work: { cdpPort: 18801, color: "#0066CC" },
|
work: { cdpPort: 18801, color: "#0066CC" },
|
||||||
|
chromeLive: {
|
||||||
|
cdpPort: 18802,
|
||||||
|
driver: "existing-session",
|
||||||
|
attachOnly: true,
|
||||||
|
color: "#00AA00",
|
||||||
|
},
|
||||||
remote: { cdpUrl: "http://10.0.0.42:9222", color: "#00AA00" },
|
remote: { cdpUrl: "http://10.0.0.42:9222", color: "#00AA00" },
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
@ -100,6 +108,8 @@ Notes:
|
||||||
- Default profile is `openclaw` (OpenClaw-managed standalone browser). Use `defaultProfile: "chrome"` to opt into the Chrome extension relay.
|
- Default profile is `openclaw` (OpenClaw-managed standalone browser). Use `defaultProfile: "chrome"` to opt into the Chrome extension relay.
|
||||||
- Auto-detect order: system default browser if Chromium-based; otherwise Chrome → Brave → Edge → Chromium → Chrome Canary.
|
- Auto-detect order: system default browser if Chromium-based; otherwise Chrome → Brave → Edge → Chromium → Chrome Canary.
|
||||||
- Local `openclaw` profiles auto-assign `cdpPort`/`cdpUrl` — set those only for remote CDP.
|
- Local `openclaw` profiles auto-assign `cdpPort`/`cdpUrl` — set those only for remote CDP.
|
||||||
|
- `driver: "existing-session"` uses Chrome DevTools MCP instead of raw CDP. Do
|
||||||
|
not set `cdpUrl` for that driver.
|
||||||
|
|
||||||
## Use Brave (or another Chromium-based browser)
|
## Use Brave (or another Chromium-based browser)
|
||||||
|
|
||||||
|
|
@ -264,11 +274,13 @@ OpenClaw supports multiple named profiles (routing configs). Profiles can be:
|
||||||
- **openclaw-managed**: a dedicated Chromium-based browser instance with its own user data directory + CDP port
|
- **openclaw-managed**: a dedicated Chromium-based browser instance with its own user data directory + CDP port
|
||||||
- **remote**: an explicit CDP URL (Chromium-based browser running elsewhere)
|
- **remote**: an explicit CDP URL (Chromium-based browser running elsewhere)
|
||||||
- **extension relay**: your existing Chrome tab(s) via the local relay + Chrome extension
|
- **extension relay**: your existing Chrome tab(s) via the local relay + Chrome extension
|
||||||
|
- **existing session**: your existing Chrome profile via Chrome DevTools MCP auto-connect
|
||||||
|
|
||||||
Defaults:
|
Defaults:
|
||||||
|
|
||||||
- The `openclaw` profile is auto-created if missing.
|
- The `openclaw` profile is auto-created if missing.
|
||||||
- The `chrome` profile is built-in for the Chrome extension relay (points at `http://127.0.0.1:18792` by default).
|
- The `chrome` profile is built-in for the Chrome extension relay (points at `http://127.0.0.1:18792` by default).
|
||||||
|
- Existing-session profiles are opt-in; create them with `--driver existing-session`.
|
||||||
- Local CDP ports allocate from **18800–18899** by default.
|
- Local CDP ports allocate from **18800–18899** by default.
|
||||||
- Deleting a profile moves its local data directory to Trash.
|
- Deleting a profile moves its local data directory to Trash.
|
||||||
|
|
||||||
|
|
@ -328,6 +340,66 @@ Notes:
|
||||||
|
|
||||||
- This mode relies on Playwright-on-CDP for most operations (screenshots/snapshots/actions).
|
- This mode relies on Playwright-on-CDP for most operations (screenshots/snapshots/actions).
|
||||||
- Detach by clicking the extension icon again.
|
- Detach by clicking the extension icon again.
|
||||||
|
|
||||||
|
## Chrome existing-session via MCP
|
||||||
|
|
||||||
|
OpenClaw can also attach to a running Chrome profile through the official
|
||||||
|
Chrome DevTools MCP server. This reuses the tabs and login state already open in
|
||||||
|
that Chrome profile.
|
||||||
|
|
||||||
|
Official background and setup references:
|
||||||
|
|
||||||
|
- [Chrome for Developers: Use Chrome DevTools MCP with your browser session](https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session)
|
||||||
|
- [Chrome DevTools MCP README](https://github.com/ChromeDevTools/chrome-devtools-mcp)
|
||||||
|
|
||||||
|
Create a profile:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
openclaw browser create-profile \
|
||||||
|
--name chrome-live \
|
||||||
|
--driver existing-session \
|
||||||
|
--color "#00AA00"
|
||||||
|
```
|
||||||
|
|
||||||
|
Then in Chrome:
|
||||||
|
|
||||||
|
1. Open `chrome://inspect/#remote-debugging`
|
||||||
|
2. Enable remote debugging
|
||||||
|
3. Keep Chrome running and approve the connection prompt when OpenClaw attaches
|
||||||
|
|
||||||
|
Live attach smoke test:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
openclaw browser --browser-profile chrome-live start
|
||||||
|
openclaw browser --browser-profile chrome-live status
|
||||||
|
openclaw browser --browser-profile chrome-live tabs
|
||||||
|
openclaw browser --browser-profile chrome-live snapshot --format ai
|
||||||
|
```
|
||||||
|
|
||||||
|
What success looks like:
|
||||||
|
|
||||||
|
- `status` shows `driver: existing-session`
|
||||||
|
- `status` shows `running: true`
|
||||||
|
- `tabs` lists your already-open Chrome tabs
|
||||||
|
- `snapshot` returns refs from the selected live tab
|
||||||
|
|
||||||
|
What to check if attach does not work:
|
||||||
|
|
||||||
|
- Chrome is version `144+`
|
||||||
|
- remote debugging is enabled at `chrome://inspect/#remote-debugging`
|
||||||
|
- Chrome showed and you accepted the attach consent prompt
|
||||||
|
- the Gateway or node host can spawn `npx chrome-devtools-mcp@latest --autoConnect`
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
|
||||||
|
- This path is higher-risk than the isolated `openclaw` profile because it can
|
||||||
|
act inside your signed-in browser session.
|
||||||
|
- OpenClaw does not launch Chrome for this driver; it attaches to an existing
|
||||||
|
session only.
|
||||||
|
- OpenClaw uses the official Chrome DevTools MCP `--autoConnect` flow here, not
|
||||||
|
the legacy default-profile remote debugging port workflow.
|
||||||
|
- Some features still require the extension relay or managed browser path, such
|
||||||
|
as PDF export and download interception.
|
||||||
- Leave the relay loopback-only by default. If the relay must be reachable from a different network namespace (for example Gateway in WSL2, Chrome on Windows), set `browser.relayBindHost` to an explicit bind address such as `0.0.0.0` while keeping the surrounding network private and authenticated.
|
- Leave the relay loopback-only by default. If the relay must be reachable from a different network namespace (for example Gateway in WSL2, Chrome on Windows), set `browser.relayBindHost` to an explicit bind address such as `0.0.0.0` while keeping the surrounding network private and authenticated.
|
||||||
|
|
||||||
WSL2 / cross-namespace example:
|
WSL2 / cross-namespace example:
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,13 @@ The OpenClaw Chrome extension lets the agent control your **existing Chrome tabs
|
||||||
|
|
||||||
Attach/detach happens via a **single Chrome toolbar button**.
|
Attach/detach happens via a **single Chrome toolbar button**.
|
||||||
|
|
||||||
|
If you want Chrome’s official DevTools MCP attach flow instead of the OpenClaw
|
||||||
|
extension relay, use an `existing-session` browser profile instead. See
|
||||||
|
[Browser](/tools/browser#chrome-existing-session-via-mcp). For Chrome’s own
|
||||||
|
setup docs, see [Chrome for Developers: Use Chrome DevTools MCP with your
|
||||||
|
browser session](https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session)
|
||||||
|
and the [Chrome DevTools MCP README](https://github.com/ChromeDevTools/chrome-devtools-mcp).
|
||||||
|
|
||||||
## What it is (concept)
|
## What it is (concept)
|
||||||
|
|
||||||
There are three parts:
|
There are three parts:
|
||||||
|
|
|
||||||
|
|
@ -353,6 +353,7 @@
|
||||||
"@mariozechner/pi-ai": "0.57.1",
|
"@mariozechner/pi-ai": "0.57.1",
|
||||||
"@mariozechner/pi-coding-agent": "0.57.1",
|
"@mariozechner/pi-coding-agent": "0.57.1",
|
||||||
"@mariozechner/pi-tui": "0.57.1",
|
"@mariozechner/pi-tui": "0.57.1",
|
||||||
|
"@modelcontextprotocol/sdk": "1.27.1",
|
||||||
"@mozilla/readability": "^0.6.0",
|
"@mozilla/readability": "^0.6.0",
|
||||||
"@sinclair/typebox": "0.34.48",
|
"@sinclair/typebox": "0.34.48",
|
||||||
"@slack/bolt": "^4.6.0",
|
"@slack/bolt": "^4.6.0",
|
||||||
|
|
|
||||||
124
pnpm-lock.yaml
124
pnpm-lock.yaml
|
|
@ -60,16 +60,19 @@ importers:
|
||||||
version: 1.2.0-beta.3
|
version: 1.2.0-beta.3
|
||||||
'@mariozechner/pi-agent-core':
|
'@mariozechner/pi-agent-core':
|
||||||
specifier: 0.57.1
|
specifier: 0.57.1
|
||||||
version: 0.57.1(ws@8.19.0)(zod@4.3.6)
|
version: 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@mariozechner/pi-ai':
|
'@mariozechner/pi-ai':
|
||||||
specifier: 0.57.1
|
specifier: 0.57.1
|
||||||
version: 0.57.1(ws@8.19.0)(zod@4.3.6)
|
version: 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@mariozechner/pi-coding-agent':
|
'@mariozechner/pi-coding-agent':
|
||||||
specifier: 0.57.1
|
specifier: 0.57.1
|
||||||
version: 0.57.1(ws@8.19.0)(zod@4.3.6)
|
version: 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@mariozechner/pi-tui':
|
'@mariozechner/pi-tui':
|
||||||
specifier: 0.57.1
|
specifier: 0.57.1
|
||||||
version: 0.57.1
|
version: 0.57.1
|
||||||
|
'@modelcontextprotocol/sdk':
|
||||||
|
specifier: 1.27.1
|
||||||
|
version: 1.27.1(zod@4.3.6)
|
||||||
'@mozilla/readability':
|
'@mozilla/readability':
|
||||||
specifier: ^0.6.0
|
specifier: ^0.6.0
|
||||||
version: 0.6.0
|
version: 0.6.0
|
||||||
|
|
@ -346,7 +349,7 @@ importers:
|
||||||
version: 10.6.1
|
version: 10.6.1
|
||||||
openclaw:
|
openclaw:
|
||||||
specifier: '>=2026.3.11'
|
specifier: '>=2026.3.11'
|
||||||
version: 2026.3.11(@discordjs/opus@0.10.0)(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3))
|
version: 2026.3.11(@discordjs/opus@0.10.0)(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3))
|
||||||
|
|
||||||
extensions/imessage: {}
|
extensions/imessage: {}
|
||||||
|
|
||||||
|
|
@ -377,7 +380,7 @@ importers:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@mariozechner/pi-agent-core':
|
'@mariozechner/pi-agent-core':
|
||||||
specifier: 0.57.1
|
specifier: 0.57.1
|
||||||
version: 0.57.1(ws@8.19.0)(zod@4.3.6)
|
version: 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@matrix-org/matrix-sdk-crypto-nodejs':
|
'@matrix-org/matrix-sdk-crypto-nodejs':
|
||||||
specifier: ^0.4.0
|
specifier: ^0.4.0
|
||||||
version: 0.4.0
|
version: 0.4.0
|
||||||
|
|
@ -407,7 +410,7 @@ importers:
|
||||||
dependencies:
|
dependencies:
|
||||||
openclaw:
|
openclaw:
|
||||||
specifier: '>=2026.3.11'
|
specifier: '>=2026.3.11'
|
||||||
version: 2026.3.11(@discordjs/opus@0.10.0)(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3))
|
version: 2026.3.11(@discordjs/opus@0.10.0)(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3))
|
||||||
|
|
||||||
extensions/memory-lancedb:
|
extensions/memory-lancedb:
|
||||||
dependencies:
|
dependencies:
|
||||||
|
|
@ -1828,6 +1831,16 @@ packages:
|
||||||
'@mistralai/mistralai@1.14.1':
|
'@mistralai/mistralai@1.14.1':
|
||||||
resolution: {integrity: sha512-IiLmmZFCCTReQgPAT33r7KQ1nYo5JPdvGkrkZqA8qQ2qB1GHgs5LoP5K2ICyrjnpw2n8oSxMM/VP+liiKcGNlQ==}
|
resolution: {integrity: sha512-IiLmmZFCCTReQgPAT33r7KQ1nYo5JPdvGkrkZqA8qQ2qB1GHgs5LoP5K2ICyrjnpw2n8oSxMM/VP+liiKcGNlQ==}
|
||||||
|
|
||||||
|
'@modelcontextprotocol/sdk@1.27.1':
|
||||||
|
resolution: {integrity: sha512-sr6GbP+4edBwFndLbM60gf07z0FQ79gaExpnsjMGePXqFcSSb7t6iscpjk9DhFhwd+mTEQrzNafGP8/iGGFYaA==}
|
||||||
|
engines: {node: '>=18'}
|
||||||
|
peerDependencies:
|
||||||
|
'@cfworker/json-schema': ^4.1.1
|
||||||
|
zod: ^3.25 || ^4.0
|
||||||
|
peerDependenciesMeta:
|
||||||
|
'@cfworker/json-schema':
|
||||||
|
optional: true
|
||||||
|
|
||||||
'@mozilla/readability@0.6.0':
|
'@mozilla/readability@0.6.0':
|
||||||
resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==}
|
resolution: {integrity: sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==}
|
||||||
engines: {node: '>=14.0.0'}
|
engines: {node: '>=14.0.0'}
|
||||||
|
|
@ -4271,6 +4284,10 @@ packages:
|
||||||
core-util-is@1.0.3:
|
core-util-is@1.0.3:
|
||||||
resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
|
resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
|
||||||
|
|
||||||
|
cors@2.8.6:
|
||||||
|
resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==}
|
||||||
|
engines: {node: '>= 0.10'}
|
||||||
|
|
||||||
croner@10.0.1:
|
croner@10.0.1:
|
||||||
resolution: {integrity: sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==}
|
resolution: {integrity: sha512-ixNtAJndqh173VQ4KodSdJEI6nuioBWI0V1ITNKhZZsO0pEMoDxz539T4FTTbSZ/xIOSuDnzxLVRqBVSvPNE2g==}
|
||||||
engines: {node: '>=18.0'}
|
engines: {node: '>=18.0'}
|
||||||
|
|
@ -4550,6 +4567,14 @@ packages:
|
||||||
events-universal@1.0.1:
|
events-universal@1.0.1:
|
||||||
resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==}
|
resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==}
|
||||||
|
|
||||||
|
eventsource-parser@3.0.6:
|
||||||
|
resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==}
|
||||||
|
engines: {node: '>=18.0.0'}
|
||||||
|
|
||||||
|
eventsource@3.0.7:
|
||||||
|
resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==}
|
||||||
|
engines: {node: '>=18.0.0'}
|
||||||
|
|
||||||
execa@4.1.0:
|
execa@4.1.0:
|
||||||
resolution: {integrity: sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==}
|
resolution: {integrity: sha512-j5W0//W7f8UxAn8hXVnwG8tLwdiUy4FJLcSupCg6maBYZDpyBvTApK7KyuI4bKj8KOh1r2YH+6ucuYtJv1bTZA==}
|
||||||
engines: {node: '>=10'}
|
engines: {node: '>=10'}
|
||||||
|
|
@ -4561,6 +4586,12 @@ packages:
|
||||||
exponential-backoff@3.1.3:
|
exponential-backoff@3.1.3:
|
||||||
resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==}
|
resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==}
|
||||||
|
|
||||||
|
express-rate-limit@8.3.1:
|
||||||
|
resolution: {integrity: sha512-D1dKN+cmyPWuvB+G2SREQDzPY1agpBIcTa9sJxOPMCNeH3gwzhqJRDWCXW3gg0y//+LQ/8j52JbMROWyrKdMdw==}
|
||||||
|
engines: {node: '>= 16'}
|
||||||
|
peerDependencies:
|
||||||
|
express: '>= 4.11'
|
||||||
|
|
||||||
express@4.22.1:
|
express@4.22.1:
|
||||||
resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==}
|
resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==}
|
||||||
engines: {node: '>= 0.10.0'}
|
engines: {node: '>= 0.10.0'}
|
||||||
|
|
@ -5058,6 +5089,9 @@ packages:
|
||||||
jose@4.15.9:
|
jose@4.15.9:
|
||||||
resolution: {integrity: sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==}
|
resolution: {integrity: sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==}
|
||||||
|
|
||||||
|
jose@6.2.1:
|
||||||
|
resolution: {integrity: sha512-jUaKr1yrbfaImV7R2TN/b3IcZzsw38/chqMpo2XJ7i2F8AfM/lA4G1goC3JVEwg0H7UldTmSt3P68nt31W7/mw==}
|
||||||
|
|
||||||
js-stringify@1.0.2:
|
js-stringify@1.0.2:
|
||||||
resolution: {integrity: sha512-rtS5ATOo2Q5k1G+DADISilDA6lv79zIiwFd6CcjuIxGKLFm5C+RLImRscVap9k55i+MOZwgliw+NejvkLuGD5g==}
|
resolution: {integrity: sha512-rtS5ATOo2Q5k1G+DADISilDA6lv79zIiwFd6CcjuIxGKLFm5C+RLImRscVap9k55i+MOZwgliw+NejvkLuGD5g==}
|
||||||
|
|
||||||
|
|
@ -5102,6 +5136,9 @@ packages:
|
||||||
json-schema-traverse@1.0.0:
|
json-schema-traverse@1.0.0:
|
||||||
resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
|
resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
|
||||||
|
|
||||||
|
json-schema-typed@8.0.2:
|
||||||
|
resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==}
|
||||||
|
|
||||||
json-schema@0.4.0:
|
json-schema@0.4.0:
|
||||||
resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==}
|
resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==}
|
||||||
|
|
||||||
|
|
@ -5870,6 +5907,10 @@ packages:
|
||||||
resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==}
|
resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
pkce-challenge@5.0.1:
|
||||||
|
resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==}
|
||||||
|
engines: {node: '>=16.20.0'}
|
||||||
|
|
||||||
playwright-core@1.58.2:
|
playwright-core@1.58.2:
|
||||||
resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==}
|
resolution: {integrity: sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==}
|
||||||
engines: {node: '>=18'}
|
engines: {node: '>=18'}
|
||||||
|
|
@ -8645,12 +8686,14 @@ snapshots:
|
||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
'@noble/hashes': 2.0.1
|
'@noble/hashes': 2.0.1
|
||||||
|
|
||||||
'@google/genai@1.44.0':
|
'@google/genai@1.44.0(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))':
|
||||||
dependencies:
|
dependencies:
|
||||||
google-auth-library: 10.6.1
|
google-auth-library: 10.6.1
|
||||||
p-retry: 4.6.2
|
p-retry: 4.6.2
|
||||||
protobufjs: 7.5.4
|
protobufjs: 7.5.4
|
||||||
ws: 8.19.0
|
ws: 8.19.0
|
||||||
|
optionalDependencies:
|
||||||
|
'@modelcontextprotocol/sdk': 1.27.1(zod@4.3.6)
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- bufferutil
|
- bufferutil
|
||||||
- supports-color
|
- supports-color
|
||||||
|
|
@ -8698,7 +8741,6 @@ snapshots:
|
||||||
'@hono/node-server@1.19.10(hono@4.12.7)':
|
'@hono/node-server@1.19.10(hono@4.12.7)':
|
||||||
dependencies:
|
dependencies:
|
||||||
hono: 4.12.7
|
hono: 4.12.7
|
||||||
optional: true
|
|
||||||
|
|
||||||
'@huggingface/jinja@0.5.5': {}
|
'@huggingface/jinja@0.5.5': {}
|
||||||
|
|
||||||
|
|
@ -9025,9 +9067,9 @@ snapshots:
|
||||||
std-env: 3.10.0
|
std-env: 3.10.0
|
||||||
yoctocolors: 2.1.2
|
yoctocolors: 2.1.2
|
||||||
|
|
||||||
'@mariozechner/pi-agent-core@0.57.1(ws@8.19.0)(zod@4.3.6)':
|
'@mariozechner/pi-agent-core@0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@mariozechner/pi-ai': 0.57.1(ws@8.19.0)(zod@4.3.6)
|
'@mariozechner/pi-ai': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- '@modelcontextprotocol/sdk'
|
- '@modelcontextprotocol/sdk'
|
||||||
- aws-crt
|
- aws-crt
|
||||||
|
|
@ -9037,11 +9079,11 @@ snapshots:
|
||||||
- ws
|
- ws
|
||||||
- zod
|
- zod
|
||||||
|
|
||||||
'@mariozechner/pi-ai@0.57.1(ws@8.19.0)(zod@4.3.6)':
|
'@mariozechner/pi-ai@0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@anthropic-ai/sdk': 0.73.0(zod@4.3.6)
|
'@anthropic-ai/sdk': 0.73.0(zod@4.3.6)
|
||||||
'@aws-sdk/client-bedrock-runtime': 3.1004.0
|
'@aws-sdk/client-bedrock-runtime': 3.1004.0
|
||||||
'@google/genai': 1.44.0
|
'@google/genai': 1.44.0(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))
|
||||||
'@mistralai/mistralai': 1.14.1
|
'@mistralai/mistralai': 1.14.1
|
||||||
'@sinclair/typebox': 0.34.48
|
'@sinclair/typebox': 0.34.48
|
||||||
ajv: 8.18.0
|
ajv: 8.18.0
|
||||||
|
|
@ -9061,11 +9103,11 @@ snapshots:
|
||||||
- ws
|
- ws
|
||||||
- zod
|
- zod
|
||||||
|
|
||||||
'@mariozechner/pi-coding-agent@0.57.1(ws@8.19.0)(zod@4.3.6)':
|
'@mariozechner/pi-coding-agent@0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@mariozechner/jiti': 2.6.5
|
'@mariozechner/jiti': 2.6.5
|
||||||
'@mariozechner/pi-agent-core': 0.57.1(ws@8.19.0)(zod@4.3.6)
|
'@mariozechner/pi-agent-core': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@mariozechner/pi-ai': 0.57.1(ws@8.19.0)(zod@4.3.6)
|
'@mariozechner/pi-ai': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@mariozechner/pi-tui': 0.57.1
|
'@mariozechner/pi-tui': 0.57.1
|
||||||
'@silvia-odwyer/photon-node': 0.3.4
|
'@silvia-odwyer/photon-node': 0.3.4
|
||||||
chalk: 5.6.2
|
chalk: 5.6.2
|
||||||
|
|
@ -9141,6 +9183,28 @@ snapshots:
|
||||||
- bufferutil
|
- bufferutil
|
||||||
- utf-8-validate
|
- utf-8-validate
|
||||||
|
|
||||||
|
'@modelcontextprotocol/sdk@1.27.1(zod@4.3.6)':
|
||||||
|
dependencies:
|
||||||
|
'@hono/node-server': 1.19.10(hono@4.12.7)
|
||||||
|
ajv: 8.18.0
|
||||||
|
ajv-formats: 3.0.1(ajv@8.18.0)
|
||||||
|
content-type: 1.0.5
|
||||||
|
cors: 2.8.6
|
||||||
|
cross-spawn: 7.0.6
|
||||||
|
eventsource: 3.0.7
|
||||||
|
eventsource-parser: 3.0.6
|
||||||
|
express: 5.2.1
|
||||||
|
express-rate-limit: 8.3.1(express@5.2.1)
|
||||||
|
hono: 4.12.7
|
||||||
|
jose: 6.2.1
|
||||||
|
json-schema-typed: 8.0.2
|
||||||
|
pkce-challenge: 5.0.1
|
||||||
|
raw-body: 3.0.2
|
||||||
|
zod: 4.3.6
|
||||||
|
zod-to-json-schema: 3.25.1(zod@4.3.6)
|
||||||
|
transitivePeerDependencies:
|
||||||
|
- supports-color
|
||||||
|
|
||||||
'@mozilla/readability@0.6.0': {}
|
'@mozilla/readability@0.6.0': {}
|
||||||
|
|
||||||
'@napi-rs/canvas-android-arm64@0.1.95':
|
'@napi-rs/canvas-android-arm64@0.1.95':
|
||||||
|
|
@ -11916,6 +11980,11 @@ snapshots:
|
||||||
|
|
||||||
core-util-is@1.0.3: {}
|
core-util-is@1.0.3: {}
|
||||||
|
|
||||||
|
cors@2.8.6:
|
||||||
|
dependencies:
|
||||||
|
object-assign: 4.1.1
|
||||||
|
vary: 1.1.2
|
||||||
|
|
||||||
croner@10.0.1: {}
|
croner@10.0.1: {}
|
||||||
|
|
||||||
cross-spawn@7.0.6:
|
cross-spawn@7.0.6:
|
||||||
|
|
@ -12167,6 +12236,12 @@ snapshots:
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- bare-abort-controller
|
- bare-abort-controller
|
||||||
|
|
||||||
|
eventsource-parser@3.0.6: {}
|
||||||
|
|
||||||
|
eventsource@3.0.7:
|
||||||
|
dependencies:
|
||||||
|
eventsource-parser: 3.0.6
|
||||||
|
|
||||||
execa@4.1.0:
|
execa@4.1.0:
|
||||||
dependencies:
|
dependencies:
|
||||||
cross-spawn: 7.0.6
|
cross-spawn: 7.0.6
|
||||||
|
|
@ -12183,6 +12258,11 @@ snapshots:
|
||||||
|
|
||||||
exponential-backoff@3.1.3: {}
|
exponential-backoff@3.1.3: {}
|
||||||
|
|
||||||
|
express-rate-limit@8.3.1(express@5.2.1):
|
||||||
|
dependencies:
|
||||||
|
express: 5.2.1
|
||||||
|
ip-address: 10.1.0
|
||||||
|
|
||||||
express@4.22.1:
|
express@4.22.1:
|
||||||
dependencies:
|
dependencies:
|
||||||
accepts: 1.3.8
|
accepts: 1.3.8
|
||||||
|
|
@ -12826,6 +12906,8 @@ snapshots:
|
||||||
|
|
||||||
jose@4.15.9: {}
|
jose@4.15.9: {}
|
||||||
|
|
||||||
|
jose@6.2.1: {}
|
||||||
|
|
||||||
js-stringify@1.0.2: {}
|
js-stringify@1.0.2: {}
|
||||||
|
|
||||||
js-tokens@10.0.0: {}
|
js-tokens@10.0.0: {}
|
||||||
|
|
@ -12893,6 +12975,8 @@ snapshots:
|
||||||
|
|
||||||
json-schema-traverse@1.0.0: {}
|
json-schema-traverse@1.0.0: {}
|
||||||
|
|
||||||
|
json-schema-typed@8.0.2: {}
|
||||||
|
|
||||||
json-schema@0.4.0: {}
|
json-schema@0.4.0: {}
|
||||||
|
|
||||||
json-stringify-safe@5.0.1: {}
|
json-stringify-safe@5.0.1: {}
|
||||||
|
|
@ -13497,7 +13581,7 @@ snapshots:
|
||||||
ws: 8.19.0
|
ws: 8.19.0
|
||||||
zod: 4.3.6
|
zod: 4.3.6
|
||||||
|
|
||||||
openclaw@2026.3.11(@discordjs/opus@0.10.0)(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3)):
|
openclaw@2026.3.11(@discordjs/opus@0.10.0)(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(@napi-rs/canvas@0.1.95)(@types/express@5.0.6)(audio-decode@2.2.3)(node-llama-cpp@3.16.2(typescript@5.9.3)):
|
||||||
dependencies:
|
dependencies:
|
||||||
'@agentclientprotocol/sdk': 0.16.1(zod@4.3.6)
|
'@agentclientprotocol/sdk': 0.16.1(zod@4.3.6)
|
||||||
'@aws-sdk/client-bedrock': 3.1007.0
|
'@aws-sdk/client-bedrock': 3.1007.0
|
||||||
|
|
@ -13510,9 +13594,9 @@ snapshots:
|
||||||
'@larksuiteoapi/node-sdk': 1.59.0
|
'@larksuiteoapi/node-sdk': 1.59.0
|
||||||
'@line/bot-sdk': 10.6.0
|
'@line/bot-sdk': 10.6.0
|
||||||
'@lydell/node-pty': 1.2.0-beta.3
|
'@lydell/node-pty': 1.2.0-beta.3
|
||||||
'@mariozechner/pi-agent-core': 0.57.1(ws@8.19.0)(zod@4.3.6)
|
'@mariozechner/pi-agent-core': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@mariozechner/pi-ai': 0.57.1(ws@8.19.0)(zod@4.3.6)
|
'@mariozechner/pi-ai': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@mariozechner/pi-coding-agent': 0.57.1(ws@8.19.0)(zod@4.3.6)
|
'@mariozechner/pi-coding-agent': 0.57.1(@modelcontextprotocol/sdk@1.27.1(zod@4.3.6))(ws@8.19.0)(zod@4.3.6)
|
||||||
'@mariozechner/pi-tui': 0.57.1
|
'@mariozechner/pi-tui': 0.57.1
|
||||||
'@mozilla/readability': 0.6.0
|
'@mozilla/readability': 0.6.0
|
||||||
'@napi-rs/canvas': 0.1.95
|
'@napi-rs/canvas': 0.1.95
|
||||||
|
|
@ -13784,6 +13868,8 @@ snapshots:
|
||||||
sonic-boom: 4.2.1
|
sonic-boom: 4.2.1
|
||||||
thread-stream: 3.1.0
|
thread-stream: 3.1.0
|
||||||
|
|
||||||
|
pkce-challenge@5.0.1: {}
|
||||||
|
|
||||||
playwright-core@1.58.2: {}
|
playwright-core@1.58.2: {}
|
||||||
|
|
||||||
playwright@1.58.2:
|
playwright@1.58.2:
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
buildAiSnapshotFromChromeMcpSnapshot,
|
||||||
|
flattenChromeMcpSnapshotToAriaNodes,
|
||||||
|
} from "./chrome-mcp.snapshot.js";
|
||||||
|
|
||||||
|
const snapshot = {
|
||||||
|
id: "root",
|
||||||
|
role: "document",
|
||||||
|
name: "Example",
|
||||||
|
children: [
|
||||||
|
{
|
||||||
|
id: "btn-1",
|
||||||
|
role: "button",
|
||||||
|
name: "Continue",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "txt-1",
|
||||||
|
role: "textbox",
|
||||||
|
name: "Email",
|
||||||
|
value: "peter@example.com",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
describe("chrome MCP snapshot conversion", () => {
|
||||||
|
it("flattens structured snapshots into aria-style nodes", () => {
|
||||||
|
const nodes = flattenChromeMcpSnapshotToAriaNodes(snapshot, 10);
|
||||||
|
expect(nodes).toEqual([
|
||||||
|
{
|
||||||
|
ref: "root",
|
||||||
|
role: "document",
|
||||||
|
name: "Example",
|
||||||
|
value: undefined,
|
||||||
|
description: undefined,
|
||||||
|
depth: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ref: "btn-1",
|
||||||
|
role: "button",
|
||||||
|
name: "Continue",
|
||||||
|
value: undefined,
|
||||||
|
description: undefined,
|
||||||
|
depth: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ref: "txt-1",
|
||||||
|
role: "textbox",
|
||||||
|
name: "Email",
|
||||||
|
value: "peter@example.com",
|
||||||
|
description: undefined,
|
||||||
|
depth: 1,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("builds AI snapshots that preserve Chrome MCP uids as refs", () => {
|
||||||
|
const result = buildAiSnapshotFromChromeMcpSnapshot({ root: snapshot });
|
||||||
|
|
||||||
|
expect(result.snapshot).toContain('- button "Continue" [ref=btn-1]');
|
||||||
|
expect(result.snapshot).toContain('- textbox "Email" [ref=txt-1] value="peter@example.com"');
|
||||||
|
expect(result.refs).toEqual({
|
||||||
|
"btn-1": { role: "button", name: "Continue" },
|
||||||
|
"txt-1": { role: "textbox", name: "Email" },
|
||||||
|
});
|
||||||
|
expect(result.stats.refs).toBe(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,246 @@
|
||||||
|
import type { SnapshotAriaNode } from "./client.js";
|
||||||
|
import {
|
||||||
|
getRoleSnapshotStats,
|
||||||
|
type RoleRefMap,
|
||||||
|
type RoleSnapshotOptions,
|
||||||
|
} from "./pw-role-snapshot.js";
|
||||||
|
|
||||||
|
export type ChromeMcpSnapshotNode = {
|
||||||
|
id?: string;
|
||||||
|
role?: string;
|
||||||
|
name?: string;
|
||||||
|
value?: string | number | boolean;
|
||||||
|
description?: string;
|
||||||
|
children?: ChromeMcpSnapshotNode[];
|
||||||
|
};
|
||||||
|
|
||||||
|
const INTERACTIVE_ROLES = new Set([
|
||||||
|
"button",
|
||||||
|
"checkbox",
|
||||||
|
"combobox",
|
||||||
|
"link",
|
||||||
|
"listbox",
|
||||||
|
"menuitem",
|
||||||
|
"menuitemcheckbox",
|
||||||
|
"menuitemradio",
|
||||||
|
"option",
|
||||||
|
"radio",
|
||||||
|
"searchbox",
|
||||||
|
"slider",
|
||||||
|
"spinbutton",
|
||||||
|
"switch",
|
||||||
|
"tab",
|
||||||
|
"textbox",
|
||||||
|
"treeitem",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const CONTENT_ROLES = new Set([
|
||||||
|
"article",
|
||||||
|
"cell",
|
||||||
|
"columnheader",
|
||||||
|
"gridcell",
|
||||||
|
"heading",
|
||||||
|
"listitem",
|
||||||
|
"main",
|
||||||
|
"navigation",
|
||||||
|
"region",
|
||||||
|
"rowheader",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const STRUCTURAL_ROLES = new Set([
|
||||||
|
"application",
|
||||||
|
"directory",
|
||||||
|
"document",
|
||||||
|
"generic",
|
||||||
|
"group",
|
||||||
|
"ignored",
|
||||||
|
"list",
|
||||||
|
"menu",
|
||||||
|
"menubar",
|
||||||
|
"none",
|
||||||
|
"presentation",
|
||||||
|
"row",
|
||||||
|
"rowgroup",
|
||||||
|
"tablist",
|
||||||
|
"table",
|
||||||
|
"toolbar",
|
||||||
|
"tree",
|
||||||
|
"treegrid",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function normalizeRole(node: ChromeMcpSnapshotNode): string {
|
||||||
|
const role = typeof node.role === "string" ? node.role.trim().toLowerCase() : "";
|
||||||
|
return role || "generic";
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeString(value: unknown): string | undefined {
|
||||||
|
if (typeof value === "string") {
|
||||||
|
const trimmed = value.trim();
|
||||||
|
return trimmed || undefined;
|
||||||
|
}
|
||||||
|
if (typeof value === "number" || typeof value === "boolean") {
|
||||||
|
return String(value);
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function escapeQuoted(value: string): string {
|
||||||
|
return value.replaceAll("\\", "\\\\").replaceAll('"', '\\"');
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldIncludeNode(params: {
|
||||||
|
role: string;
|
||||||
|
name?: string;
|
||||||
|
options?: RoleSnapshotOptions;
|
||||||
|
}): boolean {
|
||||||
|
if (params.options?.interactive && !INTERACTIVE_ROLES.has(params.role)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (params.options?.compact && STRUCTURAL_ROLES.has(params.role) && !params.name) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldCreateRef(role: string, name?: string): boolean {
|
||||||
|
return INTERACTIVE_ROLES.has(role) || (CONTENT_ROLES.has(role) && Boolean(name));
|
||||||
|
}
|
||||||
|
|
||||||
|
type DuplicateTracker = {
|
||||||
|
counts: Map<string, number>;
|
||||||
|
keysByRef: Map<string, string>;
|
||||||
|
duplicates: Set<string>;
|
||||||
|
};
|
||||||
|
|
||||||
|
function createDuplicateTracker(): DuplicateTracker {
|
||||||
|
return {
|
||||||
|
counts: new Map(),
|
||||||
|
keysByRef: new Map(),
|
||||||
|
duplicates: new Set(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function registerRef(
|
||||||
|
tracker: DuplicateTracker,
|
||||||
|
ref: string,
|
||||||
|
role: string,
|
||||||
|
name?: string,
|
||||||
|
): number | undefined {
|
||||||
|
const key = `${role}:${name ?? ""}`;
|
||||||
|
const count = tracker.counts.get(key) ?? 0;
|
||||||
|
tracker.counts.set(key, count + 1);
|
||||||
|
tracker.keysByRef.set(ref, key);
|
||||||
|
if (count > 0) {
|
||||||
|
tracker.duplicates.add(key);
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function flattenChromeMcpSnapshotToAriaNodes(
|
||||||
|
root: ChromeMcpSnapshotNode,
|
||||||
|
limit = 500,
|
||||||
|
): SnapshotAriaNode[] {
|
||||||
|
const boundedLimit = Math.max(1, Math.min(2000, Math.floor(limit)));
|
||||||
|
const out: SnapshotAriaNode[] = [];
|
||||||
|
|
||||||
|
const visit = (node: ChromeMcpSnapshotNode, depth: number) => {
|
||||||
|
if (out.length >= boundedLimit) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const ref = normalizeString(node.id);
|
||||||
|
if (ref) {
|
||||||
|
out.push({
|
||||||
|
ref,
|
||||||
|
role: normalizeRole(node),
|
||||||
|
name: normalizeString(node.name) ?? "",
|
||||||
|
value: normalizeString(node.value),
|
||||||
|
description: normalizeString(node.description),
|
||||||
|
depth,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for (const child of node.children ?? []) {
|
||||||
|
visit(child, depth + 1);
|
||||||
|
if (out.length >= boundedLimit) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
visit(root, 0);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildAiSnapshotFromChromeMcpSnapshot(params: {
|
||||||
|
root: ChromeMcpSnapshotNode;
|
||||||
|
options?: RoleSnapshotOptions;
|
||||||
|
maxChars?: number;
|
||||||
|
}): {
|
||||||
|
snapshot: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
refs: RoleRefMap;
|
||||||
|
stats: { lines: number; chars: number; refs: number; interactive: number };
|
||||||
|
} {
|
||||||
|
const refs: RoleRefMap = {};
|
||||||
|
const tracker = createDuplicateTracker();
|
||||||
|
const lines: string[] = [];
|
||||||
|
|
||||||
|
const visit = (node: ChromeMcpSnapshotNode, depth: number) => {
|
||||||
|
const role = normalizeRole(node);
|
||||||
|
const name = normalizeString(node.name);
|
||||||
|
const value = normalizeString(node.value);
|
||||||
|
const description = normalizeString(node.description);
|
||||||
|
const maxDepth = params.options?.maxDepth;
|
||||||
|
if (maxDepth !== undefined && depth > maxDepth) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const includeNode = shouldIncludeNode({ role, name, options: params.options });
|
||||||
|
if (includeNode) {
|
||||||
|
let line = `${" ".repeat(depth)}- ${role}`;
|
||||||
|
if (name) {
|
||||||
|
line += ` "${escapeQuoted(name)}"`;
|
||||||
|
}
|
||||||
|
const ref = normalizeString(node.id);
|
||||||
|
if (ref && shouldCreateRef(role, name)) {
|
||||||
|
const nth = registerRef(tracker, ref, role, name);
|
||||||
|
refs[ref] = nth === undefined ? { role, name } : { role, name, nth };
|
||||||
|
line += ` [ref=${ref}]`;
|
||||||
|
}
|
||||||
|
if (value) {
|
||||||
|
line += ` value="${escapeQuoted(value)}"`;
|
||||||
|
}
|
||||||
|
if (description) {
|
||||||
|
line += ` description="${escapeQuoted(description)}"`;
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const child of node.children ?? []) {
|
||||||
|
visit(child, depth + 1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
visit(params.root, 0);
|
||||||
|
|
||||||
|
for (const [ref, data] of Object.entries(refs)) {
|
||||||
|
const key = tracker.keysByRef.get(ref);
|
||||||
|
if (key && !tracker.duplicates.has(key)) {
|
||||||
|
delete data.nth;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let snapshot = lines.join("\n");
|
||||||
|
let truncated = false;
|
||||||
|
const maxChars =
|
||||||
|
typeof params.maxChars === "number" && Number.isFinite(params.maxChars) && params.maxChars > 0
|
||||||
|
? Math.floor(params.maxChars)
|
||||||
|
: undefined;
|
||||||
|
if (maxChars && snapshot.length > maxChars) {
|
||||||
|
snapshot = `${snapshot.slice(0, maxChars)}\n\n[...TRUNCATED - page too large]`;
|
||||||
|
truncated = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const stats = getRoleSnapshotStats(snapshot, refs);
|
||||||
|
return truncated ? { snapshot, truncated, refs, stats } : { snapshot, refs, stats };
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import {
|
||||||
|
listChromeMcpTabs,
|
||||||
|
openChromeMcpTab,
|
||||||
|
resetChromeMcpSessionsForTest,
|
||||||
|
setChromeMcpSessionFactoryForTest,
|
||||||
|
} from "./chrome-mcp.js";
|
||||||
|
|
||||||
|
type ToolCall = {
|
||||||
|
name: string;
|
||||||
|
arguments?: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
function createFakeSession() {
|
||||||
|
const callTool = vi.fn(async ({ name }: ToolCall) => {
|
||||||
|
if (name === "list_pages") {
|
||||||
|
return {
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: [
|
||||||
|
"## Pages",
|
||||||
|
"1: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session [selected]",
|
||||||
|
"2: https://github.com/openclaw/openclaw/pull/45318",
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (name === "new_page") {
|
||||||
|
return {
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text: [
|
||||||
|
"## Pages",
|
||||||
|
"1: https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session",
|
||||||
|
"2: https://github.com/openclaw/openclaw/pull/45318",
|
||||||
|
"3: https://example.com/ [selected]",
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected tool ${name}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
client: {
|
||||||
|
callTool,
|
||||||
|
listTools: vi.fn().mockResolvedValue({ tools: [{ name: "list_pages" }] }),
|
||||||
|
close: vi.fn().mockResolvedValue(undefined),
|
||||||
|
connect: vi.fn().mockResolvedValue(undefined),
|
||||||
|
},
|
||||||
|
transport: {
|
||||||
|
pid: 123,
|
||||||
|
},
|
||||||
|
ready: Promise.resolve(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("chrome MCP page parsing", () => {
|
||||||
|
beforeEach(async () => {
|
||||||
|
await resetChromeMcpSessionsForTest();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("parses list_pages text responses when structuredContent is missing", async () => {
|
||||||
|
setChromeMcpSessionFactoryForTest(async () => createFakeSession());
|
||||||
|
|
||||||
|
const tabs = await listChromeMcpTabs("chrome-live");
|
||||||
|
|
||||||
|
expect(tabs).toEqual([
|
||||||
|
{
|
||||||
|
targetId: "1",
|
||||||
|
title: "",
|
||||||
|
url: "https://developer.chrome.com/blog/chrome-devtools-mcp-debug-your-browser-session",
|
||||||
|
type: "page",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
targetId: "2",
|
||||||
|
title: "",
|
||||||
|
url: "https://github.com/openclaw/openclaw/pull/45318",
|
||||||
|
type: "page",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("parses new_page text responses and returns the created tab", async () => {
|
||||||
|
setChromeMcpSessionFactoryForTest(async () => createFakeSession());
|
||||||
|
|
||||||
|
const tab = await openChromeMcpTab("chrome-live", "https://example.com/");
|
||||||
|
|
||||||
|
expect(tab).toEqual({
|
||||||
|
targetId: "3",
|
||||||
|
title: "",
|
||||||
|
url: "https://example.com/",
|
||||||
|
type: "page",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -0,0 +1,488 @@
|
||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||||
|
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||||
|
import type { ChromeMcpSnapshotNode } from "./chrome-mcp.snapshot.js";
|
||||||
|
import type { BrowserTab } from "./client.js";
|
||||||
|
import { BrowserProfileUnavailableError, BrowserTabNotFoundError } from "./errors.js";
|
||||||
|
|
||||||
|
type ChromeMcpStructuredPage = {
|
||||||
|
id: number;
|
||||||
|
url?: string;
|
||||||
|
selected?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ChromeMcpToolResult = {
|
||||||
|
structuredContent?: Record<string, unknown>;
|
||||||
|
content?: Array<Record<string, unknown>>;
|
||||||
|
isError?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ChromeMcpSession = {
|
||||||
|
client: Client;
|
||||||
|
transport: StdioClientTransport;
|
||||||
|
ready: Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ChromeMcpSessionFactory = (profileName: string) => Promise<ChromeMcpSession>;
|
||||||
|
|
||||||
|
const DEFAULT_CHROME_MCP_COMMAND = "npx";
|
||||||
|
const DEFAULT_CHROME_MCP_ARGS = [
|
||||||
|
"-y",
|
||||||
|
"chrome-devtools-mcp@latest",
|
||||||
|
"--autoConnect",
|
||||||
|
"--experimental-page-id-routing",
|
||||||
|
];
|
||||||
|
|
||||||
|
const sessions = new Map<string, ChromeMcpSession>();
|
||||||
|
let sessionFactory: ChromeMcpSessionFactory | null = null;
|
||||||
|
|
||||||
|
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||||
|
return value && typeof value === "object" && !Array.isArray(value)
|
||||||
|
? (value as Record<string, unknown>)
|
||||||
|
: null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function asPages(value: unknown): ChromeMcpStructuredPage[] {
|
||||||
|
if (!Array.isArray(value)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
const out: ChromeMcpStructuredPage[] = [];
|
||||||
|
for (const entry of value) {
|
||||||
|
const record = asRecord(entry);
|
||||||
|
if (!record || typeof record.id !== "number") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
out.push({
|
||||||
|
id: record.id,
|
||||||
|
url: typeof record.url === "string" ? record.url : undefined,
|
||||||
|
selected: record.selected === true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parsePageId(targetId: string): number {
|
||||||
|
const parsed = Number.parseInt(targetId.trim(), 10);
|
||||||
|
if (!Number.isFinite(parsed)) {
|
||||||
|
throw new BrowserTabNotFoundError();
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toBrowserTabs(pages: ChromeMcpStructuredPage[]): BrowserTab[] {
|
||||||
|
return pages.map((page) => ({
|
||||||
|
targetId: String(page.id),
|
||||||
|
title: "",
|
||||||
|
url: page.url ?? "",
|
||||||
|
type: "page",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractStructuredContent(result: ChromeMcpToolResult): Record<string, unknown> {
|
||||||
|
return asRecord(result.structuredContent) ?? {};
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractTextContent(result: ChromeMcpToolResult): string[] {
|
||||||
|
const content = Array.isArray(result.content) ? result.content : [];
|
||||||
|
return content
|
||||||
|
.map((entry) => {
|
||||||
|
const record = asRecord(entry);
|
||||||
|
return record && typeof record.text === "string" ? record.text : "";
|
||||||
|
})
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractTextPages(result: ChromeMcpToolResult): ChromeMcpStructuredPage[] {
|
||||||
|
const pages: ChromeMcpStructuredPage[] = [];
|
||||||
|
for (const block of extractTextContent(result)) {
|
||||||
|
for (const line of block.split(/\r?\n/)) {
|
||||||
|
const match = line.match(/^\s*(\d+):\s+(.+?)(?:\s+\[(selected)\])?\s*$/i);
|
||||||
|
if (!match) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
pages.push({
|
||||||
|
id: Number.parseInt(match[1] ?? "", 10),
|
||||||
|
url: match[2]?.trim() || undefined,
|
||||||
|
selected: Boolean(match[3]),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return pages;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractStructuredPages(result: ChromeMcpToolResult): ChromeMcpStructuredPage[] {
|
||||||
|
const structured = asPages(extractStructuredContent(result).pages);
|
||||||
|
return structured.length > 0 ? structured : extractTextPages(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractSnapshot(result: ChromeMcpToolResult): ChromeMcpSnapshotNode {
|
||||||
|
const structured = extractStructuredContent(result);
|
||||||
|
const snapshot = asRecord(structured.snapshot);
|
||||||
|
if (!snapshot) {
|
||||||
|
throw new Error("Chrome MCP snapshot response was missing structured snapshot data.");
|
||||||
|
}
|
||||||
|
return snapshot as unknown as ChromeMcpSnapshotNode;
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractJsonBlock(text: string): unknown {
|
||||||
|
const match = text.match(/```json\s*([\s\S]*?)\s*```/i);
|
||||||
|
const raw = match?.[1]?.trim() || text.trim();
|
||||||
|
return raw ? JSON.parse(raw) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createRealSession(profileName: string): Promise<ChromeMcpSession> {
|
||||||
|
const transport = new StdioClientTransport({
|
||||||
|
command: DEFAULT_CHROME_MCP_COMMAND,
|
||||||
|
args: DEFAULT_CHROME_MCP_ARGS,
|
||||||
|
stderr: "pipe",
|
||||||
|
});
|
||||||
|
const client = new Client(
|
||||||
|
{
|
||||||
|
name: "openclaw-browser",
|
||||||
|
version: "0.0.0",
|
||||||
|
},
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
|
||||||
|
const ready = (async () => {
|
||||||
|
try {
|
||||||
|
await client.connect(transport);
|
||||||
|
const tools = await client.listTools();
|
||||||
|
if (!tools.tools.some((tool) => tool.name === "list_pages")) {
|
||||||
|
throw new Error("Chrome MCP server did not expose the expected navigation tools.");
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
await client.close().catch(() => {});
|
||||||
|
throw new BrowserProfileUnavailableError(
|
||||||
|
`Chrome MCP existing-session attach failed for profile "${profileName}". ` +
|
||||||
|
`Make sure Chrome is running, enable chrome://inspect/#remote-debugging, and approve the connection. ` +
|
||||||
|
`Details: ${String(err)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
return {
|
||||||
|
client,
|
||||||
|
transport,
|
||||||
|
ready,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getSession(profileName: string): Promise<ChromeMcpSession> {
|
||||||
|
let session = sessions.get(profileName);
|
||||||
|
if (session && session.transport.pid === null) {
|
||||||
|
sessions.delete(profileName);
|
||||||
|
session = undefined;
|
||||||
|
}
|
||||||
|
if (!session) {
|
||||||
|
session = await (sessionFactory ?? createRealSession)(profileName);
|
||||||
|
sessions.set(profileName, session);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await session.ready;
|
||||||
|
return session;
|
||||||
|
} catch (err) {
|
||||||
|
const current = sessions.get(profileName);
|
||||||
|
if (current?.transport === session.transport) {
|
||||||
|
sessions.delete(profileName);
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function callTool(
|
||||||
|
profileName: string,
|
||||||
|
name: string,
|
||||||
|
args: Record<string, unknown> = {},
|
||||||
|
): Promise<ChromeMcpToolResult> {
|
||||||
|
const session = await getSession(profileName);
|
||||||
|
try {
|
||||||
|
return (await session.client.callTool({
|
||||||
|
name,
|
||||||
|
arguments: args,
|
||||||
|
})) as ChromeMcpToolResult;
|
||||||
|
} catch (err) {
|
||||||
|
sessions.delete(profileName);
|
||||||
|
await session.client.close().catch(() => {});
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function withTempFile<T>(fn: (filePath: string) => Promise<T>): Promise<T> {
|
||||||
|
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-chrome-mcp-"));
|
||||||
|
const filePath = path.join(dir, randomUUID());
|
||||||
|
try {
|
||||||
|
return await fn(filePath);
|
||||||
|
} finally {
|
||||||
|
await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function findPageById(profileName: string, pageId: number): Promise<ChromeMcpStructuredPage> {
|
||||||
|
const pages = await listChromeMcpPages(profileName);
|
||||||
|
const page = pages.find((entry) => entry.id === pageId);
|
||||||
|
if (!page) {
|
||||||
|
throw new BrowserTabNotFoundError();
|
||||||
|
}
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function ensureChromeMcpAvailable(profileName: string): Promise<void> {
|
||||||
|
await getSession(profileName);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getChromeMcpPid(profileName: string): number | null {
|
||||||
|
return sessions.get(profileName)?.transport.pid ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function closeChromeMcpSession(profileName: string): Promise<boolean> {
|
||||||
|
const session = sessions.get(profileName);
|
||||||
|
if (!session) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
sessions.delete(profileName);
|
||||||
|
await session.client.close().catch(() => {});
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function stopAllChromeMcpSessions(): Promise<void> {
|
||||||
|
const names = [...sessions.keys()];
|
||||||
|
for (const name of names) {
|
||||||
|
await closeChromeMcpSession(name).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listChromeMcpPages(profileName: string): Promise<ChromeMcpStructuredPage[]> {
|
||||||
|
const result = await callTool(profileName, "list_pages");
|
||||||
|
return extractStructuredPages(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listChromeMcpTabs(profileName: string): Promise<BrowserTab[]> {
|
||||||
|
return toBrowserTabs(await listChromeMcpPages(profileName));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function openChromeMcpTab(profileName: string, url: string): Promise<BrowserTab> {
|
||||||
|
const result = await callTool(profileName, "new_page", { url });
|
||||||
|
const pages = extractStructuredPages(result);
|
||||||
|
const chosen = pages.find((page) => page.selected) ?? pages.at(-1);
|
||||||
|
if (!chosen) {
|
||||||
|
throw new Error("Chrome MCP did not return the created page.");
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
targetId: String(chosen.id),
|
||||||
|
title: "",
|
||||||
|
url: chosen.url ?? url,
|
||||||
|
type: "page",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function focusChromeMcpTab(profileName: string, targetId: string): Promise<void> {
|
||||||
|
await callTool(profileName, "select_page", {
|
||||||
|
pageId: parsePageId(targetId),
|
||||||
|
bringToFront: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function closeChromeMcpTab(profileName: string, targetId: string): Promise<void> {
|
||||||
|
await callTool(profileName, "close_page", { pageId: parsePageId(targetId) });
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function navigateChromeMcpPage(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
url: string;
|
||||||
|
timeoutMs?: number;
|
||||||
|
}): Promise<{ url: string }> {
|
||||||
|
await callTool(params.profileName, "navigate_page", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
type: "url",
|
||||||
|
url: params.url,
|
||||||
|
...(typeof params.timeoutMs === "number" ? { timeout: params.timeoutMs } : {}),
|
||||||
|
});
|
||||||
|
const page = await findPageById(params.profileName, parsePageId(params.targetId));
|
||||||
|
return { url: page.url ?? params.url };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function takeChromeMcpSnapshot(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
}): Promise<ChromeMcpSnapshotNode> {
|
||||||
|
const result = await callTool(params.profileName, "take_snapshot", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
});
|
||||||
|
return extractSnapshot(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function takeChromeMcpScreenshot(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
uid?: string;
|
||||||
|
fullPage?: boolean;
|
||||||
|
format?: "png" | "jpeg";
|
||||||
|
}): Promise<Buffer> {
|
||||||
|
return await withTempFile(async (filePath) => {
|
||||||
|
await callTool(params.profileName, "take_screenshot", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
filePath,
|
||||||
|
format: params.format ?? "png",
|
||||||
|
...(params.uid ? { uid: params.uid } : {}),
|
||||||
|
...(params.fullPage ? { fullPage: true } : {}),
|
||||||
|
});
|
||||||
|
return await fs.readFile(filePath);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function clickChromeMcpElement(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
uid: string;
|
||||||
|
doubleClick?: boolean;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "click", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
uid: params.uid,
|
||||||
|
...(params.doubleClick ? { dblClick: true } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fillChromeMcpElement(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
uid: string;
|
||||||
|
value: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "fill", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
uid: params.uid,
|
||||||
|
value: params.value,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fillChromeMcpForm(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
elements: Array<{ uid: string; value: string }>;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "fill_form", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
elements: params.elements,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function hoverChromeMcpElement(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
uid: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "hover", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
uid: params.uid,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function dragChromeMcpElement(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
fromUid: string;
|
||||||
|
toUid: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "drag", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
from_uid: params.fromUid,
|
||||||
|
to_uid: params.toUid,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function uploadChromeMcpFile(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
uid: string;
|
||||||
|
filePath: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "upload_file", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
uid: params.uid,
|
||||||
|
filePath: params.filePath,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function pressChromeMcpKey(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
key: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "press_key", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
key: params.key,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resizeChromeMcpPage(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "resize_page", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
width: params.width,
|
||||||
|
height: params.height,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function handleChromeMcpDialog(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
action: "accept" | "dismiss";
|
||||||
|
promptText?: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "handle_dialog", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
action: params.action,
|
||||||
|
...(params.promptText ? { promptText: params.promptText } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function evaluateChromeMcpScript(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
fn: string;
|
||||||
|
args?: string[];
|
||||||
|
}): Promise<unknown> {
|
||||||
|
const result = await callTool(params.profileName, "evaluate_script", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
function: params.fn,
|
||||||
|
...(params.args?.length ? { args: params.args } : {}),
|
||||||
|
});
|
||||||
|
const message = extractStructuredContent(result).message;
|
||||||
|
const text = typeof message === "string" ? message : "";
|
||||||
|
if (!text.trim()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return extractJsonBlock(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function waitForChromeMcpText(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
text: string[];
|
||||||
|
timeoutMs?: number;
|
||||||
|
}): Promise<void> {
|
||||||
|
await callTool(params.profileName, "wait_for", {
|
||||||
|
pageId: parsePageId(params.targetId),
|
||||||
|
text: params.text,
|
||||||
|
...(typeof params.timeoutMs === "number" ? { timeout: params.timeoutMs } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function setChromeMcpSessionFactoryForTest(factory: ChromeMcpSessionFactory | null): void {
|
||||||
|
sessionFactory = factory;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function resetChromeMcpSessionsForTest(): Promise<void> {
|
||||||
|
sessionFactory = null;
|
||||||
|
await stopAllChromeMcpSessions();
|
||||||
|
}
|
||||||
|
|
@ -3,6 +3,7 @@ import { fetchBrowserJson } from "./client-fetch.js";
|
||||||
export type BrowserStatus = {
|
export type BrowserStatus = {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
profile?: string;
|
profile?: string;
|
||||||
|
driver?: "openclaw" | "extension" | "existing-session";
|
||||||
running: boolean;
|
running: boolean;
|
||||||
cdpReady?: boolean;
|
cdpReady?: boolean;
|
||||||
cdpHttp?: boolean;
|
cdpHttp?: boolean;
|
||||||
|
|
@ -26,6 +27,7 @@ export type ProfileStatus = {
|
||||||
cdpPort: number;
|
cdpPort: number;
|
||||||
cdpUrl: string;
|
cdpUrl: string;
|
||||||
color: string;
|
color: string;
|
||||||
|
driver: "openclaw" | "extension" | "existing-session";
|
||||||
running: boolean;
|
running: boolean;
|
||||||
tabCount: number;
|
tabCount: number;
|
||||||
isDefault: boolean;
|
isDefault: boolean;
|
||||||
|
|
@ -165,7 +167,7 @@ export async function browserCreateProfile(
|
||||||
name: string;
|
name: string;
|
||||||
color?: string;
|
color?: string;
|
||||||
cdpUrl?: string;
|
cdpUrl?: string;
|
||||||
driver?: "openclaw" | "extension";
|
driver?: "openclaw" | "extension" | "existing-session";
|
||||||
},
|
},
|
||||||
): Promise<BrowserCreateProfileResult> {
|
): Promise<BrowserCreateProfileResult> {
|
||||||
return await fetchBrowserJson<BrowserCreateProfileResult>(
|
return await fetchBrowserJson<BrowserCreateProfileResult>(
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ export type ResolvedBrowserProfile = {
|
||||||
cdpHost: string;
|
cdpHost: string;
|
||||||
cdpIsLoopback: boolean;
|
cdpIsLoopback: boolean;
|
||||||
color: string;
|
color: string;
|
||||||
driver: "openclaw" | "extension";
|
driver: "openclaw" | "extension" | "existing-session";
|
||||||
attachOnly: boolean;
|
attachOnly: boolean;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -335,7 +335,12 @@ export function resolveProfile(
|
||||||
let cdpHost = resolved.cdpHost;
|
let cdpHost = resolved.cdpHost;
|
||||||
let cdpPort = profile.cdpPort ?? 0;
|
let cdpPort = profile.cdpPort ?? 0;
|
||||||
let cdpUrl = "";
|
let cdpUrl = "";
|
||||||
const driver = profile.driver === "extension" ? "extension" : "openclaw";
|
const driver =
|
||||||
|
profile.driver === "extension"
|
||||||
|
? "extension"
|
||||||
|
: profile.driver === "existing-session"
|
||||||
|
? "existing-session"
|
||||||
|
: "openclaw";
|
||||||
|
|
||||||
if (rawProfileUrl) {
|
if (rawProfileUrl) {
|
||||||
const parsed = parseHttpUrl(rawProfileUrl, `browser.profiles.${profileName}.cdpUrl`);
|
const parsed = parseHttpUrl(rawProfileUrl, `browser.profiles.${profileName}.cdpUrl`);
|
||||||
|
|
@ -356,7 +361,7 @@ export function resolveProfile(
|
||||||
cdpIsLoopback: isLoopbackHost(cdpHost),
|
cdpIsLoopback: isLoopbackHost(cdpHost),
|
||||||
color: profile.color,
|
color: profile.color,
|
||||||
driver,
|
driver,
|
||||||
attachOnly: profile.attachOnly ?? resolved.attachOnly,
|
attachOnly: driver === "existing-session" ? true : (profile.attachOnly ?? resolved.attachOnly),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,10 @@
|
||||||
import type { ResolvedBrowserProfile } from "./config.js";
|
import type { ResolvedBrowserProfile } from "./config.js";
|
||||||
|
|
||||||
export type BrowserProfileMode = "local-managed" | "local-extension-relay" | "remote-cdp";
|
export type BrowserProfileMode =
|
||||||
|
| "local-managed"
|
||||||
|
| "local-extension-relay"
|
||||||
|
| "local-existing-session"
|
||||||
|
| "remote-cdp";
|
||||||
|
|
||||||
export type BrowserProfileCapabilities = {
|
export type BrowserProfileCapabilities = {
|
||||||
mode: BrowserProfileMode;
|
mode: BrowserProfileMode;
|
||||||
|
|
@ -31,6 +35,20 @@ export function getBrowserProfileCapabilities(
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
return {
|
||||||
|
mode: "local-existing-session",
|
||||||
|
isRemote: false,
|
||||||
|
requiresRelay: false,
|
||||||
|
requiresAttachedTab: false,
|
||||||
|
usesPersistentPlaywright: false,
|
||||||
|
supportsPerTabWs: false,
|
||||||
|
supportsJsonTabEndpoints: false,
|
||||||
|
supportsReset: false,
|
||||||
|
supportsManagedTabLimit: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (!profile.cdpIsLoopback) {
|
if (!profile.cdpIsLoopback) {
|
||||||
return {
|
return {
|
||||||
mode: "remote-cdp",
|
mode: "remote-cdp",
|
||||||
|
|
@ -75,6 +93,9 @@ export function resolveDefaultSnapshotFormat(params: {
|
||||||
if (capabilities.mode === "local-extension-relay") {
|
if (capabilities.mode === "local-extension-relay") {
|
||||||
return "aria";
|
return "aria";
|
||||||
}
|
}
|
||||||
|
if (capabilities.mode === "local-existing-session") {
|
||||||
|
return "ai";
|
||||||
|
}
|
||||||
|
|
||||||
return params.hasPlaywright ? "ai" : "aria";
|
return params.hasPlaywright ? "ai" : "aria";
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import fs from "node:fs";
|
import fs from "node:fs";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { resolveBrowserConfig } from "./config.js";
|
import { resolveBrowserConfig } from "./config.js";
|
||||||
import { createBrowserProfilesService } from "./profiles-service.js";
|
import { createBrowserProfilesService } from "./profiles-service.js";
|
||||||
import type { BrowserRouteContext, BrowserServerState } from "./server-context.js";
|
import type { BrowserRouteContext, BrowserServerState } from "./server-context.js";
|
||||||
|
|
@ -57,6 +57,10 @@ async function createWorkProfileWithConfig(params: {
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("BrowserProfilesService", () => {
|
describe("BrowserProfilesService", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
it("allocates next local port for new profiles", async () => {
|
it("allocates next local port for new profiles", async () => {
|
||||||
const { result, state } = await createWorkProfileWithConfig({
|
const { result, state } = await createWorkProfileWithConfig({
|
||||||
resolved: resolveBrowserConfig({}),
|
resolved: resolveBrowserConfig({}),
|
||||||
|
|
@ -163,6 +167,56 @@ describe("BrowserProfilesService", () => {
|
||||||
).rejects.toThrow(/requires an explicit loopback cdpUrl/i);
|
).rejects.toThrow(/requires an explicit loopback cdpUrl/i);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("creates existing-session profiles as attach-only local entries", async () => {
|
||||||
|
const resolved = resolveBrowserConfig({});
|
||||||
|
const { ctx, state } = createCtx(resolved);
|
||||||
|
vi.mocked(loadConfig).mockReturnValue({ browser: { profiles: {} } });
|
||||||
|
|
||||||
|
const service = createBrowserProfilesService(ctx);
|
||||||
|
const result = await service.createProfile({
|
||||||
|
name: "chrome-live",
|
||||||
|
driver: "existing-session",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.cdpPort).toBe(18801);
|
||||||
|
expect(result.isRemote).toBe(false);
|
||||||
|
expect(state.resolved.profiles["chrome-live"]).toEqual({
|
||||||
|
cdpPort: 18801,
|
||||||
|
driver: "existing-session",
|
||||||
|
attachOnly: true,
|
||||||
|
color: expect.any(String),
|
||||||
|
});
|
||||||
|
expect(writeConfigFile).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
browser: expect.objectContaining({
|
||||||
|
profiles: expect.objectContaining({
|
||||||
|
"chrome-live": expect.objectContaining({
|
||||||
|
cdpPort: 18801,
|
||||||
|
driver: "existing-session",
|
||||||
|
attachOnly: true,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects driver=existing-session when cdpUrl is provided", async () => {
|
||||||
|
const resolved = resolveBrowserConfig({});
|
||||||
|
const { ctx } = createCtx(resolved);
|
||||||
|
vi.mocked(loadConfig).mockReturnValue({ browser: { profiles: {} } });
|
||||||
|
|
||||||
|
const service = createBrowserProfilesService(ctx);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.createProfile({
|
||||||
|
name: "chrome-live",
|
||||||
|
driver: "existing-session",
|
||||||
|
cdpUrl: "http://127.0.0.1:9222",
|
||||||
|
}),
|
||||||
|
).rejects.toThrow(/does not accept cdpUrl/i);
|
||||||
|
});
|
||||||
|
|
||||||
it("deletes remote profiles without stopping or removing local data", async () => {
|
it("deletes remote profiles without stopping or removing local data", async () => {
|
||||||
const resolved = resolveBrowserConfig({
|
const resolved = resolveBrowserConfig({
|
||||||
profiles: {
|
profiles: {
|
||||||
|
|
@ -218,4 +272,40 @@ describe("BrowserProfilesService", () => {
|
||||||
expect(result.deleted).toBe(true);
|
expect(result.deleted).toBe(true);
|
||||||
expect(movePathToTrash).toHaveBeenCalledWith(path.dirname(userDataDir));
|
expect(movePathToTrash).toHaveBeenCalledWith(path.dirname(userDataDir));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("deletes existing-session profiles without touching local browser data", async () => {
|
||||||
|
const resolved = resolveBrowserConfig({
|
||||||
|
profiles: {
|
||||||
|
"chrome-live": {
|
||||||
|
cdpPort: 18801,
|
||||||
|
color: "#0066CC",
|
||||||
|
driver: "existing-session",
|
||||||
|
attachOnly: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const { ctx } = createCtx(resolved);
|
||||||
|
|
||||||
|
vi.mocked(loadConfig).mockReturnValue({
|
||||||
|
browser: {
|
||||||
|
defaultProfile: "openclaw",
|
||||||
|
profiles: {
|
||||||
|
openclaw: { cdpPort: 18800, color: "#FF4500" },
|
||||||
|
"chrome-live": {
|
||||||
|
cdpPort: 18801,
|
||||||
|
color: "#0066CC",
|
||||||
|
driver: "existing-session",
|
||||||
|
attachOnly: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const service = createBrowserProfilesService(ctx);
|
||||||
|
const result = await service.deleteProfile("chrome-live");
|
||||||
|
|
||||||
|
expect(result.deleted).toBe(false);
|
||||||
|
expect(ctx.forProfile).not.toHaveBeenCalled();
|
||||||
|
expect(movePathToTrash).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ export type CreateProfileParams = {
|
||||||
name: string;
|
name: string;
|
||||||
color?: string;
|
color?: string;
|
||||||
cdpUrl?: string;
|
cdpUrl?: string;
|
||||||
driver?: "openclaw" | "extension";
|
driver?: "openclaw" | "extension" | "existing-session";
|
||||||
};
|
};
|
||||||
|
|
||||||
export type CreateProfileResult = {
|
export type CreateProfileResult = {
|
||||||
|
|
@ -79,7 +79,12 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) {
|
||||||
const createProfile = async (params: CreateProfileParams): Promise<CreateProfileResult> => {
|
const createProfile = async (params: CreateProfileParams): Promise<CreateProfileResult> => {
|
||||||
const name = params.name.trim();
|
const name = params.name.trim();
|
||||||
const rawCdpUrl = params.cdpUrl?.trim() || undefined;
|
const rawCdpUrl = params.cdpUrl?.trim() || undefined;
|
||||||
const driver = params.driver === "extension" ? "extension" : undefined;
|
const driver =
|
||||||
|
params.driver === "extension"
|
||||||
|
? "extension"
|
||||||
|
: params.driver === "existing-session"
|
||||||
|
? "existing-session"
|
||||||
|
: undefined;
|
||||||
|
|
||||||
if (!isValidProfileName(name)) {
|
if (!isValidProfileName(name)) {
|
||||||
throw new BrowserValidationError(
|
throw new BrowserValidationError(
|
||||||
|
|
@ -118,6 +123,11 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (driver === "existing-session") {
|
||||||
|
throw new BrowserValidationError(
|
||||||
|
"driver=existing-session does not accept cdpUrl; it attaches via the Chrome MCP auto-connect flow",
|
||||||
|
);
|
||||||
|
}
|
||||||
profileConfig = {
|
profileConfig = {
|
||||||
cdpUrl: parsed.normalized,
|
cdpUrl: parsed.normalized,
|
||||||
...(driver ? { driver } : {}),
|
...(driver ? { driver } : {}),
|
||||||
|
|
@ -136,6 +146,7 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) {
|
||||||
profileConfig = {
|
profileConfig = {
|
||||||
cdpPort,
|
cdpPort,
|
||||||
...(driver ? { driver } : {}),
|
...(driver ? { driver } : {}),
|
||||||
|
...(driver === "existing-session" ? { attachOnly: true } : {}),
|
||||||
color: profileColor,
|
color: profileColor,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
@ -195,7 +206,7 @@ export function createBrowserProfilesService(ctx: BrowserRouteContext) {
|
||||||
const state = ctx.state();
|
const state = ctx.state();
|
||||||
const resolved = resolveProfile(state.resolved, name);
|
const resolved = resolveProfile(state.resolved, name);
|
||||||
|
|
||||||
if (resolved?.cdpIsLoopback) {
|
if (resolved?.cdpIsLoopback && resolved.driver === "openclaw") {
|
||||||
try {
|
try {
|
||||||
await ctx.forProfile(name).stopRunningBrowser();
|
await ctx.forProfile(name).stopRunningBrowser();
|
||||||
} catch {
|
} catch {
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,10 @@
|
||||||
import type { BrowserRouteContext } from "../server-context.js";
|
import type { BrowserRouteContext } from "../server-context.js";
|
||||||
import { readBody, resolveTargetIdFromBody, withPlaywrightRouteContext } from "./agent.shared.js";
|
import {
|
||||||
|
readBody,
|
||||||
|
requirePwAi,
|
||||||
|
resolveTargetIdFromBody,
|
||||||
|
withRouteTabContext,
|
||||||
|
} from "./agent.shared.js";
|
||||||
import { ensureOutputRootDir, resolveWritableOutputPathOrRespond } from "./output-paths.js";
|
import { ensureOutputRootDir, resolveWritableOutputPathOrRespond } from "./output-paths.js";
|
||||||
import { DEFAULT_DOWNLOAD_DIR } from "./path-output.js";
|
import { DEFAULT_DOWNLOAD_DIR } from "./path-output.js";
|
||||||
import type { BrowserRouteRegistrar } from "./types.js";
|
import type { BrowserRouteRegistrar } from "./types.js";
|
||||||
|
|
@ -23,13 +28,23 @@ export function registerBrowserAgentActDownloadRoutes(
|
||||||
const out = toStringOrEmpty(body.path) || "";
|
const out = toStringOrEmpty(body.path) || "";
|
||||||
const timeoutMs = toNumber(body.timeoutMs);
|
const timeoutMs = toNumber(body.timeoutMs);
|
||||||
|
|
||||||
await withPlaywrightRouteContext({
|
await withRouteTabContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
feature: "wait for download",
|
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||||
run: async ({ cdpUrl, tab, pw }) => {
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"download waiting is not supported for existing-session profiles yet.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, "wait for download");
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await ensureOutputRootDir(DEFAULT_DOWNLOAD_DIR);
|
await ensureOutputRootDir(DEFAULT_DOWNLOAD_DIR);
|
||||||
let downloadPath: string | undefined;
|
let downloadPath: string | undefined;
|
||||||
if (out.trim()) {
|
if (out.trim()) {
|
||||||
|
|
@ -67,13 +82,23 @@ export function registerBrowserAgentActDownloadRoutes(
|
||||||
return jsonError(res, 400, "path is required");
|
return jsonError(res, 400, "path is required");
|
||||||
}
|
}
|
||||||
|
|
||||||
await withPlaywrightRouteContext({
|
await withRouteTabContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
feature: "download",
|
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||||
run: async ({ cdpUrl, tab, pw }) => {
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"downloads are not supported for existing-session profiles yet.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, "download");
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await ensureOutputRootDir(DEFAULT_DOWNLOAD_DIR);
|
await ensureOutputRootDir(DEFAULT_DOWNLOAD_DIR);
|
||||||
const downloadPath = await resolveWritableOutputPathOrRespond({
|
const downloadPath = await resolveWritableOutputPathOrRespond({
|
||||||
res,
|
res,
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,11 @@
|
||||||
|
import { evaluateChromeMcpScript, uploadChromeMcpFile } from "../chrome-mcp.js";
|
||||||
import type { BrowserRouteContext } from "../server-context.js";
|
import type { BrowserRouteContext } from "../server-context.js";
|
||||||
import { readBody, resolveTargetIdFromBody, withPlaywrightRouteContext } from "./agent.shared.js";
|
import {
|
||||||
|
readBody,
|
||||||
|
requirePwAi,
|
||||||
|
resolveTargetIdFromBody,
|
||||||
|
withRouteTabContext,
|
||||||
|
} from "./agent.shared.js";
|
||||||
import { DEFAULT_UPLOAD_DIR, resolveExistingPathsWithinRoot } from "./path-output.js";
|
import { DEFAULT_UPLOAD_DIR, resolveExistingPathsWithinRoot } from "./path-output.js";
|
||||||
import type { BrowserRouteRegistrar } from "./types.js";
|
import type { BrowserRouteRegistrar } from "./types.js";
|
||||||
import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js";
|
import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js";
|
||||||
|
|
@ -20,13 +26,12 @@ export function registerBrowserAgentActHookRoutes(
|
||||||
return jsonError(res, 400, "paths are required");
|
return jsonError(res, 400, "paths are required");
|
||||||
}
|
}
|
||||||
|
|
||||||
await withPlaywrightRouteContext({
|
await withRouteTabContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
feature: "file chooser hook",
|
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||||
run: async ({ cdpUrl, tab, pw }) => {
|
|
||||||
const uploadPathsResult = await resolveExistingPathsWithinRoot({
|
const uploadPathsResult = await resolveExistingPathsWithinRoot({
|
||||||
rootDir: DEFAULT_UPLOAD_DIR,
|
rootDir: DEFAULT_UPLOAD_DIR,
|
||||||
requestedPaths: paths,
|
requestedPaths: paths,
|
||||||
|
|
@ -38,6 +43,39 @@ export function registerBrowserAgentActHookRoutes(
|
||||||
}
|
}
|
||||||
const resolvedPaths = uploadPathsResult.paths;
|
const resolvedPaths = uploadPathsResult.paths;
|
||||||
|
|
||||||
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
if (element) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session file uploads do not support element selectors; use ref/inputRef.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (resolvedPaths.length !== 1) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session file uploads currently support one file at a time.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const uid = inputRef || ref;
|
||||||
|
if (!uid) {
|
||||||
|
return jsonError(res, 501, "existing-session file uploads require ref or inputRef.");
|
||||||
|
}
|
||||||
|
await uploadChromeMcpFile({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
uid,
|
||||||
|
filePath: resolvedPaths[0] ?? "",
|
||||||
|
});
|
||||||
|
return res.json({ ok: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
const pw = await requirePwAi(res, "file chooser hook");
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (inputRef || element) {
|
if (inputRef || element) {
|
||||||
if (ref) {
|
if (ref) {
|
||||||
return jsonError(res, 400, "ref cannot be combined with inputRef/element");
|
return jsonError(res, 400, "ref cannot be combined with inputRef/element");
|
||||||
|
|
@ -79,13 +117,69 @@ export function registerBrowserAgentActHookRoutes(
|
||||||
return jsonError(res, 400, "accept is required");
|
return jsonError(res, 400, "accept is required");
|
||||||
}
|
}
|
||||||
|
|
||||||
await withPlaywrightRouteContext({
|
await withRouteTabContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
feature: "dialog hook",
|
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||||
run: async ({ cdpUrl, tab, pw }) => {
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
if (timeoutMs) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session dialog handling does not support timeoutMs.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await evaluateChromeMcpScript({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
fn: `() => {
|
||||||
|
const state = (window.__openclawDialogHook ??= {});
|
||||||
|
if (!state.originals) {
|
||||||
|
state.originals = {
|
||||||
|
alert: window.alert.bind(window),
|
||||||
|
confirm: window.confirm.bind(window),
|
||||||
|
prompt: window.prompt.bind(window),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const originals = state.originals;
|
||||||
|
const restore = () => {
|
||||||
|
window.alert = originals.alert;
|
||||||
|
window.confirm = originals.confirm;
|
||||||
|
window.prompt = originals.prompt;
|
||||||
|
delete window.__openclawDialogHook;
|
||||||
|
};
|
||||||
|
window.alert = (...args) => {
|
||||||
|
try {
|
||||||
|
return undefined;
|
||||||
|
} finally {
|
||||||
|
restore();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
window.confirm = (...args) => {
|
||||||
|
try {
|
||||||
|
return ${accept ? "true" : "false"};
|
||||||
|
} finally {
|
||||||
|
restore();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
window.prompt = (...args) => {
|
||||||
|
try {
|
||||||
|
return ${accept ? JSON.stringify(promptText ?? "") : "null"};
|
||||||
|
} finally {
|
||||||
|
restore();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return true;
|
||||||
|
}`,
|
||||||
|
});
|
||||||
|
return res.json({ ok: true });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, "dialog hook");
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.armDialogViaPlaywright({
|
await pw.armDialogViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,14 @@
|
||||||
|
import {
|
||||||
|
clickChromeMcpElement,
|
||||||
|
closeChromeMcpTab,
|
||||||
|
dragChromeMcpElement,
|
||||||
|
evaluateChromeMcpScript,
|
||||||
|
fillChromeMcpElement,
|
||||||
|
fillChromeMcpForm,
|
||||||
|
hoverChromeMcpElement,
|
||||||
|
pressChromeMcpKey,
|
||||||
|
resizeChromeMcpPage,
|
||||||
|
} from "../chrome-mcp.js";
|
||||||
import type { BrowserFormField } from "../client-actions-core.js";
|
import type { BrowserFormField } from "../client-actions-core.js";
|
||||||
import { normalizeBrowserFormField } from "../form-fields.js";
|
import { normalizeBrowserFormField } from "../form-fields.js";
|
||||||
import type { BrowserRouteContext } from "../server-context.js";
|
import type { BrowserRouteContext } from "../server-context.js";
|
||||||
|
|
@ -11,13 +22,88 @@ import {
|
||||||
} from "./agent.act.shared.js";
|
} from "./agent.act.shared.js";
|
||||||
import {
|
import {
|
||||||
readBody,
|
readBody,
|
||||||
|
requirePwAi,
|
||||||
resolveTargetIdFromBody,
|
resolveTargetIdFromBody,
|
||||||
withPlaywrightRouteContext,
|
withRouteTabContext,
|
||||||
SELECTOR_UNSUPPORTED_MESSAGE,
|
SELECTOR_UNSUPPORTED_MESSAGE,
|
||||||
} from "./agent.shared.js";
|
} from "./agent.shared.js";
|
||||||
import type { BrowserRouteRegistrar } from "./types.js";
|
import type { BrowserRouteRegistrar } from "./types.js";
|
||||||
import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js";
|
import { jsonError, toBoolean, toNumber, toStringArray, toStringOrEmpty } from "./utils.js";
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildExistingSessionWaitPredicate(params: {
|
||||||
|
text?: string;
|
||||||
|
textGone?: string;
|
||||||
|
selector?: string;
|
||||||
|
url?: string;
|
||||||
|
loadState?: "load" | "domcontentloaded" | "networkidle";
|
||||||
|
fn?: string;
|
||||||
|
}): string | null {
|
||||||
|
const checks: string[] = [];
|
||||||
|
if (params.text) {
|
||||||
|
checks.push(`Boolean(document.body?.innerText?.includes(${JSON.stringify(params.text)}))`);
|
||||||
|
}
|
||||||
|
if (params.textGone) {
|
||||||
|
checks.push(`!document.body?.innerText?.includes(${JSON.stringify(params.textGone)})`);
|
||||||
|
}
|
||||||
|
if (params.selector) {
|
||||||
|
checks.push(`Boolean(document.querySelector(${JSON.stringify(params.selector)}))`);
|
||||||
|
}
|
||||||
|
if (params.url) {
|
||||||
|
checks.push(`window.location.href === ${JSON.stringify(params.url)}`);
|
||||||
|
}
|
||||||
|
if (params.loadState === "domcontentloaded") {
|
||||||
|
checks.push(`document.readyState === "interactive" || document.readyState === "complete"`);
|
||||||
|
} else if (params.loadState === "load" || params.loadState === "networkidle") {
|
||||||
|
checks.push(`document.readyState === "complete"`);
|
||||||
|
}
|
||||||
|
if (params.fn) {
|
||||||
|
checks.push(`Boolean(await (${params.fn})())`);
|
||||||
|
}
|
||||||
|
if (checks.length === 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return checks.length === 1 ? checks[0] : checks.map((check) => `(${check})`).join(" && ");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForExistingSessionCondition(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
timeMs?: number;
|
||||||
|
text?: string;
|
||||||
|
textGone?: string;
|
||||||
|
selector?: string;
|
||||||
|
url?: string;
|
||||||
|
loadState?: "load" | "domcontentloaded" | "networkidle";
|
||||||
|
fn?: string;
|
||||||
|
timeoutMs?: number;
|
||||||
|
}): Promise<void> {
|
||||||
|
if (params.timeMs && params.timeMs > 0) {
|
||||||
|
await sleep(params.timeMs);
|
||||||
|
}
|
||||||
|
const predicate = buildExistingSessionWaitPredicate(params);
|
||||||
|
if (!predicate) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const timeoutMs = Math.max(250, params.timeoutMs ?? 10_000);
|
||||||
|
const deadline = Date.now() + timeoutMs;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
const ready = await evaluateChromeMcpScript({
|
||||||
|
profileName: params.profileName,
|
||||||
|
targetId: params.targetId,
|
||||||
|
fn: `async () => ${predicate}`,
|
||||||
|
});
|
||||||
|
if (ready) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await sleep(250);
|
||||||
|
}
|
||||||
|
throw new Error("Timed out waiting for condition");
|
||||||
|
}
|
||||||
|
|
||||||
export function registerBrowserAgentActRoutes(
|
export function registerBrowserAgentActRoutes(
|
||||||
app: BrowserRouteRegistrar,
|
app: BrowserRouteRegistrar,
|
||||||
ctx: BrowserRouteContext,
|
ctx: BrowserRouteContext,
|
||||||
|
|
@ -34,14 +120,15 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, SELECTOR_UNSUPPORTED_MESSAGE);
|
return jsonError(res, 400, SELECTOR_UNSUPPORTED_MESSAGE);
|
||||||
}
|
}
|
||||||
|
|
||||||
await withPlaywrightRouteContext({
|
await withRouteTabContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
feature: `act:${kind}`,
|
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||||
run: async ({ cdpUrl, tab, pw }) => {
|
|
||||||
const evaluateEnabled = ctx.state().resolved.evaluateEnabled;
|
const evaluateEnabled = ctx.state().resolved.evaluateEnabled;
|
||||||
|
const isExistingSession = profileCtx.profile.driver === "existing-session";
|
||||||
|
const profileName = profileCtx.profile.name;
|
||||||
|
|
||||||
switch (kind) {
|
switch (kind) {
|
||||||
case "click": {
|
case "click": {
|
||||||
|
|
@ -63,6 +150,26 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, parsedModifiers.error);
|
return jsonError(res, 400, parsedModifiers.error);
|
||||||
}
|
}
|
||||||
const modifiers = parsedModifiers.modifiers;
|
const modifiers = parsedModifiers.modifiers;
|
||||||
|
if (isExistingSession) {
|
||||||
|
if ((button && button !== "left") || (modifiers && modifiers.length > 0)) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session click currently supports left-click only (no button overrides/modifiers).",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await clickChromeMcpElement({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
uid: ref,
|
||||||
|
doubleClick,
|
||||||
|
});
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const clickRequest: Parameters<typeof pw.clickViaPlaywright>[0] = {
|
const clickRequest: Parameters<typeof pw.clickViaPlaywright>[0] = {
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -93,6 +200,33 @@ export function registerBrowserAgentActRoutes(
|
||||||
const submit = toBoolean(body.submit) ?? false;
|
const submit = toBoolean(body.submit) ?? false;
|
||||||
const slowly = toBoolean(body.slowly) ?? false;
|
const slowly = toBoolean(body.slowly) ?? false;
|
||||||
const timeoutMs = toNumber(body.timeoutMs);
|
const timeoutMs = toNumber(body.timeoutMs);
|
||||||
|
if (isExistingSession) {
|
||||||
|
if (slowly) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session type does not support slowly=true; use fill/press instead.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await fillChromeMcpElement({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
uid: ref,
|
||||||
|
value: text,
|
||||||
|
});
|
||||||
|
if (submit) {
|
||||||
|
await pressChromeMcpKey({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
key: "Enter",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const typeRequest: Parameters<typeof pw.typeViaPlaywright>[0] = {
|
const typeRequest: Parameters<typeof pw.typeViaPlaywright>[0] = {
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -113,6 +247,17 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, "key is required");
|
return jsonError(res, 400, "key is required");
|
||||||
}
|
}
|
||||||
const delayMs = toNumber(body.delayMs);
|
const delayMs = toNumber(body.delayMs);
|
||||||
|
if (isExistingSession) {
|
||||||
|
if (delayMs) {
|
||||||
|
return jsonError(res, 501, "existing-session press does not support delayMs.");
|
||||||
|
}
|
||||||
|
await pressChromeMcpKey({ profileName, targetId: tab.targetId, key });
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.pressKeyViaPlaywright({
|
await pw.pressKeyViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -127,6 +272,21 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, "ref is required");
|
return jsonError(res, 400, "ref is required");
|
||||||
}
|
}
|
||||||
const timeoutMs = toNumber(body.timeoutMs);
|
const timeoutMs = toNumber(body.timeoutMs);
|
||||||
|
if (isExistingSession) {
|
||||||
|
if (timeoutMs) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session hover does not support timeoutMs overrides.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await hoverChromeMcpElement({ profileName, targetId: tab.targetId, uid: ref });
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.hoverViaPlaywright({
|
await pw.hoverViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -141,6 +301,26 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, "ref is required");
|
return jsonError(res, 400, "ref is required");
|
||||||
}
|
}
|
||||||
const timeoutMs = toNumber(body.timeoutMs);
|
const timeoutMs = toNumber(body.timeoutMs);
|
||||||
|
if (isExistingSession) {
|
||||||
|
if (timeoutMs) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session scrollIntoView does not support timeoutMs overrides.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await evaluateChromeMcpScript({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
fn: `(el) => { el.scrollIntoView({ block: "center", inline: "center" }); return true; }`,
|
||||||
|
args: [ref],
|
||||||
|
});
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const scrollRequest: Parameters<typeof pw.scrollIntoViewViaPlaywright>[0] = {
|
const scrollRequest: Parameters<typeof pw.scrollIntoViewViaPlaywright>[0] = {
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -159,6 +339,26 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, "startRef and endRef are required");
|
return jsonError(res, 400, "startRef and endRef are required");
|
||||||
}
|
}
|
||||||
const timeoutMs = toNumber(body.timeoutMs);
|
const timeoutMs = toNumber(body.timeoutMs);
|
||||||
|
if (isExistingSession) {
|
||||||
|
if (timeoutMs) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session drag does not support timeoutMs overrides.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await dragChromeMcpElement({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
fromUid: startRef,
|
||||||
|
toUid: endRef,
|
||||||
|
});
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.dragViaPlaywright({
|
await pw.dragViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -175,6 +375,33 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, "ref and values are required");
|
return jsonError(res, 400, "ref and values are required");
|
||||||
}
|
}
|
||||||
const timeoutMs = toNumber(body.timeoutMs);
|
const timeoutMs = toNumber(body.timeoutMs);
|
||||||
|
if (isExistingSession) {
|
||||||
|
if (values.length !== 1) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session select currently supports a single value only.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (timeoutMs) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session select does not support timeoutMs overrides.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await fillChromeMcpElement({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
uid: ref,
|
||||||
|
value: values[0] ?? "",
|
||||||
|
});
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.selectOptionViaPlaywright({
|
await pw.selectOptionViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -198,6 +425,28 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, "fields are required");
|
return jsonError(res, 400, "fields are required");
|
||||||
}
|
}
|
||||||
const timeoutMs = toNumber(body.timeoutMs);
|
const timeoutMs = toNumber(body.timeoutMs);
|
||||||
|
if (isExistingSession) {
|
||||||
|
if (timeoutMs) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session fill does not support timeoutMs overrides.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await fillChromeMcpForm({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
elements: fields.map((field) => ({
|
||||||
|
uid: field.ref,
|
||||||
|
value: String(field.value ?? ""),
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.fillFormViaPlaywright({
|
await pw.fillFormViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -212,6 +461,19 @@ export function registerBrowserAgentActRoutes(
|
||||||
if (!width || !height) {
|
if (!width || !height) {
|
||||||
return jsonError(res, 400, "width and height are required");
|
return jsonError(res, 400, "width and height are required");
|
||||||
}
|
}
|
||||||
|
if (isExistingSession) {
|
||||||
|
await resizeChromeMcpPage({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
});
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId, url: tab.url });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.resizeViewportViaPlaywright({
|
await pw.resizeViewportViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -260,6 +522,25 @@ export function registerBrowserAgentActRoutes(
|
||||||
"wait requires at least one of: timeMs, text, textGone, selector, url, loadState, fn",
|
"wait requires at least one of: timeMs, text, textGone, selector, url, loadState, fn",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (isExistingSession) {
|
||||||
|
await waitForExistingSessionCondition({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
timeMs,
|
||||||
|
text,
|
||||||
|
textGone,
|
||||||
|
selector,
|
||||||
|
url,
|
||||||
|
loadState,
|
||||||
|
fn,
|
||||||
|
timeoutMs,
|
||||||
|
});
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.waitForViaPlaywright({
|
await pw.waitForViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -291,6 +572,31 @@ export function registerBrowserAgentActRoutes(
|
||||||
}
|
}
|
||||||
const ref = toStringOrEmpty(body.ref) || undefined;
|
const ref = toStringOrEmpty(body.ref) || undefined;
|
||||||
const evalTimeoutMs = toNumber(body.timeoutMs);
|
const evalTimeoutMs = toNumber(body.timeoutMs);
|
||||||
|
if (isExistingSession) {
|
||||||
|
if (evalTimeoutMs !== undefined) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"existing-session evaluate does not support timeoutMs overrides.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const result = await evaluateChromeMcpScript({
|
||||||
|
profileName,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
fn,
|
||||||
|
args: ref ? [ref] : undefined,
|
||||||
|
});
|
||||||
|
return res.json({
|
||||||
|
ok: true,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
url: tab.url,
|
||||||
|
result,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const evalRequest: Parameters<typeof pw.evaluateViaPlaywright>[0] = {
|
const evalRequest: Parameters<typeof pw.evaluateViaPlaywright>[0] = {
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -310,6 +616,14 @@ export function registerBrowserAgentActRoutes(
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
case "close": {
|
case "close": {
|
||||||
|
if (isExistingSession) {
|
||||||
|
await closeChromeMcpTab(profileName, tab.targetId);
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, `act:${kind}`);
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.closePageViaPlaywright({ cdpUrl, targetId: tab.targetId });
|
await pw.closePageViaPlaywright({ cdpUrl, targetId: tab.targetId });
|
||||||
return res.json({ ok: true, targetId: tab.targetId });
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
}
|
}
|
||||||
|
|
@ -334,13 +648,23 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, "url is required");
|
return jsonError(res, 400, "url is required");
|
||||||
}
|
}
|
||||||
|
|
||||||
await withPlaywrightRouteContext({
|
await withRouteTabContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
feature: "response body",
|
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||||
run: async ({ cdpUrl, tab, pw }) => {
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"response body is not supported for existing-session profiles yet.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, "response body");
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const result = await pw.responseBodyViaPlaywright({
|
const result = await pw.responseBodyViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -361,13 +685,39 @@ export function registerBrowserAgentActRoutes(
|
||||||
return jsonError(res, 400, "ref is required");
|
return jsonError(res, 400, "ref is required");
|
||||||
}
|
}
|
||||||
|
|
||||||
await withPlaywrightRouteContext({
|
await withRouteTabContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
feature: "highlight",
|
run: async ({ profileCtx, cdpUrl, tab }) => {
|
||||||
run: async ({ cdpUrl, tab, pw }) => {
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
await evaluateChromeMcpScript({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
args: [ref],
|
||||||
|
fn: `(el) => {
|
||||||
|
if (!(el instanceof Element)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
el.scrollIntoView({ block: "center", inline: "center" });
|
||||||
|
const previousOutline = el.style.outline;
|
||||||
|
const previousOffset = el.style.outlineOffset;
|
||||||
|
el.style.outline = "3px solid #FF4500";
|
||||||
|
el.style.outlineOffset = "2px";
|
||||||
|
setTimeout(() => {
|
||||||
|
el.style.outline = previousOutline;
|
||||||
|
el.style.outlineOffset = previousOffset;
|
||||||
|
}, 2000);
|
||||||
|
return true;
|
||||||
|
}`,
|
||||||
|
});
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, "highlight");
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
await pw.highlightViaPlaywright({
|
await pw.highlightViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,20 @@
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
import { ensureMediaDir, saveMediaBuffer } from "../../media/store.js";
|
||||||
import { captureScreenshot, snapshotAria } from "../cdp.js";
|
import { captureScreenshot, snapshotAria } from "../cdp.js";
|
||||||
|
import {
|
||||||
|
evaluateChromeMcpScript,
|
||||||
|
navigateChromeMcpPage,
|
||||||
|
takeChromeMcpScreenshot,
|
||||||
|
takeChromeMcpSnapshot,
|
||||||
|
} from "../chrome-mcp.js";
|
||||||
|
import {
|
||||||
|
buildAiSnapshotFromChromeMcpSnapshot,
|
||||||
|
flattenChromeMcpSnapshotToAriaNodes,
|
||||||
|
} from "../chrome-mcp.snapshot.js";
|
||||||
|
import {
|
||||||
|
assertBrowserNavigationAllowed,
|
||||||
|
assertBrowserNavigationResultAllowed,
|
||||||
|
} from "../navigation-guard.js";
|
||||||
import { withBrowserNavigationPolicy } from "../navigation-guard.js";
|
import { withBrowserNavigationPolicy } from "../navigation-guard.js";
|
||||||
import {
|
import {
|
||||||
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||||
|
|
@ -25,6 +39,89 @@ import {
|
||||||
import type { BrowserResponse, BrowserRouteRegistrar } from "./types.js";
|
import type { BrowserResponse, BrowserRouteRegistrar } from "./types.js";
|
||||||
import { jsonError, toBoolean, toStringOrEmpty } from "./utils.js";
|
import { jsonError, toBoolean, toStringOrEmpty } from "./utils.js";
|
||||||
|
|
||||||
|
const CHROME_MCP_OVERLAY_ATTR = "data-openclaw-mcp-overlay";
|
||||||
|
|
||||||
|
async function clearChromeMcpOverlay(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
await evaluateChromeMcpScript({
|
||||||
|
profileName: params.profileName,
|
||||||
|
targetId: params.targetId,
|
||||||
|
fn: `() => {
|
||||||
|
document.querySelectorAll("[${CHROME_MCP_OVERLAY_ATTR}]").forEach((node) => node.remove());
|
||||||
|
return true;
|
||||||
|
}`,
|
||||||
|
}).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renderChromeMcpLabels(params: {
|
||||||
|
profileName: string;
|
||||||
|
targetId: string;
|
||||||
|
refs: string[];
|
||||||
|
}): Promise<{ labels: number; skipped: number }> {
|
||||||
|
const refList = JSON.stringify(params.refs);
|
||||||
|
const result = await evaluateChromeMcpScript({
|
||||||
|
profileName: params.profileName,
|
||||||
|
targetId: params.targetId,
|
||||||
|
args: params.refs,
|
||||||
|
fn: `(...elements) => {
|
||||||
|
const refs = ${refList};
|
||||||
|
document.querySelectorAll("[${CHROME_MCP_OVERLAY_ATTR}]").forEach((node) => node.remove());
|
||||||
|
const root = document.createElement("div");
|
||||||
|
root.setAttribute("${CHROME_MCP_OVERLAY_ATTR}", "labels");
|
||||||
|
root.style.position = "fixed";
|
||||||
|
root.style.inset = "0";
|
||||||
|
root.style.pointerEvents = "none";
|
||||||
|
root.style.zIndex = "2147483647";
|
||||||
|
let labels = 0;
|
||||||
|
let skipped = 0;
|
||||||
|
elements.forEach((el, index) => {
|
||||||
|
if (!(el instanceof Element)) {
|
||||||
|
skipped += 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
if (rect.width <= 0 && rect.height <= 0) {
|
||||||
|
skipped += 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
labels += 1;
|
||||||
|
const badge = document.createElement("div");
|
||||||
|
badge.setAttribute("${CHROME_MCP_OVERLAY_ATTR}", "label");
|
||||||
|
badge.textContent = refs[index] || String(labels);
|
||||||
|
badge.style.position = "fixed";
|
||||||
|
badge.style.left = \`\${Math.max(0, rect.left)}px\`;
|
||||||
|
badge.style.top = \`\${Math.max(0, rect.top)}px\`;
|
||||||
|
badge.style.transform = "translateY(-100%)";
|
||||||
|
badge.style.padding = "2px 6px";
|
||||||
|
badge.style.borderRadius = "999px";
|
||||||
|
badge.style.background = "#FF4500";
|
||||||
|
badge.style.color = "#fff";
|
||||||
|
badge.style.font = "600 12px ui-monospace, SFMono-Regular, Menlo, monospace";
|
||||||
|
badge.style.boxShadow = "0 2px 6px rgba(0,0,0,0.35)";
|
||||||
|
badge.style.whiteSpace = "nowrap";
|
||||||
|
root.appendChild(badge);
|
||||||
|
});
|
||||||
|
document.documentElement.appendChild(root);
|
||||||
|
return { labels, skipped };
|
||||||
|
}`,
|
||||||
|
});
|
||||||
|
const labels =
|
||||||
|
result &&
|
||||||
|
typeof result === "object" &&
|
||||||
|
typeof (result as { labels?: unknown }).labels === "number"
|
||||||
|
? (result as { labels: number }).labels
|
||||||
|
: 0;
|
||||||
|
const skipped =
|
||||||
|
result &&
|
||||||
|
typeof result === "object" &&
|
||||||
|
typeof (result as { skipped?: unknown }).skipped === "number"
|
||||||
|
? (result as { skipped: number }).skipped
|
||||||
|
: 0;
|
||||||
|
return { labels, skipped };
|
||||||
|
}
|
||||||
|
|
||||||
async function saveBrowserMediaResponse(params: {
|
async function saveBrowserMediaResponse(params: {
|
||||||
res: BrowserResponse;
|
res: BrowserResponse;
|
||||||
buffer: Buffer;
|
buffer: Buffer;
|
||||||
|
|
@ -96,13 +193,27 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||||
if (!url) {
|
if (!url) {
|
||||||
return jsonError(res, 400, "url is required");
|
return jsonError(res, 400, "url is required");
|
||||||
}
|
}
|
||||||
await withPlaywrightRouteContext({
|
await withRouteTabContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
feature: "navigate",
|
run: async ({ profileCtx, tab, cdpUrl }) => {
|
||||||
run: async ({ cdpUrl, tab, pw, profileCtx }) => {
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
const ssrfPolicyOpts = withBrowserNavigationPolicy(ctx.state().resolved.ssrfPolicy);
|
||||||
|
await assertBrowserNavigationAllowed({ url, ...ssrfPolicyOpts });
|
||||||
|
const result = await navigateChromeMcpPage({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
url,
|
||||||
|
});
|
||||||
|
await assertBrowserNavigationResultAllowed({ url: result.url, ...ssrfPolicyOpts });
|
||||||
|
return res.json({ ok: true, targetId: tab.targetId, ...result });
|
||||||
|
}
|
||||||
|
const pw = await requirePwAi(res, "navigate");
|
||||||
|
if (!pw) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const result = await pw.navigateViaPlaywright({
|
const result = await pw.navigateViaPlaywright({
|
||||||
cdpUrl,
|
cdpUrl,
|
||||||
targetId: tab.targetId,
|
targetId: tab.targetId,
|
||||||
|
|
@ -122,6 +233,17 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||||
app.post("/pdf", async (req, res) => {
|
app.post("/pdf", async (req, res) => {
|
||||||
const body = readBody(req);
|
const body = readBody(req);
|
||||||
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
||||||
|
const profileCtx = resolveProfileContext(req, res, ctx);
|
||||||
|
if (!profileCtx) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
501,
|
||||||
|
"pdf is not supported for existing-session profiles yet; use screenshot/snapshot instead.",
|
||||||
|
);
|
||||||
|
}
|
||||||
await withPlaywrightRouteContext({
|
await withPlaywrightRouteContext({
|
||||||
req,
|
req,
|
||||||
res,
|
res,
|
||||||
|
|
@ -163,6 +285,36 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||||
ctx,
|
ctx,
|
||||||
targetId,
|
targetId,
|
||||||
run: async ({ profileCtx, tab, cdpUrl }) => {
|
run: async ({ profileCtx, tab, cdpUrl }) => {
|
||||||
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
if (element) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
400,
|
||||||
|
"element screenshots are not supported for existing-session profiles; use ref from snapshot.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const buffer = await takeChromeMcpScreenshot({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
uid: ref,
|
||||||
|
fullPage,
|
||||||
|
format: type,
|
||||||
|
});
|
||||||
|
const normalized = await normalizeBrowserScreenshot(buffer, {
|
||||||
|
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||||
|
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||||
|
});
|
||||||
|
await saveBrowserMediaResponse({
|
||||||
|
res,
|
||||||
|
buffer: normalized.buffer,
|
||||||
|
contentType: normalized.contentType ?? `image/${type}`,
|
||||||
|
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
url: tab.url,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let buffer: Buffer;
|
let buffer: Buffer;
|
||||||
const shouldUsePlaywright = shouldUsePlaywrightForScreenshot({
|
const shouldUsePlaywright = shouldUsePlaywrightForScreenshot({
|
||||||
profile: profileCtx.profile,
|
profile: profileCtx.profile,
|
||||||
|
|
@ -227,6 +379,90 @@ export function registerBrowserAgentSnapshotRoutes(
|
||||||
if ((plan.labels || plan.mode === "efficient") && plan.format === "aria") {
|
if ((plan.labels || plan.mode === "efficient") && plan.format === "aria") {
|
||||||
return jsonError(res, 400, "labels/mode=efficient require format=ai");
|
return jsonError(res, 400, "labels/mode=efficient require format=ai");
|
||||||
}
|
}
|
||||||
|
if (profileCtx.profile.driver === "existing-session") {
|
||||||
|
if (plan.labels) {
|
||||||
|
return jsonError(res, 501, "labels are not supported for existing-session profiles yet.");
|
||||||
|
}
|
||||||
|
if (plan.selectorValue || plan.frameSelectorValue) {
|
||||||
|
return jsonError(
|
||||||
|
res,
|
||||||
|
400,
|
||||||
|
"selector/frame snapshots are not supported for existing-session profiles; snapshot the whole page and use refs.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const snapshot = await takeChromeMcpSnapshot({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
});
|
||||||
|
if (plan.format === "aria") {
|
||||||
|
return res.json({
|
||||||
|
ok: true,
|
||||||
|
format: "aria",
|
||||||
|
targetId: tab.targetId,
|
||||||
|
url: tab.url,
|
||||||
|
nodes: flattenChromeMcpSnapshotToAriaNodes(snapshot, plan.limit),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const built = buildAiSnapshotFromChromeMcpSnapshot({
|
||||||
|
root: snapshot,
|
||||||
|
options: {
|
||||||
|
interactive: plan.interactive ?? undefined,
|
||||||
|
compact: plan.compact ?? undefined,
|
||||||
|
maxDepth: plan.depth ?? undefined,
|
||||||
|
},
|
||||||
|
maxChars: plan.resolvedMaxChars,
|
||||||
|
});
|
||||||
|
if (plan.labels) {
|
||||||
|
const refs = Object.keys(built.refs);
|
||||||
|
const labelResult = await renderChromeMcpLabels({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
refs,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const labeled = await takeChromeMcpScreenshot({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
format: "png",
|
||||||
|
});
|
||||||
|
const normalized = await normalizeBrowserScreenshot(labeled, {
|
||||||
|
maxSide: DEFAULT_BROWSER_SCREENSHOT_MAX_SIDE,
|
||||||
|
maxBytes: DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||||
|
});
|
||||||
|
await ensureMediaDir();
|
||||||
|
const saved = await saveMediaBuffer(
|
||||||
|
normalized.buffer,
|
||||||
|
normalized.contentType ?? "image/png",
|
||||||
|
"browser",
|
||||||
|
DEFAULT_BROWSER_SCREENSHOT_MAX_BYTES,
|
||||||
|
);
|
||||||
|
return res.json({
|
||||||
|
ok: true,
|
||||||
|
format: "ai",
|
||||||
|
targetId: tab.targetId,
|
||||||
|
url: tab.url,
|
||||||
|
labels: true,
|
||||||
|
labelsCount: labelResult.labels,
|
||||||
|
labelsSkipped: labelResult.skipped,
|
||||||
|
imagePath: path.resolve(saved.path),
|
||||||
|
imageType: normalized.contentType?.includes("jpeg") ? "jpeg" : "png",
|
||||||
|
...built,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
await clearChromeMcpOverlay({
|
||||||
|
profileName: profileCtx.profile.name,
|
||||||
|
targetId: tab.targetId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res.json({
|
||||||
|
ok: true,
|
||||||
|
format: "ai",
|
||||||
|
targetId: tab.targetId,
|
||||||
|
url: tab.url,
|
||||||
|
...built,
|
||||||
|
});
|
||||||
|
}
|
||||||
if (plan.format === "ai") {
|
if (plan.format === "ai") {
|
||||||
const pw = await requirePwAi(res, "ai snapshot");
|
const pw = await requirePwAi(res, "ai snapshot");
|
||||||
if (!pw) {
|
if (!pw) {
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { getChromeMcpPid } from "../chrome-mcp.js";
|
||||||
import { resolveBrowserExecutableForPlatform } from "../chrome.executables.js";
|
import { resolveBrowserExecutableForPlatform } from "../chrome.executables.js";
|
||||||
import { toBrowserErrorResponse } from "../errors.js";
|
import { toBrowserErrorResponse } from "../errors.js";
|
||||||
import { createBrowserProfilesService } from "../profiles-service.js";
|
import { createBrowserProfilesService } from "../profiles-service.js";
|
||||||
|
|
@ -76,10 +77,14 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow
|
||||||
res.json({
|
res.json({
|
||||||
enabled: current.resolved.enabled,
|
enabled: current.resolved.enabled,
|
||||||
profile: profileCtx.profile.name,
|
profile: profileCtx.profile.name,
|
||||||
|
driver: profileCtx.profile.driver,
|
||||||
running: cdpReady,
|
running: cdpReady,
|
||||||
cdpReady,
|
cdpReady,
|
||||||
cdpHttp,
|
cdpHttp,
|
||||||
pid: profileState?.running?.pid ?? null,
|
pid:
|
||||||
|
profileCtx.profile.driver === "existing-session"
|
||||||
|
? getChromeMcpPid(profileCtx.profile.name)
|
||||||
|
: (profileState?.running?.pid ?? null),
|
||||||
cdpPort: profileCtx.profile.cdpPort,
|
cdpPort: profileCtx.profile.cdpPort,
|
||||||
cdpUrl: profileCtx.profile.cdpUrl,
|
cdpUrl: profileCtx.profile.cdpUrl,
|
||||||
chosenBrowser: profileState?.running?.exe.kind ?? null,
|
chosenBrowser: profileState?.running?.exe.kind ?? null,
|
||||||
|
|
@ -146,6 +151,7 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow
|
||||||
const driver = toStringOrEmpty((req.body as { driver?: unknown })?.driver) as
|
const driver = toStringOrEmpty((req.body as { driver?: unknown })?.driver) as
|
||||||
| "openclaw"
|
| "openclaw"
|
||||||
| "extension"
|
| "extension"
|
||||||
|
| "existing-session"
|
||||||
| "";
|
| "";
|
||||||
|
|
||||||
if (!name) {
|
if (!name) {
|
||||||
|
|
@ -158,7 +164,12 @@ export function registerBrowserBasicRoutes(app: BrowserRouteRegistrar, ctx: Brow
|
||||||
name,
|
name,
|
||||||
color: color || undefined,
|
color: color || undefined,
|
||||||
cdpUrl: cdpUrl || undefined,
|
cdpUrl: cdpUrl || undefined,
|
||||||
driver: driver === "extension" ? "extension" : undefined,
|
driver:
|
||||||
|
driver === "extension"
|
||||||
|
? "extension"
|
||||||
|
: driver === "existing-session"
|
||||||
|
? "existing-session"
|
||||||
|
: undefined,
|
||||||
});
|
});
|
||||||
res.json(result);
|
res.json(result);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,11 @@ import {
|
||||||
PROFILE_POST_RESTART_WS_TIMEOUT_MS,
|
PROFILE_POST_RESTART_WS_TIMEOUT_MS,
|
||||||
resolveCdpReachabilityTimeouts,
|
resolveCdpReachabilityTimeouts,
|
||||||
} from "./cdp-timeouts.js";
|
} from "./cdp-timeouts.js";
|
||||||
|
import {
|
||||||
|
closeChromeMcpSession,
|
||||||
|
ensureChromeMcpAvailable,
|
||||||
|
listChromeMcpTabs,
|
||||||
|
} from "./chrome-mcp.js";
|
||||||
import {
|
import {
|
||||||
isChromeCdpReady,
|
isChromeCdpReady,
|
||||||
isChromeReachable,
|
isChromeReachable,
|
||||||
|
|
@ -60,11 +65,19 @@ export function createProfileAvailability({
|
||||||
});
|
});
|
||||||
|
|
||||||
const isReachable = async (timeoutMs?: number) => {
|
const isReachable = async (timeoutMs?: number) => {
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
await ensureChromeMcpAvailable(profile.name);
|
||||||
|
await listChromeMcpTabs(profile.name);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
const { httpTimeoutMs, wsTimeoutMs } = resolveTimeouts(timeoutMs);
|
const { httpTimeoutMs, wsTimeoutMs } = resolveTimeouts(timeoutMs);
|
||||||
return await isChromeCdpReady(profile.cdpUrl, httpTimeoutMs, wsTimeoutMs);
|
return await isChromeCdpReady(profile.cdpUrl, httpTimeoutMs, wsTimeoutMs);
|
||||||
};
|
};
|
||||||
|
|
||||||
const isHttpReachable = async (timeoutMs?: number) => {
|
const isHttpReachable = async (timeoutMs?: number) => {
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
return await isReachable(timeoutMs);
|
||||||
|
}
|
||||||
const { httpTimeoutMs } = resolveTimeouts(timeoutMs);
|
const { httpTimeoutMs } = resolveTimeouts(timeoutMs);
|
||||||
return await isChromeReachable(profile.cdpUrl, httpTimeoutMs);
|
return await isChromeReachable(profile.cdpUrl, httpTimeoutMs);
|
||||||
};
|
};
|
||||||
|
|
@ -109,6 +122,9 @@ export function createProfileAvailability({
|
||||||
if (previousProfile.driver === "extension") {
|
if (previousProfile.driver === "extension") {
|
||||||
await stopChromeExtensionRelayServer({ cdpUrl: previousProfile.cdpUrl }).catch(() => false);
|
await stopChromeExtensionRelayServer({ cdpUrl: previousProfile.cdpUrl }).catch(() => false);
|
||||||
}
|
}
|
||||||
|
if (previousProfile.driver === "existing-session") {
|
||||||
|
await closeChromeMcpSession(previousProfile.name).catch(() => false);
|
||||||
|
}
|
||||||
await closePlaywrightBrowserConnectionForProfile(previousProfile.cdpUrl);
|
await closePlaywrightBrowserConnectionForProfile(previousProfile.cdpUrl);
|
||||||
if (previousProfile.cdpUrl !== profile.cdpUrl) {
|
if (previousProfile.cdpUrl !== profile.cdpUrl) {
|
||||||
await closePlaywrightBrowserConnectionForProfile(profile.cdpUrl);
|
await closePlaywrightBrowserConnectionForProfile(profile.cdpUrl);
|
||||||
|
|
@ -138,6 +154,10 @@ export function createProfileAvailability({
|
||||||
|
|
||||||
const ensureBrowserAvailable = async (): Promise<void> => {
|
const ensureBrowserAvailable = async (): Promise<void> => {
|
||||||
await reconcileProfileRuntime();
|
await reconcileProfileRuntime();
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
await ensureChromeMcpAvailable(profile.name);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const current = state();
|
const current = state();
|
||||||
const remoteCdp = capabilities.isRemote;
|
const remoteCdp = capabilities.isRemote;
|
||||||
const attachOnly = profile.attachOnly;
|
const attachOnly = profile.attachOnly;
|
||||||
|
|
@ -238,6 +258,10 @@ export function createProfileAvailability({
|
||||||
|
|
||||||
const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => {
|
const stopRunningBrowser = async (): Promise<{ stopped: boolean }> => {
|
||||||
await reconcileProfileRuntime();
|
await reconcileProfileRuntime();
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
const stopped = await closeChromeMcpSession(profile.name);
|
||||||
|
return { stopped };
|
||||||
|
}
|
||||||
if (capabilities.requiresRelay) {
|
if (capabilities.requiresRelay) {
|
||||||
const stopped = await stopChromeExtensionRelayServer({
|
const stopped = await stopChromeExtensionRelayServer({
|
||||||
cdpUrl: profile.cdpUrl,
|
cdpUrl: profile.cdpUrl,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,102 @@
|
||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { createBrowserRouteContext } from "./server-context.js";
|
||||||
|
import type { BrowserServerState } from "./server-context.js";
|
||||||
|
|
||||||
|
vi.mock("./chrome-mcp.js", () => ({
|
||||||
|
closeChromeMcpSession: vi.fn(async () => true),
|
||||||
|
ensureChromeMcpAvailable: vi.fn(async () => {}),
|
||||||
|
focusChromeMcpTab: vi.fn(async () => {}),
|
||||||
|
listChromeMcpTabs: vi.fn(async () => [
|
||||||
|
{ targetId: "7", title: "", url: "https://example.com", type: "page" },
|
||||||
|
]),
|
||||||
|
openChromeMcpTab: vi.fn(async () => ({
|
||||||
|
targetId: "8",
|
||||||
|
title: "",
|
||||||
|
url: "https://openclaw.ai",
|
||||||
|
type: "page",
|
||||||
|
})),
|
||||||
|
closeChromeMcpTab: vi.fn(async () => {}),
|
||||||
|
getChromeMcpPid: vi.fn(() => 4321),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import * as chromeMcp from "./chrome-mcp.js";
|
||||||
|
|
||||||
|
function makeState(): BrowserServerState {
|
||||||
|
return {
|
||||||
|
server: null,
|
||||||
|
port: 0,
|
||||||
|
resolved: {
|
||||||
|
enabled: true,
|
||||||
|
evaluateEnabled: true,
|
||||||
|
controlPort: 18791,
|
||||||
|
cdpPortRangeStart: 18800,
|
||||||
|
cdpPortRangeEnd: 18899,
|
||||||
|
cdpProtocol: "http",
|
||||||
|
cdpHost: "127.0.0.1",
|
||||||
|
cdpIsLoopback: true,
|
||||||
|
remoteCdpTimeoutMs: 1500,
|
||||||
|
remoteCdpHandshakeTimeoutMs: 3000,
|
||||||
|
color: "#FF4500",
|
||||||
|
headless: false,
|
||||||
|
noSandbox: false,
|
||||||
|
attachOnly: false,
|
||||||
|
defaultProfile: "chrome-live",
|
||||||
|
profiles: {
|
||||||
|
"chrome-live": {
|
||||||
|
cdpPort: 18801,
|
||||||
|
color: "#0066CC",
|
||||||
|
driver: "existing-session",
|
||||||
|
attachOnly: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
extraArgs: [],
|
||||||
|
ssrfPolicy: { dangerouslyAllowPrivateNetwork: true },
|
||||||
|
},
|
||||||
|
profiles: new Map(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("browser server-context existing-session profile", () => {
|
||||||
|
it("routes tab operations through the Chrome MCP backend", async () => {
|
||||||
|
const state = makeState();
|
||||||
|
const ctx = createBrowserRouteContext({ getState: () => state });
|
||||||
|
const live = ctx.forProfile("chrome-live");
|
||||||
|
|
||||||
|
vi.mocked(chromeMcp.listChromeMcpTabs)
|
||||||
|
.mockResolvedValueOnce([
|
||||||
|
{ targetId: "7", title: "", url: "https://example.com", type: "page" },
|
||||||
|
])
|
||||||
|
.mockResolvedValueOnce([
|
||||||
|
{ targetId: "8", title: "", url: "https://openclaw.ai", type: "page" },
|
||||||
|
])
|
||||||
|
.mockResolvedValueOnce([
|
||||||
|
{ targetId: "8", title: "", url: "https://openclaw.ai", type: "page" },
|
||||||
|
])
|
||||||
|
.mockResolvedValueOnce([
|
||||||
|
{ targetId: "7", title: "", url: "https://example.com", type: "page" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
await live.ensureBrowserAvailable();
|
||||||
|
const tabs = await live.listTabs();
|
||||||
|
expect(tabs.map((tab) => tab.targetId)).toEqual(["7"]);
|
||||||
|
|
||||||
|
const opened = await live.openTab("https://openclaw.ai");
|
||||||
|
expect(opened.targetId).toBe("8");
|
||||||
|
|
||||||
|
const selected = await live.ensureTabAvailable();
|
||||||
|
expect(selected.targetId).toBe("8");
|
||||||
|
|
||||||
|
await live.focusTab("7");
|
||||||
|
await live.stopRunningBrowser();
|
||||||
|
|
||||||
|
expect(chromeMcp.ensureChromeMcpAvailable).toHaveBeenCalledWith("chrome-live");
|
||||||
|
expect(chromeMcp.listChromeMcpTabs).toHaveBeenCalledWith("chrome-live");
|
||||||
|
expect(chromeMcp.openChromeMcpTab).toHaveBeenCalledWith("chrome-live", "https://openclaw.ai");
|
||||||
|
expect(chromeMcp.focusChromeMcpTab).toHaveBeenCalledWith("chrome-live", "7");
|
||||||
|
expect(chromeMcp.closeChromeMcpSession).toHaveBeenCalledWith("chrome-live");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import { fetchOk, normalizeCdpHttpBaseForJsonEndpoints } from "./cdp.helpers.js";
|
import { fetchOk, normalizeCdpHttpBaseForJsonEndpoints } from "./cdp.helpers.js";
|
||||||
import { appendCdpPath } from "./cdp.js";
|
import { appendCdpPath } from "./cdp.js";
|
||||||
|
import { closeChromeMcpTab, focusChromeMcpTab } from "./chrome-mcp.js";
|
||||||
import type { ResolvedBrowserProfile } from "./config.js";
|
import type { ResolvedBrowserProfile } from "./config.js";
|
||||||
import { BrowserTabNotFoundError, BrowserTargetAmbiguousError } from "./errors.js";
|
import { BrowserTabNotFoundError, BrowserTargetAmbiguousError } from "./errors.js";
|
||||||
import { getBrowserProfileCapabilities } from "./profile-capabilities.js";
|
import { getBrowserProfileCapabilities } from "./profile-capabilities.js";
|
||||||
|
|
@ -111,6 +112,13 @@ export function createProfileSelectionOps({
|
||||||
const focusTab = async (targetId: string): Promise<void> => {
|
const focusTab = async (targetId: string): Promise<void> => {
|
||||||
const resolvedTargetId = await resolveTargetIdOrThrow(targetId);
|
const resolvedTargetId = await resolveTargetIdOrThrow(targetId);
|
||||||
|
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
await focusChromeMcpTab(profile.name, resolvedTargetId);
|
||||||
|
const profileState = getProfileState();
|
||||||
|
profileState.lastTargetId = resolvedTargetId;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (capabilities.usesPersistentPlaywright) {
|
if (capabilities.usesPersistentPlaywright) {
|
||||||
const mod = await getPwAiModule({ mode: "strict" });
|
const mod = await getPwAiModule({ mode: "strict" });
|
||||||
const focusPageByTargetIdViaPlaywright = (mod as Partial<PwAiModule> | null)
|
const focusPageByTargetIdViaPlaywright = (mod as Partial<PwAiModule> | null)
|
||||||
|
|
@ -134,6 +142,11 @@ export function createProfileSelectionOps({
|
||||||
const closeTab = async (targetId: string): Promise<void> => {
|
const closeTab = async (targetId: string): Promise<void> => {
|
||||||
const resolvedTargetId = await resolveTargetIdOrThrow(targetId);
|
const resolvedTargetId = await resolveTargetIdOrThrow(targetId);
|
||||||
|
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
await closeChromeMcpTab(profile.name, resolvedTargetId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// For remote profiles, use Playwright's persistent connection to close tabs
|
// For remote profiles, use Playwright's persistent connection to close tabs
|
||||||
if (capabilities.usesPersistentPlaywright) {
|
if (capabilities.usesPersistentPlaywright) {
|
||||||
const mod = await getPwAiModule({ mode: "strict" });
|
const mod = await getPwAiModule({ mode: "strict" });
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import { CDP_JSON_NEW_TIMEOUT_MS } from "./cdp-timeouts.js";
|
import { CDP_JSON_NEW_TIMEOUT_MS } from "./cdp-timeouts.js";
|
||||||
import { fetchJson, fetchOk, normalizeCdpHttpBaseForJsonEndpoints } from "./cdp.helpers.js";
|
import { fetchJson, fetchOk, normalizeCdpHttpBaseForJsonEndpoints } from "./cdp.helpers.js";
|
||||||
import { appendCdpPath, createTargetViaCdp, normalizeCdpWsUrl } from "./cdp.js";
|
import { appendCdpPath, createTargetViaCdp, normalizeCdpWsUrl } from "./cdp.js";
|
||||||
|
import { listChromeMcpTabs, openChromeMcpTab } from "./chrome-mcp.js";
|
||||||
import type { ResolvedBrowserProfile } from "./config.js";
|
import type { ResolvedBrowserProfile } from "./config.js";
|
||||||
import {
|
import {
|
||||||
assertBrowserNavigationAllowed,
|
assertBrowserNavigationAllowed,
|
||||||
|
|
@ -65,6 +66,10 @@ export function createProfileTabOps({
|
||||||
const capabilities = getBrowserProfileCapabilities(profile);
|
const capabilities = getBrowserProfileCapabilities(profile);
|
||||||
|
|
||||||
const listTabs = async (): Promise<BrowserTab[]> => {
|
const listTabs = async (): Promise<BrowserTab[]> => {
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
return await listChromeMcpTabs(profile.name);
|
||||||
|
}
|
||||||
|
|
||||||
if (capabilities.usesPersistentPlaywright) {
|
if (capabilities.usesPersistentPlaywright) {
|
||||||
const mod = await getPwAiModule({ mode: "strict" });
|
const mod = await getPwAiModule({ mode: "strict" });
|
||||||
const listPagesViaPlaywright = (mod as Partial<PwAiModule> | null)?.listPagesViaPlaywright;
|
const listPagesViaPlaywright = (mod as Partial<PwAiModule> | null)?.listPagesViaPlaywright;
|
||||||
|
|
@ -134,6 +139,15 @@ export function createProfileTabOps({
|
||||||
const openTab = async (url: string): Promise<BrowserTab> => {
|
const openTab = async (url: string): Promise<BrowserTab> => {
|
||||||
const ssrfPolicyOpts = withBrowserNavigationPolicy(state().resolved.ssrfPolicy);
|
const ssrfPolicyOpts = withBrowserNavigationPolicy(state().resolved.ssrfPolicy);
|
||||||
|
|
||||||
|
if (profile.driver === "existing-session") {
|
||||||
|
await assertBrowserNavigationAllowed({ url, ...ssrfPolicyOpts });
|
||||||
|
const page = await openChromeMcpTab(profile.name, url);
|
||||||
|
const profileState = getProfileState();
|
||||||
|
profileState.lastTargetId = page.targetId;
|
||||||
|
await assertBrowserNavigationResultAllowed({ url: page.url, ...ssrfPolicyOpts });
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
if (capabilities.usesPersistentPlaywright) {
|
if (capabilities.usesPersistentPlaywright) {
|
||||||
const mod = await getPwAiModule({ mode: "strict" });
|
const mod = await getPwAiModule({ mode: "strict" });
|
||||||
const createPageViaPlaywright = (mod as Partial<PwAiModule> | null)?.createPageViaPlaywright;
|
const createPageViaPlaywright = (mod as Partial<PwAiModule> | null)?.createPageViaPlaywright;
|
||||||
|
|
|
||||||
|
|
@ -162,12 +162,22 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon
|
||||||
|
|
||||||
let tabCount = 0;
|
let tabCount = 0;
|
||||||
let running = false;
|
let running = false;
|
||||||
|
const profileCtx = createProfileContext(opts, profile);
|
||||||
|
|
||||||
if (profileState?.running) {
|
if (profile.driver === "existing-session") {
|
||||||
|
try {
|
||||||
|
running = await profileCtx.isReachable(300);
|
||||||
|
if (running) {
|
||||||
|
const tabs = await profileCtx.listTabs();
|
||||||
|
tabCount = tabs.filter((t) => t.type === "page").length;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Chrome MCP not available
|
||||||
|
}
|
||||||
|
} else if (profileState?.running) {
|
||||||
running = true;
|
running = true;
|
||||||
try {
|
try {
|
||||||
const ctx = createProfileContext(opts, profile);
|
const tabs = await profileCtx.listTabs();
|
||||||
const tabs = await ctx.listTabs();
|
|
||||||
tabCount = tabs.filter((t) => t.type === "page").length;
|
tabCount = tabs.filter((t) => t.type === "page").length;
|
||||||
} catch {
|
} catch {
|
||||||
// Browser might not be responsive
|
// Browser might not be responsive
|
||||||
|
|
@ -178,8 +188,7 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon
|
||||||
const reachable = await isChromeReachable(profile.cdpUrl, 200);
|
const reachable = await isChromeReachable(profile.cdpUrl, 200);
|
||||||
if (reachable) {
|
if (reachable) {
|
||||||
running = true;
|
running = true;
|
||||||
const ctx = createProfileContext(opts, profile);
|
const tabs = await profileCtx.listTabs().catch(() => []);
|
||||||
const tabs = await ctx.listTabs().catch(() => []);
|
|
||||||
tabCount = tabs.filter((t) => t.type === "page").length;
|
tabCount = tabs.filter((t) => t.type === "page").length;
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
|
|
@ -192,6 +201,7 @@ export function createBrowserRouteContext(opts: ContextOptions): BrowserRouteCon
|
||||||
cdpPort: profile.cdpPort,
|
cdpPort: profile.cdpPort,
|
||||||
cdpUrl: profile.cdpUrl,
|
cdpUrl: profile.cdpUrl,
|
||||||
color: profile.color,
|
color: profile.color,
|
||||||
|
driver: profile.driver,
|
||||||
running,
|
running,
|
||||||
tabCount,
|
tabCount,
|
||||||
isDefault: name === current.resolved.defaultProfile,
|
isDefault: name === current.resolved.defaultProfile,
|
||||||
|
|
|
||||||
|
|
@ -56,6 +56,7 @@ export type ProfileStatus = {
|
||||||
cdpPort: number;
|
cdpPort: number;
|
||||||
cdpUrl: string;
|
cdpUrl: string;
|
||||||
color: string;
|
color: string;
|
||||||
|
driver: ResolvedBrowserProfile["driver"];
|
||||||
running: boolean;
|
running: boolean;
|
||||||
tabCount: number;
|
tabCount: number;
|
||||||
isDefault: boolean;
|
isDefault: boolean;
|
||||||
|
|
|
||||||
|
|
@ -407,7 +407,8 @@ export function registerBrowserManageCommands(
|
||||||
const def = p.isDefault ? " [default]" : "";
|
const def = p.isDefault ? " [default]" : "";
|
||||||
const loc = p.isRemote ? `cdpUrl: ${p.cdpUrl}` : `port: ${p.cdpPort}`;
|
const loc = p.isRemote ? `cdpUrl: ${p.cdpUrl}` : `port: ${p.cdpPort}`;
|
||||||
const remote = p.isRemote ? " [remote]" : "";
|
const remote = p.isRemote ? " [remote]" : "";
|
||||||
return `${p.name}: ${status}${tabs}${def}${remote}\n ${loc}, color: ${p.color}`;
|
const driver = p.driver !== "openclaw" ? ` [${p.driver}]` : "";
|
||||||
|
return `${p.name}: ${status}${tabs}${def}${remote}${driver}\n ${loc}, color: ${p.color}`;
|
||||||
})
|
})
|
||||||
.join("\n"),
|
.join("\n"),
|
||||||
);
|
);
|
||||||
|
|
@ -420,7 +421,10 @@ export function registerBrowserManageCommands(
|
||||||
.requiredOption("--name <name>", "Profile name (lowercase, numbers, hyphens)")
|
.requiredOption("--name <name>", "Profile name (lowercase, numbers, hyphens)")
|
||||||
.option("--color <hex>", "Profile color (hex format, e.g. #0066CC)")
|
.option("--color <hex>", "Profile color (hex format, e.g. #0066CC)")
|
||||||
.option("--cdp-url <url>", "CDP URL for remote Chrome (http/https)")
|
.option("--cdp-url <url>", "CDP URL for remote Chrome (http/https)")
|
||||||
.option("--driver <driver>", "Profile driver (openclaw|extension). Default: openclaw")
|
.option(
|
||||||
|
"--driver <driver>",
|
||||||
|
"Profile driver (openclaw|extension|existing-session). Default: openclaw",
|
||||||
|
)
|
||||||
.action(
|
.action(
|
||||||
async (opts: { name: string; color?: string; cdpUrl?: string; driver?: string }, cmd) => {
|
async (opts: { name: string; color?: string; cdpUrl?: string; driver?: string }, cmd) => {
|
||||||
const parent = parentOpts(cmd);
|
const parent = parentOpts(cmd);
|
||||||
|
|
@ -434,7 +438,12 @@ export function registerBrowserManageCommands(
|
||||||
name: opts.name,
|
name: opts.name,
|
||||||
color: opts.color,
|
color: opts.color,
|
||||||
cdpUrl: opts.cdpUrl,
|
cdpUrl: opts.cdpUrl,
|
||||||
driver: opts.driver === "extension" ? "extension" : undefined,
|
driver:
|
||||||
|
opts.driver === "extension"
|
||||||
|
? "extension"
|
||||||
|
: opts.driver === "existing-session"
|
||||||
|
? "existing-session"
|
||||||
|
: undefined,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{ timeoutMs: 10_000 },
|
{ timeoutMs: 10_000 },
|
||||||
|
|
@ -446,7 +455,11 @@ export function registerBrowserManageCommands(
|
||||||
defaultRuntime.log(
|
defaultRuntime.log(
|
||||||
info(
|
info(
|
||||||
`🦞 Created profile "${result.profile}"\n${loc}\n color: ${result.color}${
|
`🦞 Created profile "${result.profile}"\n${loc}\n color: ${result.color}${
|
||||||
opts.driver === "extension" ? "\n driver: extension" : ""
|
opts.driver === "extension"
|
||||||
|
? "\n driver: extension"
|
||||||
|
: opts.driver === "existing-session"
|
||||||
|
? "\n driver: existing-session"
|
||||||
|
: ""
|
||||||
}`,
|
}`,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ export type BrowserProfileConfig = {
|
||||||
/** CDP URL for this profile (use for remote Chrome). */
|
/** CDP URL for this profile (use for remote Chrome). */
|
||||||
cdpUrl?: string;
|
cdpUrl?: string;
|
||||||
/** Profile driver (default: openclaw). */
|
/** Profile driver (default: openclaw). */
|
||||||
driver?: "openclaw" | "clawd" | "extension";
|
driver?: "openclaw" | "clawd" | "extension" | "existing-session";
|
||||||
/** If true, never launch a browser for this profile; only attach. Falls back to browser.attachOnly. */
|
/** If true, never launch a browser for this profile; only attach. Falls back to browser.attachOnly. */
|
||||||
attachOnly?: boolean;
|
attachOnly?: boolean;
|
||||||
/** Profile color (hex). Auto-assigned at creation. */
|
/** Profile color (hex). Auto-assigned at creation. */
|
||||||
|
|
|
||||||
|
|
@ -360,7 +360,12 @@ export const OpenClawSchema = z
|
||||||
cdpPort: z.number().int().min(1).max(65535).optional(),
|
cdpPort: z.number().int().min(1).max(65535).optional(),
|
||||||
cdpUrl: z.string().optional(),
|
cdpUrl: z.string().optional(),
|
||||||
driver: z
|
driver: z
|
||||||
.union([z.literal("openclaw"), z.literal("clawd"), z.literal("extension")])
|
.union([
|
||||||
|
z.literal("openclaw"),
|
||||||
|
z.literal("clawd"),
|
||||||
|
z.literal("extension"),
|
||||||
|
z.literal("existing-session"),
|
||||||
|
])
|
||||||
.optional(),
|
.optional(),
|
||||||
attachOnly: z.boolean().optional(),
|
attachOnly: z.boolean().optional(),
|
||||||
color: HexColorSchema,
|
color: HexColorSchema,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue