Compare commits

..
Author SHA1 Message Date
Mohamed Boudra 9eec33773a refactor(config): use schema stripping for persisted fields 2026-07-02 18:44:17 +02:00
Mohamed Boudra 5616c6af6a fix(config): tolerate unknown persisted fields
Config files can be shared across checkouts with different schema versions. Strip unrecognized keys on load so read paths like daemon status do not fail before they can report state; known malformed fields still fail validation.
2026-07-02 18:39:46 +02:00
113 changed files with 7096 additions and 14995 deletions
-20
View File
@@ -1,31 +1,11 @@
.git
.debug.conversations
.debug
.dev
.playwright-mcp
**/.playwright-mcp
.paseo
**/.paseo-provider-history
.plans
.tasks
.valknut
.claude/settings.local.json
**/.claude/settings.local.json
.claude/scheduled_tasks.lock
.claude/worktrees
.wrangler
**/.wrangler
**/.tanstack
PLAN.md
valknut-report.html
valknut-report.json
.env*
**/.env*
.dev.vars
**/.dev.vars
*.pem
**/*.pem
**/.secrets
**/node_modules
**/dist
**/build
+39 -23
View File
@@ -5,22 +5,7 @@ on:
branches: [main]
paths:
- "docker/**"
- ".dockerignore"
- ".github/workflows/docker.yml"
- "package.json"
- "package-lock.json"
- "patches/**"
- "scripts/**"
- "tsconfig.json"
- "tsconfig.base.json"
- "packages/app/**"
- "packages/cli/**"
- "packages/client/**"
- "packages/expo-two-way-audio/**"
- "packages/highlight/**"
- "packages/protocol/**"
- "packages/relay/**"
- "packages/server/**"
push:
branches: [main]
tags:
@@ -28,7 +13,7 @@ on:
workflow_dispatch:
inputs:
paseo_version:
description: "Expected source version to build. Required when publish is true."
description: "Paseo version to build. Required when publish is true."
required: false
default: ""
publish:
@@ -39,6 +24,15 @@ on:
options:
- "false"
- "true"
source_build:
description: "Build from the checked-out source tree instead of npm."
required: false
default: "auto"
type: choice
options:
- auto
- "false"
- "true"
publish_latest:
description: "Also publish ghcr.io/getpaseo/paseo:latest. Ignored for prereleases."
required: false
@@ -63,6 +57,7 @@ jobs:
image: ${{ steps.values.outputs.image }}
install_version: ${{ steps.values.outputs.install_version }}
publish: ${{ steps.values.outputs.publish }}
source_build: ${{ steps.values.outputs.source_build }}
check_tag: ${{ steps.values.outputs.check_tag }}
publish_tags: ${{ steps.values.outputs.publish_tags }}
steps:
@@ -73,6 +68,7 @@ jobs:
INPUT_PASEO_VERSION: ${{ inputs.paseo_version }}
INPUT_PUBLISH: ${{ inputs.publish }}
INPUT_PUBLISH_LATEST: ${{ inputs.publish_latest }}
INPUT_SOURCE_BUILD: ${{ inputs.source_build }}
REPO_OWNER: ${{ github.repository_owner }}
REF_NAME: ${{ github.ref_name }}
run: |
@@ -80,9 +76,9 @@ jobs:
owner="$(printf '%s' "${REPO_OWNER}" | tr '[:upper:]' '[:lower:]')"
image="ghcr.io/${owner}/paseo"
package_version="$(node -p "require('./package.json').version")"
install_version="${INPUT_PASEO_VERSION:-${package_version}}"
install_version="${INPUT_PASEO_VERSION:-latest}"
publish=false
source_build=false
publish_latest=false
prerelease=false
@@ -91,6 +87,7 @@ jobs:
publish=true
if [[ "${REF_NAME}" == *-* ]]; then
prerelease=true
source_build=true
else
publish_latest=true
fi
@@ -107,6 +104,24 @@ jobs:
prerelease=true
fi
case "${INPUT_SOURCE_BUILD:-auto}" in
true)
source_build=true
;;
false)
source_build=false
;;
auto)
if [[ "${prerelease}" == "true" ]]; then
source_build=true
fi
;;
*)
echo "::error::source_build must be auto, true, or false."
exit 1
;;
esac
if [[ "${INPUT_PUBLISH_LATEST:-false}" == "true" && "${prerelease}" != "true" ]]; then
publish_latest=true
fi
@@ -122,13 +137,14 @@ jobs:
echo "image=${image}"
echo "install_version=${install_version}"
echo "publish=${publish}"
echo "source_build=${source_build}"
echo "check_tag=${check_tag}"
echo "publish_tags<<EOF"
echo "${publish_tags}"
echo "EOF"
} >> "$GITHUB_OUTPUT"
echo "Resolved image=${image} install_version=${install_version} publish=${publish}"
echo "Resolved image=${image} install_version=${install_version} publish=${publish} source_build=${source_build}"
build:
needs: setup
@@ -144,8 +160,8 @@ jobs:
- uses: docker/build-push-action@v7
with:
context: .
file: docker/base/Dockerfile
context: ${{ needs.setup.outputs.source_build == 'true' && '.' || 'docker/base' }}
file: ${{ needs.setup.outputs.source_build == 'true' && 'docker/base/Dockerfile.source' || 'docker/base/Dockerfile' }}
platforms: ${{ env.PLATFORMS }}
build-args: |
PASEO_VERSION=${{ needs.setup.outputs.install_version }}
@@ -177,8 +193,8 @@ jobs:
- uses: docker/build-push-action@v7
with:
context: .
file: docker/base/Dockerfile
context: ${{ needs.setup.outputs.source_build == 'true' && '.' || 'docker/base' }}
file: ${{ needs.setup.outputs.source_build == 'true' && 'docker/base/Dockerfile.source' || 'docker/base/Dockerfile' }}
platforms: ${{ env.PLATFORMS }}
build-args: |
PASEO_VERSION=${{ needs.setup.outputs.install_version }}
-26
View File
@@ -1,31 +1,5 @@
# Changelog
## 0.1.104-beta.3 - 2026-07-04
### Added
- Agents can drive the in-app browser with page snapshots, trusted input, dialogs, and tab controls ([#1881](https://github.com/getpaseo/paseo/pull/1881))
- Inspect, annotate, and send page elements from a browser tab to the agent ([#1708](https://github.com/getpaseo/paseo/pull/1708) by [@huiliaoning](https://github.com/huiliaoning))
- Schedules screen to create and manage recurring agents ([#1246](https://github.com/getpaseo/paseo/pull/1246))
- Open a project from anywhere with Cmd+O ([#1849](https://github.com/getpaseo/paseo/pull/1849))
- Agents can rename workspaces after they understand the task ([#1876](https://github.com/getpaseo/paseo/pull/1876))
- Claude Ultra Code is available for supported Claude models ([#1872](https://github.com/getpaseo/paseo/pull/1872))
- ByteDance TRAE CLI available as an agent provider ([#1831](https://github.com/getpaseo/paseo/pull/1831) by [@park0er](https://github.com/park0er))
### Improved
- Workspaces created by agents now get readable generated names ([#1887](https://github.com/getpaseo/paseo/pull/1887))
- Browser tabs opened by agents stay in the background until you switch to them ([#1875](https://github.com/getpaseo/paseo/pull/1875))
- Clearer cards when an agent asks a question ([#1643](https://github.com/getpaseo/paseo/pull/1643) by [@cleiter](https://github.com/cleiter))
### Fixed
- Docker images keep running during provider cleanup and diagnostics ([#1877](https://github.com/getpaseo/paseo/pull/1877))
- New Workspace drafts survive archiving a workspace ([#1838](https://github.com/getpaseo/paseo/pull/1838))
- Composer autocomplete stays open after switching screens ([#1851](https://github.com/getpaseo/paseo/pull/1851))
- Claude usage appears when a quota window has no scheduled reset ([#1855](https://github.com/getpaseo/paseo/pull/1855))
- New workspace action shows for non-git projects in the sidebar ([#1857](https://github.com/getpaseo/paseo/pull/1857) by [@cleiter](https://github.com/cleiter))
## 0.1.103 - 2026-07-01
### Added
+28 -29
View File
@@ -21,35 +21,34 @@ This is an npm workspace monorepo:
At the start of non-trivial work, list `docs/` and skim anything relevant to the task. When you learn something meta worth preserving — a gotcha, a convention, a workflow, a piece of system context that will outlive the current task — update an existing doc or propose a new one. Code-level facts belong in inline comments next to the code; system, process, and gotcha-level facts belong in `docs/`.
| Doc | What's in it |
| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ |
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
| [docs/browser-capture-harness.md](docs/browser-capture-harness.md) | Real-Electron browser screenshot harness and compositor-surface gotcha |
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
| Doc | What's in it |
| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
## Quick start
+6 -30
View File
@@ -1,33 +1,10 @@
# syntax=docker/dockerfile:1
ARG NODE_IMAGE=node:22-bookworm-slim
FROM --platform=$BUILDPLATFORM ${NODE_IMAGE} AS source-pack
ARG PASEO_VERSION
ENV ONNXRUNTIME_NODE_INSTALL=skip
WORKDIR /tmp/paseo-src
COPY . .
RUN set -eux; \
if [ -n "${PASEO_VERSION:-}" ]; then \
test "$(node -p "require('./package.json').version")" = "${PASEO_VERSION}"; \
fi; \
node -e 'const fs=require("node:fs"); const pkg=JSON.parse(fs.readFileSync("package.json","utf8")); delete pkg.scripts.prepare; fs.writeFileSync("package.json", `${JSON.stringify(pkg)}\n`);'; \
npm ci
RUN set -eux; \
mkdir -p /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/highlight --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/relay --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/protocol --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/client --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/server --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/cli --pack-destination /tmp/paseo-packs
FROM ${NODE_IMAGE}
ARG PASEO_VERSION=latest
ENV HOME=/home/paseo \
PASEO_HOME=/home/paseo/.paseo \
PASEO_LISTEN=0.0.0.0:6767 \
@@ -52,14 +29,13 @@ RUN set -eux; \
gosu \
lbzip2 \
openssh-client \
procps \
tini; \
rm -rf /var/lib/apt/lists/*
COPY --from=source-pack /tmp/paseo-packs /tmp/paseo-packs
RUN set -eux; \
npm install -g /tmp/paseo-packs/*.tgz; \
rm -rf /tmp/paseo-packs; \
npm install -g --omit=optional \
"@getpaseo/server@${PASEO_VERSION}" \
"@getpaseo/cli@${PASEO_VERSION}"; \
npm cache clean --force; \
server_entry="$(npm root -g)/@getpaseo/server/dist/scripts/supervisor-entrypoint.js"; \
test -f "$server_entry"; \
@@ -90,7 +66,7 @@ RUN set -eux; \
"$XDG_CACHE_HOME"; \
chown -R paseo:paseo /home/paseo /workspace
COPY docker/base/rootfs/ /
COPY rootfs/ /
RUN chmod +x /usr/local/bin/paseo-docker-entrypoint
WORKDIR /workspace
+101
View File
@@ -0,0 +1,101 @@
# syntax=docker/dockerfile:1
ARG NODE_IMAGE=node:22-bookworm-slim
FROM --platform=$BUILDPLATFORM ${NODE_IMAGE} AS source-pack
ARG PASEO_VERSION
ENV ONNXRUNTIME_NODE_INSTALL=skip
WORKDIR /tmp/paseo-src
COPY . .
RUN set -eux; \
test "$(node -p "require('./package.json').version")" = "${PASEO_VERSION}"; \
node -e 'const fs=require("node:fs"); const pkg=JSON.parse(fs.readFileSync("package.json","utf8")); delete pkg.scripts.prepare; fs.writeFileSync("package.json", `${JSON.stringify(pkg)}\n`);'; \
npm ci
RUN set -eux; \
mkdir -p /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/highlight --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/relay --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/protocol --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/client --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/server --pack-destination /tmp/paseo-packs; \
npm pack --workspace=@getpaseo/cli --pack-destination /tmp/paseo-packs
FROM ${NODE_IMAGE}
ENV HOME=/home/paseo \
PASEO_HOME=/home/paseo/.paseo \
PASEO_LISTEN=0.0.0.0:6767 \
PASEO_WEB_UI_ENABLED=true \
PASEO_LOG_FORMAT=json \
PASEO_LOG_LEVEL=info \
CLAUDE_CONFIG_DIR=/home/paseo/.claude \
CODEX_HOME=/home/paseo/.codex \
XDG_CONFIG_HOME=/home/paseo/.config \
XDG_DATA_HOME=/home/paseo/.local/share \
XDG_STATE_HOME=/home/paseo/.local/state \
XDG_CACHE_HOME=/home/paseo/.cache \
ONNXRUNTIME_NODE_INSTALL=skip
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends \
bash \
ca-certificates \
curl \
git \
gosu \
lbzip2 \
openssh-client \
tini; \
rm -rf /var/lib/apt/lists/*
COPY --from=source-pack /tmp/paseo-packs /tmp/paseo-packs
RUN set -eux; \
npm install -g --omit=optional /tmp/paseo-packs/*.tgz; \
rm -rf /tmp/paseo-packs; \
npm cache clean --force; \
server_entry="$(npm root -g)/@getpaseo/server/dist/scripts/supervisor-entrypoint.js"; \
test -f "$server_entry"; \
printf '%s\n' "$server_entry" > /etc/paseo-server-entry; \
node --check "$server_entry"
RUN set -eux; \
existing_group="$(getent group 1000 | cut -d: -f1 || true)"; \
if [ -n "$existing_group" ] && [ "$existing_group" != "paseo" ]; then \
groupmod --new-name paseo "$existing_group"; \
elif [ -z "$existing_group" ]; then \
groupadd --gid 1000 paseo; \
fi; \
existing_user="$(getent passwd 1000 | cut -d: -f1 || true)"; \
if [ -n "$existing_user" ] && [ "$existing_user" != "paseo" ]; then \
usermod --login paseo --gid paseo --home /home/paseo --shell /bin/bash "$existing_user"; \
elif [ -z "$existing_user" ]; then \
useradd --uid 1000 --gid paseo --create-home --home-dir /home/paseo --shell /bin/bash paseo; \
fi; \
mkdir -p \
/workspace \
"$PASEO_HOME" \
"$CLAUDE_CONFIG_DIR" \
"$CODEX_HOME" \
"$XDG_CONFIG_HOME" \
"$XDG_DATA_HOME" \
"$XDG_STATE_HOME" \
"$XDG_CACHE_HOME"; \
chown -R paseo:paseo /home/paseo /workspace
COPY docker/base/rootfs/ /
RUN chmod +x /usr/local/bin/paseo-docker-entrypoint
WORKDIR /workspace
EXPOSE 6767
VOLUME ["/home/paseo"]
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
CMD node -e "const listen=process.env.PASEO_LISTEN||'0.0.0.0:6767'; const m=listen.match(/:(\\d+)$/); const port=m?Number(m[1]):6767; require('http').get({hostname:'127.0.0.1',port,path:'/api/health'},r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))"
ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/paseo-docker-entrypoint"]
+1 -1
View File
@@ -138,7 +138,7 @@ Electron wrapper for macOS, Linux, and Windows.
> **Window-state v1 limitation:** only the _first_ window of a session restores and persists saved geometry (size/position/maximized). Windows opened via ⌘⇧N / second-instance / "Open in new window" open at the default size, OS-cascaded, and do not persist — this avoids every window stacking on the same restored bounds and fighting over the single window-state store. Lifting this needs per-window state keys.
>
> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus still records the workspace-active browser for UI state and `list_tabs` reporting, but agent automation targets only explicit browser ids returned by `browser_new_tab` or `browser_list_tabs`. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up.
> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus and agent automation targets are intentionally separate: the workspace-active browser follows the user's focused tab, while the agent-active browser is the default target for browser MCP commands. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up.
### `packages/website` — Marketing site
-69
View File
@@ -1,69 +0,0 @@
# Browser Capture Harness
The desktop capture harness is the real-Electron verification path for browser screenshots.
It validates the compositor behavior that unit tests cannot see:
- the resident automation `<webview>` starts in the production parking state;
- the parked guest remains paintable and has a copyable viewport frame;
- the resident webview guest is sized to 1280x800 logical pixels;
- multiple resident webviews are parked as an overlapping stack without per-capture
stacking changes;
- a newly attached resident webview whose first useful frame is delayed can be captured
by retrying until the frame appears;
- both viewport `capturePage` and full-page CDP screenshots return real pixels from
the permanent production parking state;
- guest background throttling can be disabled once at attach without per-capture
renderer coordination.
Run it with the repo Electron:
```bash
npm run capture-harness --workspace=@getpaseo/desktop
```
Run the browser automation fixture with:
```bash
PASEO_CAPTURE_HARNESS_GROUP=automation npm run capture-harness --workspace=@getpaseo/desktop
```
The automation group uses a real guest webview to verify the page-side ref contract:
ARIA-like snapshot text includes headings, static text, and controls; refs survive
`pushState` when the element still matches; same-URL rerenders stale old refs; and a
file-input ref can be resolved to a CDP backend node id for upload. It also verifies
page-context evaluation, including passing a resolved ref element as the function argument.
On macOS the harness process must set `app.setActivationPolicy("accessory")` and
hide the Dock icon before creating any window. `showInactive()` only prevents window
focus; a normal Electron app launch can still activate the app and steal focus.
Harness windows are then created hidden, positioned in a screen corner, skipped from
the taskbar where Electron supports it, and revealed with `showInactive()` from
`ready-to-show`. Do not replace this with `show()`, `focus()`, or `app.focus()`:
the compositor only needs visible inactive windows, and harness runs must not steal
focus from the person using the machine.
The harness writes PNG evidence and `results.json` to:
```text
packages/desktop/capture-harness/out/
```
A passing run prints `PASS` lines for the production P1 attach-off parking state,
including fresh, settled, 75-second soak, multi-tab, viewport, and full-page checks. The
PNG sizes may be device-pixel scaled; on a Retina display the 1280x800 logical viewport
is usually saved as 2560x1600.
## Mechanism
Electron captures copy from the guest web contents' compositor surface. A resident
webview parked with `display:none`, offscreen coordinates, or `opacity:0` can lose its
copyable surface. The production parking state keeps the host fixed at `left:0`, `top:0`,
`width:1px`, `height:1px`, `overflow:hidden`, `opacity:1`, and `pointer-events:none`.
The webviews inside stay full-size at 1280x800, `display:inline-flex`, and absolutely
overlap at `left:0`, `top:0`.
There is no renderer prep/restore handshake. Main disables guest background throttling
once when the webview attaches, then screenshot capture uses the shared serialized queue,
invalidates before each attempt, and retries known first-frame failures within the
5-second capture budget. Viewport screenshots use `capturePage({ stayHidden:false })`;
full-page screenshots use the existing CDP path with layout metrics and screenshot clip.
+3
View File
@@ -195,6 +195,9 @@ Single file, validated with `PersistedConfigSchema`.
All fields are optional with sensible defaults.
Config parsing strips unrecognized object keys so a config written by a newer daemon does not brick
older read paths such as `paseo daemon status`. Malformed known fields still fail validation.
`agents.metadataGeneration.providers` controls the preferred structured-generation fallback order for daemon-side metadata tasks such as commit messages, PR text, branch names, and generated agent titles. Entries are tried first in the configured order, then Paseo falls through to dynamically discovered defaults and finally the current selection when available.
Local speech model ids are intentionally narrow: STT uses `parakeet-tdt-0.6b-v2-int8`, TTS uses `kokoro-en-v0_19`, and turn detection uses the bundled Silero VAD model.
+10 -9
View File
@@ -10,7 +10,8 @@ The image source lives in [`docker/`](../docker/).
The official image:
- builds `@getpaseo/server` and `@getpaseo/cli` from source-built workspace tarballs
- installs `@getpaseo/server` and `@getpaseo/cli` from npm for stable images,
or from source-built workspace tarballs for beta images
- runs the daemon as the non-root `paseo` user
- listens on `0.0.0.0:6767` inside the container
- enables the bundled daemon web UI with `PASEO_WEB_UI_ENABLED=true`
@@ -189,17 +190,16 @@ See [SECURITY.md](../SECURITY.md) for the daemon trust model.
## Building Locally
```bash
docker build -f docker/base/Dockerfile -t paseo:local .
docker build -t paseo:local docker/base
```
To assert the source tree version while building:
To bake a specific published npm version:
```bash
docker build \
--build-arg PASEO_VERSION=0.1.102 \
-t paseo:0.1.102 \
-f docker/base/Dockerfile \
.
docker/base
```
The Docker workflow builds the image on pull requests and on `main` as a
@@ -216,12 +216,13 @@ pushing a `v*` release tag:
gh workflow run docker.yml \
--ref main \
-f paseo_version=0.1.102-beta.1 \
-f publish=true
-f publish=true \
-f source_build=auto
```
Manual Docker publishes require an explicit `paseo_version`. The workflow builds
from the checked-out source tree and publishes only the exact prerelease image
tag for prerelease versions.
Manual Docker publishes require an explicit `paseo_version`. Prerelease
versions build from the checked-out source tree by default and publish only the
exact prerelease image tag.
The published image is multi-arch for `linux/amd64` and `linux/arm64`.
+1 -1
View File
@@ -73,7 +73,7 @@ Anyone who builds software:
- Built-in providers: Claude Code (Agent SDK), Codex (app-server), GitHub Copilot (ACP), OpenCode, Pi, OMP
- One-click ACP provider catalog: CodeWhale, Cursor, Hermes, Qwen Coder, Kimi Code, and others — plus custom ACP providers
- Voice mode: dictate prompts or talk through problems hands-free
- MCP server exposes the daemon to other agents (create_agent, send_agent_prompt, schedules, terminals, worktrees, workspace renaming)
- MCP server exposes the daemon to other agents (create_agent, send_agent_prompt, schedules, terminals, worktrees)
- Scheduled agents (cron-style triggers) via app, CLI, and MCP
- Frequent releases (multiple per week)
- Community contributions across packaging, providers, and bug fixes
-2
View File
@@ -18,8 +18,6 @@ Implement the `AgentClient` and `AgentSession` interfaces from `agent-sdk-types.
Existing direct providers: `claude` (in `providers/claude/agent.ts`), `codex` (`codex-app-server-agent.ts`), `opencode` (`opencode-agent.ts`), `pi` (`providers/pi/agent.ts`), and `omp` (a Pi-compatible built-in backed by the Pi adapter). The dev-only `mock` provider (`mock-load-test-agent.ts`) is also direct.
Claude first-party model metadata lives in `packages/server/src/server/agent/providers/claude/model-manifest.ts`. When adding or updating a Claude model, update that manifest only; the model picker thinking options and Claude-specific feature gates are derived from the manifest. Do not add model-specific Claude capability lists in feature code.
Paseo tools are not implemented as MCP tools internally. They live in a shared tool catalog under `packages/server/src/server/agent/tools/`; MCP is only the fallback adapter. A provider that can register runtime tools directly should set `supportsNativePaseoTools: true` and consume `launchContext.paseoTools` in `createSession`/`resumeSession`. When native tools are present, `AgentManager` strips the internal Paseo MCP server from the provider launch config so the provider does not receive the same tools twice. Providers that only know MCP should keep `supportsMcpServers: true` and let the daemon inject `/mcp/agents`.
Pi is a process-backed provider. Paseo requires the user to have the `pi` binary installed and talks to it through `pi --mode rpc`; the server package does not embed Pi's SDK/runtime packages.
+3 -2
View File
@@ -50,7 +50,7 @@ npm run release:patch
This bumps the version across all workspaces, runs checks, publishes to npm, and pushes the branch + tag. The tag push triggers `Desktop Release`, `Android APK Release`, `Docker`, and `Release Notes Sync` on GitHub Actions. EAS picks up the same tag via the EAS GitHub app and starts the iOS + Android store builds in parallel (see "Mobile builds (EAS)" below) — there is no `release-mobile.yml` in this repo.
The Docker workflow builds images from the checked-out source tree on pull requests and on `main` as non-publishing checks. Stable `vX.Y.Z` tag pushes publish `ghcr.io/getpaseo/paseo:X.Y.Z` and `ghcr.io/getpaseo/paseo:latest`; beta `vX.Y.Z-beta.N` tag pushes publish only `ghcr.io/getpaseo/paseo:X.Y.Z-beta.N` and never move `latest`.
The Docker workflow builds images on pull requests and on `main` as non-publishing checks. Stable `vX.Y.Z` tag pushes publish `ghcr.io/getpaseo/paseo:X.Y.Z` and `ghcr.io/getpaseo/paseo:latest`; beta `vX.Y.Z-beta.N` tag pushes publish only `ghcr.io/getpaseo/paseo:X.Y.Z-beta.N` and never move `latest`. Beta Docker images build from the checked-out source tree so the beta flow can intentionally skip npm publishing.
**Releases are always patch.** "Release paseo", "release stable", "ship stable", and similar always mean a patch bump from the previous stable. Never bump minor or major to trigger a build, ever — minor and major bumps are reserved for genuinely larger product cuts and require an explicit user instruction with the word "minor" or "major". If you find yourself reaching for `release:minor` to retrigger a failed build, you are doing the wrong thing — push a retry tag instead (see "Fixing a failed release build" below).
@@ -281,7 +281,8 @@ and EAS mobile release builds. Use the Docker workflow dispatch instead:
gh workflow run docker.yml \
--ref main \
-f paseo_version=X.Y.Z-beta.N \
-f publish=true
-f publish=true \
-f source_build=auto
```
This replaces `ghcr.io/getpaseo/paseo:X.Y.Z-beta.N` in place without touching
+1 -1
View File
@@ -1 +1 @@
sha256-2QXdrDE327fIF5THUve9NRk6Jx1gSUIWaNc7aTOrX+s=
sha256-o+VzG7lK0qpyUXF4F5Hk08ooW5CPoZSsOG7DyIReUKQ=
+21 -21
View File
@@ -1,12 +1,12 @@
{
"name": "paseo",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "paseo",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"hasInstallScript": true,
"license": "AGPL-3.0-or-later",
"workspaces": [
@@ -35132,7 +35132,7 @@
},
"packages/app": {
"name": "@getpaseo/app",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"dependencies": {
"@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0",
@@ -36150,12 +36150,12 @@
},
"packages/cli": {
"name": "@getpaseo/cli",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"dependencies": {
"@clack/prompts": "^1.0.0",
"@getpaseo/client": "0.1.104-beta.3",
"@getpaseo/protocol": "0.1.104-beta.3",
"@getpaseo/server": "0.1.104-beta.3",
"@getpaseo/client": "0.1.103",
"@getpaseo/protocol": "0.1.103",
"@getpaseo/server": "0.1.103",
"chalk": "^5.3.0",
"commander": "^12.0.0",
"mime-types": "^2.1.35",
@@ -36401,10 +36401,10 @@
},
"packages/client": {
"name": "@getpaseo/client",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"dependencies": {
"@getpaseo/protocol": "0.1.104-beta.3",
"@getpaseo/relay": "0.1.104-beta.3",
"@getpaseo/protocol": "0.1.103",
"@getpaseo/relay": "0.1.103",
"zod": "^4.4.3"
},
"devDependencies": {
@@ -36415,7 +36415,7 @@
},
"packages/desktop": {
"name": "@getpaseo/desktop",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@getpaseo/cli": "*",
@@ -36658,7 +36658,7 @@
},
"packages/expo-two-way-audio": {
"name": "@getpaseo/expo-two-way-audio",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"license": "MIT",
"devDependencies": {
"@types/jest": "^29.5.14",
@@ -37554,7 +37554,7 @@
},
"packages/highlight": {
"name": "@getpaseo/highlight",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"dependencies": {
"@codemirror/language": "^6.12.3",
"@codemirror/legacy-modes": "^6.5.3",
@@ -37786,7 +37786,7 @@
},
"packages/protocol": {
"name": "@getpaseo/protocol",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"dependencies": {
"zod": "^4.4.3"
},
@@ -37798,7 +37798,7 @@
},
"packages/relay": {
"name": "@getpaseo/relay",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"dependencies": {
"base64-js": "^1.5.1",
"tweetnacl": "^1.0.3",
@@ -38016,15 +38016,15 @@
},
"packages/server": {
"name": "@getpaseo/server",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"dependencies": {
"@agentclientprotocol/sdk": "^0.17.1",
"@anthropic-ai/claude-agent-sdk": "^0.3.195",
"@anthropic-ai/sdk": "^0.104.2",
"@getpaseo/client": "0.1.104-beta.3",
"@getpaseo/highlight": "0.1.104-beta.3",
"@getpaseo/protocol": "0.1.104-beta.3",
"@getpaseo/relay": "0.1.104-beta.3",
"@getpaseo/client": "0.1.103",
"@getpaseo/highlight": "0.1.103",
"@getpaseo/protocol": "0.1.103",
"@getpaseo/relay": "0.1.103",
"@isaacs/ttlcache": "^2.1.4",
"@modelcontextprotocol/sdk": "^1.20.1",
"@opencode-ai/sdk": "1.14.46",
@@ -38561,7 +38561,7 @@
},
"packages/website": {
"name": "@getpaseo/website",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"dependencies": {
"@cloudflare/vite-plugin": "^1.29.1",
"@cloudflare/workers-types": "^4.20260317.1",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "paseo",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"private": true,
"description": "Paseo: voice-controlled development environment for local AI coding agents",
"keywords": [
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/app",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"private": true,
"main": "index.ts",
"scripts": {
@@ -9,7 +9,7 @@ import {
} from "lucide-react-native";
import { withUnistyles } from "react-native-unistyles";
import type { AgentAttachment } from "@getpaseo/protocol/messages";
import type { WorkspaceComposerAttachment } from "@/attachments/types";
import type { BrowserAnnotationIntent, WorkspaceComposerAttachment } from "@/attachments/types";
import { getFileTypeLabel } from "@/attachments/file-types";
import { isPullRequestContextAttachment } from "@/attachments/workspace-attachment-utils";
import { ICON_SIZE, type Theme } from "@/styles/theme";
@@ -36,6 +36,13 @@ function getPullRequestContextSubtitle(attachment: WorkspaceComposerAttachment):
return "Review";
}
const BROWSER_INTENT_LABEL_KEYS: Record<BrowserAnnotationIntent, string> = {
fix: "workspace.browser.annotate.intents.fix",
change: "workspace.browser.annotate.intents.change",
question: "workspace.browser.annotate.intents.question",
approve: "workspace.browser.annotate.intents.approve",
};
function getTextAttachmentSubtitle(
attachment: Extract<AgentAttachment, { type: "text" }>,
t: TFunction,
@@ -89,10 +96,11 @@ export function getWorkspaceAttachmentPillContent(
t: TFunction,
): AttachmentPillContent {
if (attachment.kind === "browser_element") {
const intent = attachment.attachment.intent;
return {
icon: attachmentBrowserIcon,
title: attachment.attachment.tag,
subtitle: t("composer.attachments.element"),
subtitle: intent ? t(BROWSER_INTENT_LABEL_KEYS[intent]) : t("composer.attachments.element"),
};
}
if (isPullRequestContextAttachment(attachment)) {
+8
View File
@@ -21,6 +21,12 @@ export interface AttachmentMetadata {
createdAt: number;
}
/**
* The kind of review feedback the user is attaching to a selected browser
* element, sent to the agent alongside the element context.
*/
export type BrowserAnnotationIntent = "fix" | "change" | "question" | "approve";
export interface BrowserElementAttachment {
url: string;
selector: string;
@@ -44,6 +50,8 @@ export interface BrowserElementAttachment {
children: string[];
/** Free-text review note the user wrote about this element, if any. */
comment?: string;
/** What the user wants the agent to do with this element, if annotated. */
intent?: BrowserAnnotationIntent;
/**
* Cropped screenshot of the selected element, sent to the agent as an image
* alongside the textual element context. Persisted via the attachment store;
@@ -1,6 +1,5 @@
import { beforeEach, describe, expect, test } from "vitest";
import { beforeEach, describe, expect, test, vi } from "vitest";
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
import { createJSONStorage, type StateStorage } from "zustand/middleware";
import { mountBrowserAutomationHandler } from "./handler";
import type { DesktopHostBridge } from "@/desktop/host";
import { useBrowserStore } from "@/stores/browser-store";
@@ -9,6 +8,20 @@ import {
useWorkspaceLayoutStore,
} from "@/stores/workspace-layout-store";
vi.mock("expo-router", () => ({
router: {
navigate: vi.fn(),
},
}));
vi.mock("@react-native-async-storage/async-storage", () => ({
default: {
getItem: vi.fn(async () => null),
setItem: vi.fn(async () => undefined),
removeItem: vi.fn(async () => undefined),
},
}));
type BrowserAutomationExecuteRequest = Extract<
SessionOutboundMessage,
{ type: "browser.automation.execute.request" }
@@ -17,28 +30,9 @@ type BrowserAutomationExecuteResponse = Extract<
SessionInboundMessage,
{ type: "browser.automation.execute.response" }
>;
type BrowserAutomationResponsePayload = BrowserAutomationExecuteResponse["payload"];
class FakeStateStorage implements StateStorage {
private readonly values = new Map<string, string>();
public getItem = (key: string): string | null => this.values.get(key) ?? null;
public setItem = (key: string, value: string): void => {
this.values.set(key, value);
};
public removeItem = (key: string): void => {
this.values.delete(key);
};
public clear(): void {
this.values.clear();
}
}
class FakeDaemonClient {
public readonly sentResponses: BrowserAutomationExecuteResponse[] = [];
public sentResponses: BrowserAutomationExecuteResponse[] = [];
private handler: ((request: BrowserAutomationExecuteRequest) => void) | null = null;
public on(
@@ -61,107 +55,6 @@ class FakeDaemonClient {
public receive(nextRequest: BrowserAutomationExecuteRequest): void {
this.handler?.(nextRequest);
}
public payloadAt(index: number): BrowserAutomationResponsePayload {
const response = this.sentResponses[index];
if (!response) {
throw new Error(`Missing browser automation response at index ${index}`);
}
return response.payload;
}
}
class FakeBrowserBridge {
public readonly executedRequests: BrowserAutomationExecuteRequest[] = [];
public readonly registeredWorkspaceBrowsers: Array<{ browserId: string; workspaceId: string }> =
[];
public readonly unregisteredWorkspaceBrowsers: string[] = [];
public readonly clearedPartitions: string[] = [];
public readonly activeWorkspaceBrowsers: Array<{
browserId: string | null;
workspaceId: string;
}> = [];
public response: BrowserAutomationResponsePayload | null = null;
public thrownError: unknown = null;
public executeAutomationCommand = async (
request: BrowserAutomationExecuteRequest,
): Promise<BrowserAutomationResponsePayload> => {
this.executedRequests.push(request);
if (this.thrownError) {
throw this.thrownError;
}
return this.response ?? currentListTabsPayload(request.requestId);
};
public registerWorkspaceBrowser = async (input: {
browserId: string;
workspaceId: string;
}): Promise<void> => {
this.registeredWorkspaceBrowsers.push(input);
};
public unregisterWorkspaceBrowser = async (browserId: string): Promise<void> => {
this.unregisteredWorkspaceBrowsers.push(browserId);
};
public clearPartition = async (browserId: string): Promise<void> => {
this.clearedPartitions.push(browserId);
};
public setWorkspaceActiveBrowser = async (input: {
browserId: string | null;
workspaceId: string;
}): Promise<void> => {
this.activeWorkspaceBrowsers.push(input);
};
}
class FakeResidentBrowser {
public readonly ensuredWebviews: Array<{ browserId: string; url: string }> = [];
public ensure = (input: { browserId: string; url: string }): HTMLElement | null => {
this.ensuredWebviews.push(input);
return null;
};
}
class BrowserAutomationHandlerHarness {
public readonly client = new FakeDaemonClient();
public readonly browser = new FakeBrowserBridge();
public readonly resident = new FakeResidentBrowser();
private unsubscribe: (() => void) | null = null;
public mount(
input: {
serverId?: string;
host?: DesktopHostBridge | null;
registrationWaitTimeoutMs?: number;
registrationPollIntervalMs?: number;
} = {},
): void {
this.unsubscribe = mountBrowserAutomationHandler({
client: this.client,
...(input.serverId ? { serverId: input.serverId } : {}),
getHost: () => (input.host === undefined ? { browser: this.browser } : input.host),
ensureResidentBrowserWebview: this.resident.ensure,
...(input.registrationWaitTimeoutMs !== undefined
? { registrationWaitTimeoutMs: input.registrationWaitTimeoutMs }
: {}),
...(input.registrationPollIntervalMs !== undefined
? { registrationPollIntervalMs: input.registrationPollIntervalMs }
: {}),
});
}
public unmount(): void {
this.unsubscribe?.();
this.unsubscribe = null;
}
public receive(request: BrowserAutomationExecuteRequest): void {
this.client.receive(request);
}
}
function browserAutomationRequest(): BrowserAutomationExecuteRequest {
@@ -177,69 +70,45 @@ function browserNewTabRequest(): BrowserAutomationExecuteRequest {
type: "browser.automation.execute.request",
requestId: "req-new",
agentId: "agent-1",
workspaceId: "wks_workspace_a",
command: {
command: "new_tab",
args: { url: "https://example.com" },
},
workspaceId: "/repo",
command: { command: "new_tab", args: { workspaceId: "/repo", url: "https://example.com" } },
};
}
function browserResizeRequest(
browserId: string,
input: { workspaceId?: string } = {},
): BrowserAutomationExecuteRequest {
return {
type: "browser.automation.execute.request",
requestId: "req-resize",
agentId: "agent-1",
workspaceId: input.workspaceId ?? "wks_workspace_a",
command: {
command: "resize",
args: { browserId, width: 1024, height: 768 },
},
};
}
function browserCloseTabRequest(browserId: string): BrowserAutomationExecuteRequest {
return {
type: "browser.automation.execute.request",
requestId: "req-close-tab",
agentId: "agent-1",
workspaceId: "wks_workspace_a",
command: {
command: "close_tab",
args: { browserId },
},
};
}
function emptyListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload {
function emptyListTabsPayload(requestId = "req-new:list_tabs") {
return {
requestId,
ok: true,
ok: true as const,
result: {
command: "list_tabs",
command: "list_tabs" as const,
tabs: [],
},
};
}
function currentListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload {
function currentListTabsPayload(requestId = "req-new:list_tabs") {
return {
requestId,
ok: true,
ok: true as const,
result: {
command: "list_tabs",
command: "list_tabs" as const,
tabs: currentBrowserTabs(),
},
};
}
function flushPromises(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 0));
}
function waitForAsyncWork(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 20));
}
function currentBrowserTabs() {
return Object.values(useBrowserStore.getState().browsersById).map((browser) => ({
browserId: browser.browserId,
workspaceId: "wks_workspace_a",
workspaceId: "/repo",
url: browser.url,
title: browser.title,
isActive: true,
@@ -247,268 +116,151 @@ function currentBrowserTabs() {
}));
}
function newTabResultFrom(payload: BrowserAutomationResponsePayload) {
expect(payload).toMatchObject({
requestId: "req-new",
ok: true,
result: { command: "new_tab", workspaceId: "wks_workspace_a", url: "https://example.com" },
});
if (!payload.ok || payload.result.command !== "new_tab") {
throw new Error("Expected browser_new_tab success payload");
}
return payload.result;
}
function workspaceBrowserTabs(workspaceKey: string, browserId: string) {
return useWorkspaceLayoutStore
.getState()
.getWorkspaceTabs(workspaceKey)
.filter((tab) => tab.target.kind === "browser" && tab.target.browserId === browserId);
}
function flushAsyncWork(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 0));
}
function waitForRegistrationTimeout(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 20));
}
const browserAutomationStorage = new FakeStateStorage();
useBrowserStore.persist.setOptions({
storage: createJSONStorage(() => browserAutomationStorage),
});
useWorkspaceLayoutStore.persist.setOptions({
storage: createJSONStorage(() => browserAutomationStorage),
});
describe("mountBrowserAutomationHandler", () => {
beforeEach(() => {
browserAutomationStorage.clear();
useBrowserStore.setState({ browsersById: {} });
useWorkspaceLayoutStore.setState({ layoutByWorkspace: {} });
});
test("browser_new_tab creates a workspace browser tab without stealing focus", async () => {
const browser = new BrowserAutomationHandlerHarness();
test("creates an unfocused workspace browser tab for browser_new_tab", async () => {
const client = new FakeDaemonClient();
const setWorkspaceActiveBrowser = vi.fn(async () => undefined);
const setAgentActiveBrowser = vi.fn(async () => undefined);
const registerWorkspaceBrowser = vi.fn(async () => undefined);
const ensureResidentBrowserWebview = vi.fn();
const executeAutomationCommand = vi.fn(async () => currentListTabsPayload());
const workspaceKey = buildWorkspaceTabPersistenceKey({
serverId: "server-1",
workspaceId: "wks_workspace_a",
workspaceId: "/repo",
});
if (!workspaceKey) {
throw new Error("Expected workspace key");
}
const previousFocusedTabId = useWorkspaceLayoutStore
if (!workspaceKey) throw new Error("expected workspace key");
const focusedTabId = useWorkspaceLayoutStore
.getState()
.openTabFocused(workspaceKey, { kind: "draft", draftId: "human-draft" });
browser.mount({ serverId: "server-1" });
if (!focusedTabId) throw new Error("expected focused tab");
mountBrowserAutomationHandler({
client,
serverId: "server-1",
getHost: () =>
({
browser: {
executeAutomationCommand,
registerWorkspaceBrowser,
setWorkspaceActiveBrowser,
setAgentActiveBrowser,
},
}) satisfies DesktopHostBridge,
ensureResidentBrowserWebview,
});
browser.receive(browserNewTabRequest());
await flushAsyncWork();
client.receive(browserNewTabRequest());
await flushPromises();
const result = newTabResultFrom(browser.client.payloadAt(0));
const openedTabs = workspaceBrowserTabs(workspaceKey, result.browserId);
const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
expect(openedTabs).toEqual([
expect.objectContaining({
target: { kind: "browser", browserId: result.browserId },
}),
]);
expect(layout?.root).toEqual(
expect.objectContaining({
kind: "pane",
pane: expect.objectContaining({ focusedTabId: previousFocusedTabId }),
}),
const payload = client.sentResponses[0]?.payload;
expect(payload?.ok).toBe(true);
if (!payload?.ok) throw new Error("expected success");
expect(payload.result.command).toBe("new_tab");
if (payload.result.command !== "new_tab") throw new Error("expected new_tab result");
expect(payload.result.url).toBe("https://example.com");
expect(useWorkspaceLayoutStore.getState().getWorkspaceTabs(workspaceKey)).toContainEqual(
expect.objectContaining({ target: { kind: "browser", browserId: payload.result.browserId } }),
);
expect(openedTabs[0]?.tabId).not.toBe(previousFocusedTabId);
expect(browser.browser.registeredWorkspaceBrowsers).toEqual([
{ browserId: result.browserId, workspaceId: "wks_workspace_a" },
]);
expect(browser.browser.activeWorkspaceBrowsers).toEqual([]);
expect(browser.resident.ensuredWebviews).toEqual([
{ browserId: result.browserId, url: "https://example.com" },
]);
expect(browser.browser.executedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-new:list_tabs",
agentId: "agent-1",
workspaceId: "wks_workspace_a",
command: { command: "list_tabs", args: {} },
},
]);
const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
expect(layout?.root.kind).toBe("pane");
if (layout?.root.kind !== "pane") throw new Error("expected root pane");
expect(layout.root.pane.focusedTabId).toBe(focusedTabId);
expect(registerWorkspaceBrowser).toHaveBeenCalledWith({
browserId: payload.result.browserId,
workspaceId: "/repo",
});
expect(setAgentActiveBrowser).toHaveBeenCalledWith({
agentId: "agent-1",
browserId: payload.result.browserId,
});
expect(setWorkspaceActiveBrowser).not.toHaveBeenCalled();
expect(ensureResidentBrowserWebview).toHaveBeenCalledWith({
browserId: payload.result.browserId,
url: "https://example.com",
});
expect(executeAutomationCommand).toHaveBeenCalledTimes(1);
});
test("browser_new_tab returns a retryable timeout when the resident webview does not register", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.browser.response = emptyListTabsPayload();
browser.mount({
test("returns browser_timeout when the resident webview does not register", async () => {
const client = new FakeDaemonClient();
const ensureResidentBrowserWebview = vi.fn();
const executeAutomationCommand = vi.fn(async () => emptyListTabsPayload());
mountBrowserAutomationHandler({
client,
serverId: "server-1",
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
ensureResidentBrowserWebview,
registrationWaitTimeoutMs: 1,
registrationPollIntervalMs: 1,
});
browser.receive(browserNewTabRequest());
await waitForRegistrationTimeout();
client.receive(browserNewTabRequest());
await waitForAsyncWork();
expect(browser.client.sentResponses).toEqual([
{
type: "browser.automation.execute.response",
payload: {
requestId: "req-new",
ok: false,
error: {
code: "browser_timeout",
message: expect.stringContaining("Timed out waiting for browser tab"),
retryable: true,
},
},
expect(client.sentResponses[0]?.payload).toMatchObject({
requestId: "req-new",
ok: false,
error: {
code: "browser_timeout",
retryable: true,
},
]);
expect(browser.resident.ensuredWebviews).toEqual([
});
expect(ensureResidentBrowserWebview).toHaveBeenCalledWith(
expect.objectContaining({ url: "https://example.com" }),
]);
});
test("browser_new_tab wraps registration bridge errors in a response", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.browser.thrownError = new Error("IPC registration check failed");
browser.mount({ serverId: "server-1" });
browser.receive(browserNewTabRequest());
await flushAsyncWork();
expect(browser.client.sentResponses).toEqual([
{
type: "browser.automation.execute.response",
payload: {
requestId: "req-new",
ok: false,
error: {
code: "browser_unknown_error",
message: "IPC registration check failed",
retryable: false,
},
},
},
]);
});
test("browser_resize updates resident webview dimensions", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.mount({ serverId: "server-1" });
browser.receive(browserNewTabRequest());
await flushAsyncWork();
const result = newTabResultFrom(browser.client.payloadAt(0));
browser.receive(browserResizeRequest(result.browserId));
await flushAsyncWork();
expect(browser.client.payloadAt(1)).toEqual({
requestId: "req-resize",
);
expect(client.sentResponses[0]?.payload).not.toMatchObject({
ok: true,
result: {
command: "resize",
browserId: result.browserId,
width: 1024,
height: 768,
},
});
expect(browser.browser.executedRequests).toHaveLength(1);
});
test("browser_resize returns not found for a tab outside the request workspace", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.mount({ serverId: "server-1" });
browser.receive(browserNewTabRequest());
await flushAsyncWork();
const result = newTabResultFrom(browser.client.payloadAt(0));
browser.receive(browserResizeRequest(result.browserId, { workspaceId: "wks_workspace_b" }));
await flushAsyncWork();
expect(browser.client.payloadAt(1)).toEqual({
requestId: "req-resize",
ok: false,
error: {
code: "browser_tab_not_found",
message: `No browser tab found for ID: ${result.browserId}`,
retryable: false,
},
result: { command: "new_tab" },
});
});
test("browser_close_tab removes the workspace tab, browser record, resident webview, registry entry, and partition", async () => {
const browser = new BrowserAutomationHandlerHarness();
const workspaceKey = buildWorkspaceTabPersistenceKey({
test("wraps browser_new_tab registration bridge errors in a response", async () => {
const client = new FakeDaemonClient();
const executeAutomationCommand = vi.fn(async () => {
throw new Error("IPC registration check failed");
});
mountBrowserAutomationHandler({
client,
serverId: "server-1",
workspaceId: "wks_workspace_a",
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
});
if (!workspaceKey) {
throw new Error("Expected workspace key");
}
browser.mount({ serverId: "server-1" });
browser.receive(browserNewTabRequest());
await flushAsyncWork();
const result = newTabResultFrom(browser.client.payloadAt(0));
client.receive(browserNewTabRequest());
await flushPromises();
browser.receive(browserCloseTabRequest(result.browserId));
await flushAsyncWork();
expect(browser.client.payloadAt(1)).toEqual({
requestId: "req-close-tab",
ok: true,
result: { command: "close_tab", browserId: result.browserId },
});
expect(workspaceBrowserTabs(workspaceKey, result.browserId)).toEqual([]);
expect(useBrowserStore.getState().browsersById[result.browserId]).toBeUndefined();
expect(browser.browser.unregisteredWorkspaceBrowsers).toEqual([result.browserId]);
expect(browser.browser.clearedPartitions).toEqual([result.browserId]);
expect(currentBrowserTabs()).toEqual([]);
});
test("browser_close_tab returns not found after the tab is gone", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.mount({ serverId: "server-1" });
browser.receive(browserNewTabRequest());
await flushAsyncWork();
const result = newTabResultFrom(browser.client.payloadAt(0));
browser.receive(browserCloseTabRequest(result.browserId));
await flushAsyncWork();
browser.receive(browserCloseTabRequest(result.browserId));
await flushAsyncWork();
expect(browser.client.payloadAt(2)).toEqual({
requestId: "req-close-tab",
expect(client.sentResponses[0]?.payload).toEqual({
requestId: "req-new",
ok: false,
error: {
code: "browser_tab_not_found",
message: `No browser tab found for ID: ${result.browserId}`,
code: "browser_unknown_error",
message: "IPC registration check failed",
retryable: false,
},
});
});
test("non-new-tab requests send the desktop bridge response", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.browser.response = {
requestId: "desktop-req",
ok: true,
result: { command: "list_tabs", tabs: [] },
};
browser.mount();
const request = browserAutomationRequest();
test("sends a success response from the desktop bridge", async () => {
const client = new FakeDaemonClient();
const executeAutomationCommand = vi.fn(async () => ({
requestId: "req-1",
ok: true as const,
result: { command: "list_tabs" as const, tabs: [] },
}));
browser.receive(request);
await flushAsyncWork();
mountBrowserAutomationHandler({
client,
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
});
expect(browser.browser.executedRequests).toEqual([request]);
expect(browser.client.sentResponses).toEqual([
client.receive(browserAutomationRequest());
await flushPromises();
expect(executeAutomationCommand).toHaveBeenCalledWith(browserAutomationRequest());
expect(client.sentResponses).toEqual([
{
type: "browser.automation.execute.response",
payload: {
@@ -520,14 +272,14 @@ describe("mountBrowserAutomationHandler", () => {
]);
});
test("missing desktop bridge sends browser_unsupported", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.mount({ host: null });
test("missing bridge sends browser_unsupported", async () => {
const client = new FakeDaemonClient();
mountBrowserAutomationHandler({ client, getHost: () => null });
browser.receive(browserAutomationRequest());
await flushAsyncWork();
client.receive(browserAutomationRequest());
await flushPromises();
expect(browser.client.sentResponses).toEqual([
expect(client.sentResponses).toEqual([
{
type: "browser.automation.execute.response",
payload: {
@@ -535,7 +287,7 @@ describe("mountBrowserAutomationHandler", () => {
ok: false,
error: {
code: "browser_unsupported",
message: "Browser automation is not available in this app runtime.",
message: "Desktop browser automation is not available in this app runtime.",
retryable: false,
},
},
@@ -544,68 +296,80 @@ describe("mountBrowserAutomationHandler", () => {
});
test("typed bridge errors become failure responses", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.browser.thrownError = {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
};
browser.mount();
browser.receive(browserAutomationRequest());
await flushAsyncWork();
expect(browser.client.sentResponses).toEqual([
{
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
const client = new FakeDaemonClient();
mountBrowserAutomationHandler({
client,
getHost: () => ({
browser: {
executeAutomationCommand: async () => {
throw {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
};
},
},
}),
});
client.receive(browserAutomationRequest());
await flushPromises();
expect(client.sentResponses[0]?.payload).toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: "Browser tab browser-1 was not found.",
retryable: false,
},
]);
});
});
test("unimplemented preload IPC reports browser_unsupported", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.browser.thrownError = new Error(
'No handler registered for "paseo:browser:execute-automation-command"',
);
browser.mount();
browser.receive(browserAutomationRequest());
await flushAsyncWork();
expect(browser.client.sentResponses).toEqual([
{
type: "browser.automation.execute.response",
payload: {
requestId: "req-1",
ok: false,
error: {
code: "browser_unsupported",
message: "Browser automation is not implemented by this app build yet.",
retryable: false,
const client = new FakeDaemonClient();
mountBrowserAutomationHandler({
client,
getHost: () => ({
browser: {
executeAutomationCommand: async () => {
throw new Error('No handler registered for "paseo:browser:execute-automation-command"');
},
},
}),
});
client.receive(browserAutomationRequest());
await flushPromises();
expect(client.sentResponses[0]?.payload).toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unsupported",
message: "Desktop browser automation is not implemented by this desktop build yet.",
retryable: false,
},
]);
});
});
test("unsubscribe stops handling browser automation requests", async () => {
const browser = new BrowserAutomationHandlerHarness();
browser.mount();
test("unsubscribe stops handling requests", async () => {
const client = new FakeDaemonClient();
const executeAutomationCommand = vi.fn(async () => ({
requestId: "req-1",
ok: true as const,
result: { command: "list_tabs" as const, tabs: [] },
}));
const unsubscribe = mountBrowserAutomationHandler({
client,
getHost: () => ({ browser: { executeAutomationCommand } }),
});
browser.unmount();
browser.receive(browserAutomationRequest());
await flushAsyncWork();
unsubscribe();
client.receive(browserAutomationRequest());
await flushPromises();
expect(browser.browser.executedRequests).toEqual([]);
expect(browser.client.sentResponses).toEqual([]);
expect(executeAutomationCommand).not.toHaveBeenCalled();
expect(client.sentResponses).toEqual([]);
});
});
+37 -166
View File
@@ -1,14 +1,9 @@
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
import {
ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault,
removeResidentBrowserWebview,
resizeResidentBrowserWebview,
} from "@/components/browser-webview-resident";
import { createWorkspaceBrowser, getBrowserRecord, useBrowserStore } from "@/stores/browser-store";
import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
import { createWorkspaceBrowser } from "@/stores/browser-store";
import {
buildWorkspaceTabPersistenceKey,
collectAllTabs,
useWorkspaceLayoutStore,
} from "@/stores/workspace-layout-store";
@@ -45,7 +40,7 @@ export function mountBrowserAutomationHandler(
options: BrowserAutomationHandlerOptions,
): () => void {
const getHost = options.getHost ?? getDesktopHost;
const unsubscribe = options.client.on("browser.automation.execute.request", (request) => {
return options.client.on("browser.automation.execute.request", (request) => {
void handleBrowserAutomationRequest({
client: options.client,
getHost,
@@ -61,9 +56,6 @@ export function mountBrowserAutomationHandler(
: {}),
});
});
return () => {
unsubscribe();
};
}
export function mountBrowserAutomationDaemonClientHandler(
@@ -119,32 +111,7 @@ async function handleBrowserAutomationRequest(params: {
return;
}
if (request.command.command === "resize") {
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: resizeBrowserTabForRequest({ request, serverId }),
});
return;
}
if (request.command.command === "close_tab") {
try {
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: await closeBrowserTabForRequest({
request,
serverId,
browserHost,
}),
});
} catch (error) {
client.sendBrowserAutomationExecuteResponse({
type: "browser.automation.execute.response",
payload: normalizeThrownBridgeError(request.requestId, error),
});
}
return;
}
await rememberAgentBrowserTarget({ request, browserHost });
if (!executeAutomationCommand) {
client.sendBrowserAutomationExecuteResponse({
@@ -152,7 +119,7 @@ async function handleBrowserAutomationRequest(params: {
payload: browserAutomationFailure({
requestId: request.requestId,
code: "browser_unsupported",
message: "Browser automation is not available in this app runtime.",
message: "Desktop browser automation is not available in this app runtime.",
}),
});
return;
@@ -172,127 +139,6 @@ async function handleBrowserAutomationRequest(params: {
}
}
function resizeBrowserTabForRequest(params: {
request: BrowserAutomationExecuteRequest;
serverId?: string;
}): BrowserAutomationResponsePayload {
const { request, serverId } = params;
const command = request.command as Extract<
BrowserAutomationExecuteRequest["command"],
{ command: "resize" }
>;
const browserId = command.args.browserId;
if (!getBrowserRecord(browserId)) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_tab_not_found",
message: `No browser tab found for ID: ${browserId}`,
});
}
const workspaceId = request.workspaceId;
if (serverId && workspaceId && !findWorkspaceBrowserTab({ serverId, workspaceId, browserId })) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_tab_not_found",
message: `No browser tab found for ID: ${browserId}`,
});
}
const dimensions = resizeResidentBrowserWebview({
browserId,
width: command.args.width,
height: command.args.height,
});
if (!dimensions) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_tab_not_found",
message: `No browser tab found for ID: ${browserId}`,
});
}
return {
requestId: request.requestId,
ok: true,
result: {
command: "resize",
browserId,
width: dimensions.width,
height: dimensions.height,
},
};
}
async function closeBrowserTabForRequest(params: {
request: BrowserAutomationExecuteRequest;
serverId?: string;
browserHost: DesktopHostBridge["browser"] | undefined;
}): Promise<BrowserAutomationResponsePayload> {
const { request, serverId, browserHost } = params;
const command = request.command as Extract<
BrowserAutomationExecuteRequest["command"],
{ command: "close_tab" }
>;
const browserId = command.args.browserId;
const workspaceId = request.workspaceId;
const workspaceTab = serverId
? findWorkspaceBrowserTab({ serverId, workspaceId, browserId })
: null;
if (!workspaceTab && (!serverId || !workspaceId)) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_unsupported",
message: "Cannot close a browser tab without a workspace context.",
});
}
if (!workspaceTab || !getBrowserRecord(browserId)) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_tab_not_found",
message: `No browser tab found for ID: ${browserId}`,
});
}
useWorkspaceLayoutStore.getState().closeTab(workspaceTab.workspaceKey, workspaceTab.tabId);
useBrowserStore.getState().removeBrowser(browserId);
removeResidentBrowserWebview(browserId);
await browserHost?.unregisterWorkspaceBrowser?.(browserId);
await browserHost?.clearPartition?.(browserId);
return {
requestId: request.requestId,
ok: true,
result: { command: "close_tab", browserId },
};
}
function findWorkspaceBrowserTab(input: {
serverId: string;
workspaceId: string | undefined;
browserId: string;
}): { workspaceKey: string; tabId: string } | null {
if (!input.workspaceId) {
return null;
}
const workspaceKey = buildWorkspaceTabPersistenceKey({
serverId: input.serverId,
workspaceId: input.workspaceId,
});
if (!workspaceKey) {
return null;
}
const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
const tab = layout
? collectAllTabs(layout.root).find((candidate) => {
return (
candidate.target.kind === "browser" && candidate.target.browserId === input.browserId
);
})
: null;
return tab ? { workspaceKey, tabId: tab.tabId } : null;
}
async function openBrowserTabForRequest(params: {
request: BrowserAutomationExecuteRequest;
serverId?: string;
@@ -313,11 +159,11 @@ async function openBrowserTabForRequest(params: {
BrowserAutomationExecuteRequest["command"],
{ command: "new_tab" }
>;
const workspaceId = request.workspaceId;
const workspaceId = request.workspaceId ?? command.args.workspaceId;
if (!serverId || !workspaceId) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_unsupported",
code: "browser_no_tab",
message: "Cannot create a browser tab without a workspace context.",
});
}
@@ -328,7 +174,7 @@ async function openBrowserTabForRequest(params: {
if (!workspaceKey) {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_unsupported",
code: "browser_no_tab",
message: "Cannot create a browser tab without a workspace context.",
});
}
@@ -338,6 +184,9 @@ async function openBrowserTabForRequest(params: {
});
await browserHost?.registerWorkspaceBrowser?.({ browserId, workspaceId });
if (request.agentId) {
await browserHost?.setAgentActiveBrowser?.({ agentId: request.agentId, browserId });
}
if (browserHost?.executeAutomationCommand) {
ensureResidentBrowserWebview({ browserId, url: normalizedUrl });
@@ -355,7 +204,7 @@ async function openBrowserTabForRequest(params: {
return browserAutomationFailure({
requestId: request.requestId,
code: "browser_timeout",
message: `Timed out waiting for browser tab ${browserId} to register with the browser automation host. Try browser_new_tab again.`,
message: `Timed out waiting for browser tab ${browserId} to register with desktop automation. Try browser_new_tab again.`,
retryable: true,
});
}
@@ -386,7 +235,7 @@ async function waitForBrowserRegistration(params: {
agentId: params.request.agentId,
cwd: params.request.cwd,
workspaceId: params.workspaceId,
command: { command: "list_tabs", args: {} },
command: { command: "list_tabs", args: { workspaceId: params.workspaceId } },
});
if (payload.ok && payload.result.command === "list_tabs") {
if (payload.result.tabs.some((tab) => tab.browserId === params.browserId)) {
@@ -402,6 +251,28 @@ function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function rememberAgentBrowserTarget(input: {
request: BrowserAutomationExecuteRequest;
browserHost: DesktopHostBridge["browser"] | undefined;
}): Promise<void> {
if (!input.request.agentId) {
return;
}
const browserId = readRequestBrowserId(input.request);
if (!browserId) {
return;
}
await input.browserHost?.setAgentActiveBrowser?.({ agentId: input.request.agentId, browserId });
}
function readRequestBrowserId(request: BrowserAutomationExecuteRequest): string | null {
if (request.browserId) {
return request.browserId;
}
const args = request.command.args as { browserId?: unknown };
return typeof args.browserId === "string" && args.browserId.length > 0 ? args.browserId : null;
}
function normalizeBridgePayload(
requestId: string,
payload: BrowserAutomationResponsePayload,
@@ -423,14 +294,14 @@ function normalizeThrownBridgeError(
return browserAutomationFailure({
requestId,
code: "browser_unsupported",
message: "Browser automation is not implemented by this app build yet.",
message: "Desktop browser automation is not implemented by this desktop build yet.",
});
}
return browserAutomationFailure({
requestId,
code: "browser_unknown_error",
message: message || "Browser automation failed.",
message: message || "Desktop browser automation failed.",
});
}
@@ -12,22 +12,21 @@ import { Pressable, Text, TextInput, View, type StyleProp, type ViewStyle } from
import {
ArrowLeft,
ArrowRight,
Camera,
ChevronDown,
Copy,
Maximize,
Monitor,
MousePointer2,
PencilRuler,
RotateCw,
Smartphone,
Tablet,
Wrench,
X,
type LucideIcon,
} from "lucide-react-native";
import { StyleSheet, useUnistyles, withUnistyles } from "react-native-unistyles";
import { useTranslation } from "react-i18next";
import * as Clipboard from "expo-clipboard";
import { Button } from "@/components/ui/button";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import { useToast } from "@/contexts/toast-context";
import {
@@ -41,7 +40,11 @@ import {
useWorkspaceAttachments,
useWorkspaceAttachmentsStore,
} from "@/attachments/workspace-attachments-store";
import type { AttachmentMetadata, BrowserElementAttachment } from "@/attachments/types";
import type {
AttachmentMetadata,
BrowserAnnotationIntent,
BrowserElementAttachment,
} from "@/attachments/types";
import { persistAttachmentFromDataUrl } from "@/attachments/service";
import { WORKSPACE_SECONDARY_HEADER_HEIGHT } from "@/constants/layout";
import {
@@ -49,6 +52,7 @@ import {
isElectronRuntime,
type DesktopBrowserShortcutEvent,
} from "@/desktop/host";
import { isDev } from "@/constants/platform";
import { useBrowserStore, normalizeWorkspaceBrowserUrl } from "@/stores/browser-store";
import {
prepareBrowserWebview,
@@ -75,14 +79,25 @@ type WebTextInput = TextInput & {
getNativeRef?: () => unknown;
};
type BrowserElementSelection = Omit<BrowserElementAttachment, "formatted" | "comment"> & {
type BrowserElementSelection = Omit<
BrowserElementAttachment,
"formatted" | "comment" | "intent"
> & {
attributes?: Record<string, string>;
};
interface BrowserElementAnnotation {
comment: string;
intent: BrowserAnnotationIntent;
}
const BROWSER_ANNOTATION_INTENTS: readonly BrowserAnnotationIntent[] = [
"fix",
"change",
"question",
"approve",
];
type DeviceSizeId =
| "responsive"
| "iphone-se"
@@ -208,6 +223,10 @@ function formatElementAttachment(
const html = truncateText(selection.outerHTML.trim(), 800);
const parts: string[] = [];
if (annotation) {
parts.push(`intent: ${annotation.intent}`);
}
if (selection.reactSource?.fileName) {
const loc = [
selection.reactSource.fileName,
@@ -270,6 +289,7 @@ function buildBrowserElementAttachment(
parentChain: selection.parentChain,
children: selection.children,
...(comment ? { comment } : {}),
...(annotation ? { intent: annotation.intent } : {}),
...(screenshot ? { screenshot } : {}),
formatted: formatElementAttachment(selection, annotation),
};
@@ -621,11 +641,10 @@ export function BrowserPane({
const pendingNavigationUrlRef = useRef<string | null>(null);
const domReadyRef = useRef(false);
const annotationMarkersRef = useRef<BrowserAnnotationMarker[]>([]);
const [selectorMode, setSelectorMode] = useState<"annotate" | "screenshot" | null>(null);
const selectorActive = selectorMode !== null;
const [selectorActive, setSelectorActive] = useState(false);
// Which action the active selector performs on click: open the annotation card
// ("annotate") or copy a screenshot of the element to the clipboard ("screenshot").
const selectorModeRef = useRef<"annotate" | "screenshot">("annotate");
// ("annotate") or copy the element to the clipboard ("grab").
const selectorModeRef = useRef<"annotate" | "grab">("annotate");
const toast = useToast();
const toastRef = useRef(toast);
toastRef.current = toast;
@@ -1055,7 +1074,7 @@ export function BrowserPane({
[],
);
const screenshotElementToClipboard = useCallback(
const grabElementToClipboard = useCallback(
async (selection: BrowserElementSelection) => {
const text = formatElementAttachment(selection);
const copyElement = getDesktopHost()?.browser?.copyElement;
@@ -1068,23 +1087,19 @@ export function BrowserPane({
const dataUrl = await captureElement(browserIdRef.current, { x, y, width, height });
imageDataUrl = dataUrl ?? undefined;
} catch (error) {
console.warn("[browser-pane] capture element for screenshot failed", error);
console.warn("[browser-pane] capture element for grab failed", error);
}
}
const copiedMessage = imageDataUrl
? t("workspace.browser.controls.screenshotCopied")
: t("workspace.browser.controls.elementCopied");
// Copy via the main process; the renderer's navigator.clipboard rejects
// with NotAllowedError because focus is inside the guest <webview>.
if (typeof copyElement === "function") {
try {
const ok = await copyElement({ text, imageDataUrl });
if (ok) {
toastRef.current?.show(copiedMessage, { variant: "success" });
toastRef.current?.copied(t("workspace.browser.controls.grabElementLabel"));
} else {
toastRef.current?.error(t("workspace.browser.controls.screenshotFailed"));
toastRef.current?.error(t("workspace.browser.controls.grabFailed"));
}
return;
} catch (error) {
@@ -1095,12 +1110,10 @@ export function BrowserPane({
// Fallback to expo-clipboard (text only) when the bridge is unavailable.
try {
await Clipboard.setStringAsync(text);
toastRef.current?.show(t("workspace.browser.controls.elementCopied"), {
variant: "success",
});
toastRef.current?.copied(t("workspace.browser.controls.grabElementLabel"));
} catch (error) {
console.warn("[browser-pane] clipboard fallback failed", error);
toastRef.current?.error(t("workspace.browser.controls.screenshotFailed"));
toastRef.current?.error(t("workspace.browser.controls.grabFailed"));
}
},
[t],
@@ -1108,8 +1121,8 @@ export function BrowserPane({
const handleSelectorResult = useCallback(
(selection: BrowserElementSelection) => {
if (selectorModeRef.current === "screenshot") {
void screenshotElementToClipboard(selection);
if (selectorModeRef.current === "grab") {
void grabElementToClipboard(selection);
return;
}
pendingScreenshotRef.current = undefined;
@@ -1119,7 +1132,7 @@ export function BrowserPane({
return undefined;
});
},
[captureElementScreenshot, screenshotElementToClipboard],
[captureElementScreenshot, grabElementToClipboard],
);
const submitAnnotation = useCallback(
@@ -1142,15 +1155,15 @@ export function BrowserPane({
}, []);
const startElementSelector = useCallback(
(mode: "annotate" | "screenshot") => {
(mode: "annotate" | "grab") => {
const webview = webviewRef.current;
if (!webview || !domReadyRef.current) return;
// Annotate needs a workspace scope to attach to; screenshot only copies.
// Annotate needs a workspace scope to attach to; grab only copies.
if (mode === "annotate" && !workspaceAttachmentScopeKey) return;
selectorModeRef.current = mode;
pendingScreenshotRef.current = undefined;
setPendingSelection(null);
setSelectorMode(mode);
setSelectorActive(true);
const js = `
(function() {
@@ -1376,11 +1389,11 @@ export function BrowserPane({
const poll = startSelectorResultPolling({
webview,
onSelection: handleSelectorResult,
onDone: () => setSelectorMode(null),
onDone: () => setSelectorActive(false),
});
window.setTimeout(() => {
window.clearInterval(poll);
setSelectorMode(null);
setSelectorActive(false);
if (webviewRef.current !== webview || !domReadyRef.current) {
return;
}
@@ -1389,10 +1402,10 @@ export function BrowserPane({
return undefined;
})
.catch(() => {
setSelectorMode(null);
setSelectorActive(false);
});
} catch {
setSelectorMode(null);
setSelectorActive(false);
}
},
[handleSelectorResult, workspaceAttachmentScopeKey],
@@ -1400,7 +1413,7 @@ export function BrowserPane({
const cancelElementSelector = useCallback(() => {
const webview = webviewRef.current;
setSelectorMode(null);
setSelectorActive(false);
if (webview && domReadyRef.current) {
try {
clearWebviewSelector(webview);
@@ -1457,12 +1470,12 @@ export function BrowserPane({
startElementSelector("annotate");
}, [cancelElementSelector, selectorActive, startElementSelector]);
const handleToggleScreenshot = useCallback(() => {
const handleToggleGrab = useCallback(() => {
if (selectorActive) {
cancelElementSelector();
return;
}
startElementSelector("screenshot");
startElementSelector("grab");
}, [cancelElementSelector, selectorActive, startElementSelector]);
const handleOpenDevTools = useCallback(() => {
@@ -1508,21 +1521,13 @@ export function BrowserPane({
],
[browser?.canGoForward],
);
const annotateIconButtonStyle = useCallback(
const selectorIconButtonStyle = useCallback(
({ hovered, pressed }: { hovered?: boolean; pressed?: boolean }) => [
styles.iconButton,
selectorMode === "annotate" && styles.selectorActiveButton,
selectorActive && styles.selectorActiveButton,
(hovered || pressed) && styles.iconButtonHovered,
],
[selectorMode],
);
const screenshotIconButtonStyle = useCallback(
({ hovered, pressed }: { hovered?: boolean; pressed?: boolean }) => [
styles.iconButton,
selectorMode === "screenshot" && styles.selectorActiveButton,
(hovered || pressed) && styles.iconButtonHovered,
],
[selectorMode],
[selectorActive],
);
const devicePreset = useMemo(
@@ -1628,46 +1633,36 @@ export function BrowserPane({
onSelect={setDeviceSizeId}
triggerStyle={baseIconButtonStyle}
/>
<ToolbarButton
label={t("workspace.browser.controls.openDevTools")}
onPress={handleOpenDevTools}
style={baseIconButtonStyle}
>
<Wrench size={16} color={theme.colors.foregroundMuted} />
</ToolbarButton>
{isDev ? (
<ToolbarButton
label={t("workspace.browser.controls.openDevTools")}
onPress={handleOpenDevTools}
style={baseIconButtonStyle}
>
<PencilRuler size={16} color={theme.colors.foregroundMuted} />
</ToolbarButton>
) : null}
<ToolbarButton
label={
selectorMode === "annotate"
selectorActive
? t("workspace.browser.controls.cancelSelector")
: t("workspace.browser.controls.annotateElement")
: t("workspace.browser.controls.selectElement")
}
active={selectorMode === "annotate"}
active={selectorActive}
onPress={handleToggleElementSelector}
style={annotateIconButtonStyle}
style={selectorIconButtonStyle}
>
<MousePointer2
size={16}
color={
selectorMode === "annotate" ? theme.colors.accent : theme.colors.foregroundMuted
}
color={selectorActive ? theme.colors.accent : theme.colors.foregroundMuted}
/>
</ToolbarButton>
<ToolbarButton
label={
selectorMode === "screenshot"
? t("workspace.browser.controls.cancelSelector")
: t("workspace.browser.controls.screenshotElement")
}
active={selectorMode === "screenshot"}
onPress={handleToggleScreenshot}
style={screenshotIconButtonStyle}
label={t("workspace.browser.controls.grabElement")}
onPress={handleToggleGrab}
style={baseIconButtonStyle}
>
<Camera
size={16}
color={
selectorMode === "screenshot" ? theme.colors.accent : theme.colors.foregroundMuted
}
/>
<Copy size={16} color={theme.colors.foregroundMuted} />
</ToolbarButton>
</View>
</View>
@@ -1695,6 +1690,45 @@ export function BrowserPane({
);
}
const INTENT_LABEL_KEYS: Record<BrowserAnnotationIntent, string> = {
fix: "workspace.browser.annotate.intents.fix",
change: "workspace.browser.annotate.intents.change",
question: "workspace.browser.annotate.intents.question",
approve: "workspace.browser.annotate.intents.approve",
};
function IntentChip({
active,
intent,
label,
onSelect,
}: {
active: boolean;
intent: BrowserAnnotationIntent;
label: string;
onSelect: (intent: BrowserAnnotationIntent) => void;
}) {
const handlePress = useCallback(() => {
onSelect(intent);
}, [intent, onSelect]);
const chipStyle = useMemo(() => [styles.intentChip, active && styles.intentChipActive], [active]);
const textStyle = useMemo(
() => [styles.intentChipText, active && styles.intentChipTextActive],
[active],
);
const accessibilityState = useMemo(() => ({ selected: active }), [active]);
return (
<Pressable
accessibilityRole="button"
accessibilityState={accessibilityState}
onPress={handlePress}
style={chipStyle}
>
<Text style={textStyle}>{label}</Text>
</Pressable>
);
}
function BrowserElementAnnotationCard({
selection,
onSubmit,
@@ -1706,11 +1740,14 @@ function BrowserElementAnnotationCard({
}) {
const { t } = useTranslation();
const [comment, setComment] = useState("");
const [intent, setIntent] = useState<BrowserAnnotationIntent>("fix");
const commentRef = useRef(comment);
commentRef.current = comment;
const intentRef = useRef(intent);
intentRef.current = intent;
const handleSubmit = useCallback(() => {
onSubmit({ comment: commentRef.current });
onSubmit({ comment: commentRef.current, intent: intentRef.current });
}, [onSubmit]);
useEffect(() => {
@@ -1755,6 +1792,17 @@ function BrowserElementAnnotationCard({
<Text numberOfLines={1} style={styles.annotationElement}>
{elementLabel}
</Text>
<View style={styles.annotationIntents}>
{BROWSER_ANNOTATION_INTENTS.map((option) => (
<IntentChip
key={option}
active={option === intent}
intent={option}
label={t(INTENT_LABEL_KEYS[option])}
onSelect={setIntent}
/>
))}
</View>
<ThemedAnnotationInput
accessibilityLabel={t("workspace.browser.annotate.placeholder")}
autoFocus
@@ -1766,12 +1814,24 @@ function BrowserElementAnnotationCard({
value={comment}
/>
<View style={styles.annotationActions}>
<Button variant="ghost" size="sm" onPress={onCancel}>
{t("workspace.browser.annotate.cancel")}
</Button>
<Button variant="default" size="sm" onPress={handleSubmit}>
{t("workspace.browser.annotate.submit")}
</Button>
<Pressable
accessibilityRole="button"
onPress={onCancel}
style={styles.annotationSecondaryButton}
>
<Text style={styles.annotationSecondaryText}>
{t("workspace.browser.annotate.cancel")}
</Text>
</Pressable>
<Pressable
accessibilityRole="button"
onPress={handleSubmit}
style={styles.annotationPrimaryButton}
>
<Text style={styles.annotationPrimaryText}>
{t("workspace.browser.annotate.submit")}
</Text>
</Pressable>
</View>
</View>
</View>
@@ -1926,7 +1986,31 @@ const styles = StyleSheet.create((theme) => ({
annotationElement: {
fontSize: theme.fontSize.xs,
color: theme.colors.foregroundMuted,
marginBottom: theme.spacing[2],
},
annotationIntents: {
flexDirection: "row",
flexWrap: "wrap",
gap: theme.spacing[1],
},
intentChip: {
paddingHorizontal: theme.spacing[2],
paddingVertical: theme.spacing[1],
borderRadius: theme.borderRadius.md,
borderWidth: 1,
borderColor: theme.colors.border,
backgroundColor: theme.colors.surface1,
},
intentChipActive: {
borderColor: theme.colors.accent,
backgroundColor: `${String(theme.colors.accent)}20`,
},
intentChipText: {
fontSize: theme.fontSize.xs,
fontWeight: "500",
color: theme.colors.foregroundMuted,
},
intentChipTextActive: {
color: theme.colors.accent,
},
annotationInput: {
minHeight: 64,
@@ -1945,6 +2029,27 @@ const styles = StyleSheet.create((theme) => ({
justifyContent: "flex-end",
gap: theme.spacing[2],
},
annotationSecondaryButton: {
paddingHorizontal: theme.spacing[3],
paddingVertical: theme.spacing[2],
borderRadius: theme.borderRadius.md,
},
annotationSecondaryText: {
fontSize: theme.fontSize.sm,
fontWeight: "500",
color: theme.colors.foregroundMuted,
},
annotationPrimaryButton: {
paddingHorizontal: theme.spacing[3],
paddingVertical: theme.spacing[2],
borderRadius: theme.borderRadius.md,
backgroundColor: theme.colors.accent,
},
annotationPrimaryText: {
fontSize: theme.fontSize.sm,
fontWeight: "600",
color: theme.colors.accentForeground ?? "#ffffff",
},
unavailableState: {
flex: 1,
alignItems: "center",
@@ -2,66 +2,33 @@ import { afterEach, describe, expect, it } from "vitest";
import {
clearResidentBrowserWebviewsForTests,
ensureResidentBrowserWebview,
prepareBrowserWebview,
releaseResidentBrowserWebview,
removeResidentBrowserWebview,
takeResidentBrowserWebview,
} from "./browser-webview-resident";
const RESIDENT_HOST_ID = "paseo-browser-resident-webviews";
function residentHost(): HTMLElement {
const host = document.getElementById(RESIDENT_HOST_ID);
if (!host) {
throw new Error("Expected resident browser host");
}
return host;
}
function expectPermanentHostParking(host: HTMLElement): void {
expect(host.style.position).toBe("fixed");
expect(host.style.left).toBe("0px");
expect(host.style.top).toBe("0px");
expect(host.style.width).toBe("1px");
expect(host.style.height).toBe("1px");
expect(host.style.overflow).toBe("hidden");
expect(host.style.opacity).toBe("1");
expect(host.style.pointerEvents).toBe("none");
expect(host.style.display).toBe("block");
expect(host.style.visibility).toBe("visible");
expect(host.style.transform).toBe("");
}
function expectResidentWebviewParking(webview: HTMLElement): void {
expect(webview.style.display).toBe("inline-flex");
expect(webview.style.flex).toBe("0 0 auto");
expect(webview.style.width).toBe("1280px");
expect(webview.style.height).toBe("800px");
expect(webview.style.position).toBe("absolute");
expect(webview.style.left).toBe("0px");
expect(webview.style.top).toBe("0px");
expect(webview.style.zIndex).toBe("0");
}
describe("resident browser webviews", () => {
afterEach(() => {
clearResidentBrowserWebviewsForTests();
});
it("parks a browser webview in the permanent paintable 1x1 host", () => {
const visibleHost = document.createElement("div");
it("keeps a browser webview mounted offscreen and reuses the same node", () => {
const host = document.createElement("div");
const webview = document.createElement("webview");
visibleHost.appendChild(webview);
document.body.appendChild(visibleHost);
host.appendChild(webview);
document.body.appendChild(host);
releaseResidentBrowserWebview("browser-a", webview);
const host = residentHost();
expect(visibleHost.children).toHaveLength(0);
expect(Array.from(host.children)).toEqual([webview]);
expect(host.children).toHaveLength(0);
expect(webview.isConnected).toBe(true);
expectPermanentHostParking(host);
expectResidentWebviewParking(webview);
expect(webview.style.width).toBe("1280px");
expect(webview.style.height).toBe("800px");
const reused = takeResidentBrowserWebview("browser-a");
expect(reused).toBe(webview);
expect(takeResidentBrowserWebview("browser-a")).toBeNull();
});
it("creates a resident webview for an agent-created unfocused tab", () => {
@@ -77,113 +44,6 @@ describe("resident browser webviews", () => {
expect((webview as HTMLUnknownElement & { src?: string })?.src).toContain(
"https://example.com",
);
expectPermanentHostParking(residentHost());
expectResidentWebviewParking(webview as HTMLElement);
});
it("normalizes an existing resident host back to permanent parking", () => {
const staleHost = document.createElement("div");
staleHost.id = RESIDENT_HOST_ID;
staleHost.style.left = "-20000px";
staleHost.style.width = "1280px";
staleHost.style.height = "800px";
staleHost.style.opacity = "0";
staleHost.style.display = "none";
document.body.appendChild(staleHost);
const webview = ensureResidentBrowserWebview({
browserId: "browser-stale-host",
url: "https://example.com",
});
expect(webview).not.toBeNull();
expectPermanentHostParking(staleHost);
expectResidentWebviewParking(webview as HTMLElement);
});
it("normalizes an existing resident webview and its stale host before reusing them", () => {
const staleHost = document.createElement("div");
staleHost.id = RESIDENT_HOST_ID;
staleHost.style.left = "-20000px";
staleHost.style.width = "1280px";
staleHost.style.height = "800px";
staleHost.style.opacity = "0";
staleHost.style.display = "none";
const staleWebview = document.createElement("webview");
prepareBrowserWebview(staleWebview, {
browserId: "browser-stale-child",
initialUrl: "https://example.com",
});
staleWebview.style.display = "none";
staleWebview.style.width = "1px";
staleWebview.style.height = "1px";
staleWebview.style.position = "fixed";
staleWebview.style.left = "-20000px";
staleHost.appendChild(staleWebview);
document.body.appendChild(staleHost);
const webview = ensureResidentBrowserWebview({
browserId: "browser-stale-child",
url: "https://example.com/agent",
});
expect(webview).toBe(staleWebview);
expect(Array.from(staleHost.children)).toEqual([staleWebview]);
expectPermanentHostParking(staleHost);
expectResidentWebviewParking(staleWebview);
});
it("parks resident webviews as an overlapping stack", () => {
const firstWebview = ensureResidentBrowserWebview({
browserId: "browser-first",
url: "https://example.com/first",
});
const secondWebview = ensureResidentBrowserWebview({
browserId: "browser-second",
url: "https://example.com/second",
});
const host = residentHost();
expect(firstWebview?.parentElement).toBe(host);
expect(secondWebview?.parentElement).toBe(host);
expectResidentWebviewParking(firstWebview as HTMLElement);
expectResidentWebviewParking(secondWebview as HTMLElement);
});
it("moves a resident webview into a visible pane without recreating the node", () => {
const webview = ensureResidentBrowserWebview({
browserId: "browser-visible",
url: "https://example.com",
});
const visibleWebview = takeResidentBrowserWebview("browser-visible");
expect(visibleWebview).toBe(webview);
expect(webview?.style.position).toBe("");
expect(webview?.style.left).toBe("");
expect(webview?.style.top).toBe("");
expect(webview?.style.zIndex).toBe("");
expect(takeResidentBrowserWebview("browser-visible")).toBeNull();
});
it("returns an existing visible pane webview instead of creating a resident duplicate", () => {
const visibleHost = document.createElement("div");
const visibleWebview = document.createElement("webview");
prepareBrowserWebview(visibleWebview, {
browserId: "browser-visible-pane",
initialUrl: "https://example.com",
});
visibleHost.appendChild(visibleWebview);
document.body.appendChild(visibleHost);
const webview = ensureResidentBrowserWebview({
browserId: "browser-visible-pane",
url: "https://example.com/agent",
});
expect(webview).toBe(visibleWebview);
expect(document.getElementById(RESIDENT_HOST_ID)).toBeNull();
});
it("removes a resident webview when its browser tab closes", () => {
@@ -4,7 +4,6 @@ const RESIDENT_VIEWPORT_WIDTH = 1280;
const RESIDENT_VIEWPORT_HEIGHT = 800;
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
const residentWebviewSizesByBrowserId = new Map<string, { width: number; height: number }>();
interface BrowserWebviewElement extends HTMLElement {
src: string;
@@ -22,84 +21,42 @@ function readDocument(): Document | null {
return typeof document === "undefined" ? null : document;
}
function applyResidentHostParkingStyle(host: HTMLElement): void {
// Parked browser webviews must remain paintable at all times; screenshot
// correctness depends on the proven states in docs/browser-capture-harness.md.
host.setAttribute("aria-hidden", "true");
host.style.position = "fixed";
host.style.left = "0";
host.style.top = "0";
host.style.width = "1px";
host.style.height = "1px";
host.style.overflow = "hidden";
host.style.opacity = "1";
host.style.pointerEvents = "none";
host.style.display = "block";
host.style.zIndex = "";
host.style.clipPath = "";
host.style.visibility = "visible";
host.style.transform = "";
}
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
if (existing) {
applyResidentHostParkingStyle(existing);
return existing;
}
const host = ownerDocument.createElement("div");
host.id = RESIDENT_BROWSER_HOST_ID;
applyResidentHostParkingStyle(host);
host.setAttribute("aria-hidden", "true");
host.style.position = "fixed";
host.style.left = "-20000px";
host.style.top = "0";
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
host.style.overflow = "hidden";
host.style.opacity = "0";
host.style.pointerEvents = "none";
ownerDocument.body.appendChild(host);
return host;
}
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
if (!(element instanceof HTMLElement)) {
continue;
}
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
return element;
return element as HTMLElement;
}
}
return null;
}
function dimensionsForBrowser(browserId: string | null): { width: number; height: number } {
if (!browserId) {
return { width: RESIDENT_VIEWPORT_WIDTH, height: RESIDENT_VIEWPORT_HEIGHT };
}
return (
residentWebviewSizesByBrowserId.get(browserId) ?? {
width: RESIDENT_VIEWPORT_WIDTH,
height: RESIDENT_VIEWPORT_HEIGHT,
}
);
}
function applyResidentWebviewStyle(webview: HTMLElement, browserId: string | null): void {
const dimensions = dimensionsForBrowser(browserId);
webview.style.display = "inline-flex";
webview.style.flex = "0 0 auto";
webview.style.width = `${dimensions.width}px`;
webview.style.height = `${dimensions.height}px`;
function applyResidentWebviewStyle(webview: HTMLElement): void {
webview.style.display = "block";
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
webview.style.border = "0";
webview.style.background = "transparent";
webview.style.position = "absolute";
webview.style.left = "0";
webview.style.top = "0";
webview.style.marginTop = "0";
webview.style.zIndex = "0";
}
function clearResidentWebviewParkingStyle(webview: HTMLElement): void {
webview.style.position = "";
webview.style.left = "";
webview.style.top = "";
webview.style.marginTop = "";
webview.style.zIndex = "";
}
export function prepareBrowserWebview(
@@ -131,15 +88,11 @@ export function ensureResidentBrowserWebview(input: {
const resident = residentWebviewsByBrowserId.get(browserId) ?? null;
if (resident?.isConnected) {
releaseResidentBrowserWebview(browserId, resident);
return resident;
}
const existing = findBrowserWebview(browserId, ownerDocument);
if (existing) {
if (existing.parentElement?.id === RESIDENT_BROWSER_HOST_ID) {
releaseResidentBrowserWebview(browserId, existing);
}
return existing;
}
@@ -161,7 +114,6 @@ export function takeResidentBrowserWebview(browserId: string): HTMLElement | nul
}
residentWebviewsByBrowserId.delete(normalizedBrowserId);
clearResidentWebviewParkingStyle(webview);
return webview;
}
@@ -177,33 +129,10 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl
}
residentWebviewsByBrowserId.set(normalizedBrowserId, webview);
applyResidentWebviewStyle(webview, normalizedBrowserId);
applyResidentWebviewStyle(webview);
getResidentBrowserHost(ownerDocument).appendChild(webview);
}
export function resizeResidentBrowserWebview(input: {
browserId: string;
width: number;
height: number;
}): { width: number; height: number } | null {
const normalizedBrowserId = trimNonEmpty(input.browserId);
if (!normalizedBrowserId) {
return null;
}
const width = Math.max(1, Math.round(input.width));
const height = Math.max(1, Math.round(input.height));
residentWebviewSizesByBrowserId.set(normalizedBrowserId, { width, height });
const ownerDocument = readDocument();
const webview = ownerDocument ? findBrowserWebview(normalizedBrowserId, ownerDocument) : null;
if (webview) {
webview.style.width = `${width}px`;
webview.style.height = `${height}px`;
}
return { width, height };
}
export function removeResidentBrowserWebview(browserId: string): void {
const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) {
@@ -212,7 +141,6 @@ export function removeResidentBrowserWebview(browserId: string): void {
const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
residentWebviewsByBrowserId.delete(normalizedBrowserId);
residentWebviewSizesByBrowserId.delete(normalizedBrowserId);
resident?.remove();
}
@@ -221,6 +149,5 @@ export function clearResidentBrowserWebviewsForTests(): void {
webview.remove();
}
residentWebviewsByBrowserId.clear();
residentWebviewSizesByBrowserId.clear();
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
}
@@ -11,6 +11,7 @@ export function getAttachmentKey(attachment: WorkspaceComposerAttachment): strin
tag: attachment.attachment.tag,
text: attachment.attachment.text,
html: attachment.attachment.outerHTML,
intent: attachment.attachment.intent ?? null,
comment: attachment.attachment.comment ?? null,
});
}
+20 -20
View File
@@ -37,10 +37,10 @@ const CATALOG_DATA = [
title: "Auggie CLI",
description:
"Augment Code's powerful software agent, backed by industry-leading context engine",
version: "0.32.0",
version: "0.31.0",
iconId: "auggie",
installLink: "https://www.augmentcode.com/",
command: ["npx", "-y", "@augmentcode/auggie@0.32.0", "--acp"],
command: ["npx", "-y", "@augmentcode/auggie@0.31.0", "--acp"],
env: {
AUGMENT_DISABLE_AUTO_UPDATE: "1",
},
@@ -59,19 +59,19 @@ const CATALOG_DATA = [
title: "Cline",
description:
"Autonomous coding agent CLI - capable of creating/editing files, running commands, using the browser, and more",
version: "3.0.37",
version: "3.0.34",
iconId: "cline",
installLink: "https://cline.bot/cli",
command: ["npx", "-y", "cline@3.0.37", "--acp"],
command: ["npx", "-y", "cline@3.0.34", "--acp"],
},
{
id: "codebuddy-code",
title: "Codebuddy Code",
description: "Tencent Cloud's official intelligent coding tool",
version: "2.116.0",
version: "2.114.0",
iconId: "codebuddy-code",
installLink: "https://www.codebuddy.cn/cli/",
command: ["npx", "-y", "@tencent-ai/codebuddy-code@2.116.0", "--acp"],
command: ["npx", "-y", "@tencent-ai/codebuddy-code@2.114.0", "--acp"],
},
{
id: "codewhale",
@@ -140,29 +140,29 @@ const CATALOG_DATA = [
id: "dimcode",
title: "DimCode",
description: "A coding agent that puts leading models at your command.",
version: "0.2.19",
version: "0.2.12",
iconId: "dimcode",
installLink: "https://dimcode.dev/docs/acp.html",
command: ["npx", "-y", "dimcode@0.2.19", "acp"],
command: ["npx", "-y", "dimcode@0.2.12", "acp"],
},
{
id: "dirac",
title: "Dirac",
description:
"Reduces API costs by more than 50%, produces better and faster work. Uses Hash anchored parallel edits, AST manipulation and a whole lot of neat optimizations. Fully Open Source.",
version: "0.4.13",
version: "0.4.12",
iconId: "dirac",
installLink: "https://dirac.run",
command: ["npx", "-y", "dirac-cli@0.4.13", "--acp"],
command: ["npx", "-y", "dirac-cli@0.4.12", "--acp"],
},
{
id: "factory-droid",
title: "Factory Droid",
description: "Factory Droid - AI coding agent powered by Factory AI",
version: "0.164.0",
version: "0.161.0",
iconId: "factory-droid",
installLink: "https://factory.ai/product/cli",
command: ["npx", "-y", "droid@0.164.0", "exec", "--output-format", "acp-daemon"],
command: ["npx", "-y", "droid@0.161.0", "exec", "--output-format", "acp-daemon"],
env: {
DROID_DISABLE_AUTO_UPDATE: "true",
FACTORY_DROID_AUTO_UPDATE_ENABLED: "false",
@@ -173,10 +173,10 @@ const CATALOG_DATA = [
id: "fast-agent",
title: "fast-agent",
description: "Code and build agents with comprehensive multi-provider support",
version: "0.8.3",
version: "0.8.1",
iconId: "fast-agent",
installLink: "https://fast-agent.ai/acp/",
command: ["uvx", "--from", "fast-agent-acp==0.8.3", "fast-agent-acp", "-x"],
command: ["uvx", "--from", "fast-agent-acp==0.8.1", "fast-agent-acp", "-x"],
},
{
id: "gemini",
@@ -284,10 +284,10 @@ const CATALOG_DATA = [
id: "nova",
title: "Nova",
description: "Nova by Compass AI - a fully-fledged software engineer at your command",
version: "1.1.24",
version: "1.1.22",
iconId: "nova",
installLink: "https://www.compassap.ai/portfolio/nova.html",
command: ["npx", "-y", "@compass-ai/nova@1.1.24", "acp"],
command: ["npx", "-y", "@compass-ai/nova@1.1.22", "acp"],
},
{
id: "poolside",
@@ -302,19 +302,19 @@ const CATALOG_DATA = [
id: "qoder",
title: "Qoder CLI",
description: "AI coding assistant with agentic capabilities",
version: "1.0.37",
version: "1.0.33",
iconId: "qoder",
installLink: "https://qoder.com",
command: ["npx", "-y", "@qoder-ai/qodercli@1.0.37", "--acp"],
command: ["npx", "-y", "@qoder-ai/qodercli@1.0.33", "--acp"],
},
{
id: "qwen-code",
title: "Qwen Code",
description: "Alibaba's Qwen coding assistant",
version: "0.19.6",
version: "0.19.3",
iconId: "qwen-code",
installLink: "https://qwenlm.github.io/qwen-code-docs/en/users/overview",
command: ["npx", "-y", "@qwen-code/qwen-code@0.19.6", "--acp", "--experimental-skills"],
command: ["npx", "-y", "@qwen-code/qwen-code@0.19.3", "--acp", "--experimental-skills"],
},
{
id: "sigit",
+1 -1
View File
@@ -128,11 +128,11 @@ export interface DesktopBrowserNewTabRequestEvent {
export interface DesktopBrowserBridge {
registerWorkspaceBrowser?: (input: { browserId: string; workspaceId: string }) => Promise<void>;
unregisterWorkspaceBrowser?: (browserId: string) => Promise<void>;
setWorkspaceActiveBrowser?: (input: {
workspaceId: string;
browserId: string | null;
}) => Promise<void>;
setAgentActiveBrowser?: (input: { agentId: string; browserId: string | null }) => Promise<void>;
openDevTools?: (browserId: string) => Promise<unknown>;
clearPartition?: (browserId: string) => Promise<void>;
executeAutomationCommand?: (
+12 -7
View File
@@ -422,17 +422,22 @@ export const ar: TranslationResources = {
enterUrl: "أدخل URL",
openDevTools: "افتح أدوات تطوير المتصفح",
cancelSelector: "إلغاء محدد العنصر",
annotateElement: "التعليق على العنصر",
screenshotElement: "لقطة للعنصر",
screenshotCopied: "تم نسخ لقطة الشاشة إلى الحافظة",
elementCopied: "تم نسخ العنصر إلى الحافظة",
screenshotFailed: "تعذّر نسخ لقطة الشاشة",
selectElement: "حدد العنصر",
grabElement: "نسخ العنصر إلى الحافظة",
grabElementLabel: "العنصر",
grabFailed: "تعذّر نسخ العنصر",
},
annotate: {
title: "التعليق على العنصر",
placeholder: "رسالة إلى الوكيل حول هذا العنصر…",
title: "إرسال ملاحظات إلى الوكيل",
placeholder: "صف ما الذي ينبغي تغييره…",
submit: "إرفاق",
cancel: "إلغاء",
intents: {
fix: "إصلاح",
change: "تغيير",
question: "سؤال",
approve: "موافقة",
},
},
devices: {
label: "حجم الجهاز",
+12 -7
View File
@@ -422,17 +422,22 @@ export const en = {
enterUrl: "Enter URL",
openDevTools: "Open browser dev tools",
cancelSelector: "Cancel element selector",
annotateElement: "Annotate element",
screenshotElement: "Screenshot element",
screenshotCopied: "Copied screenshot to clipboard",
elementCopied: "Copied element to clipboard",
screenshotFailed: "Couldn't copy screenshot",
selectElement: "Select element",
grabElement: "Copy element to clipboard",
grabElementLabel: "element",
grabFailed: "Couldn't copy element",
},
annotate: {
title: "Annotate element",
placeholder: "Message to the agent about this element…",
title: "Send feedback to agent",
placeholder: "Describe what should change…",
submit: "Attach",
cancel: "Cancel",
intents: {
fix: "Fix",
change: "Change",
question: "Question",
approve: "Approve",
},
},
devices: {
label: "Device size",
+12 -7
View File
@@ -426,17 +426,22 @@ export const es: TranslationResources = {
enterUrl: "IngreseURL",
openDevTools: "Abrir herramientas de desarrollo del navegador",
cancelSelector: "Cancelar selector de elementos",
annotateElement: "Anotar elemento",
screenshotElement: "Capturar elemento",
screenshotCopied: "Captura copiada al portapapeles",
elementCopied: "Elemento copiado al portapapeles",
screenshotFailed: "No se pudo copiar la captura",
selectElement: "Seleccionar elemento",
grabElement: "Copiar elemento al portapapeles",
grabElementLabel: "elemento",
grabFailed: "No se pudo copiar el elemento",
},
annotate: {
title: "Anotar elemento",
placeholder: "Mensaje al agente sobre este elemento…",
title: "Enviar comentarios al agente",
placeholder: "Describe qué debería cambiar…",
submit: "Adjuntar",
cancel: "Cancelar",
intents: {
fix: "Corregir",
change: "Cambiar",
question: "Pregunta",
approve: "Aprobar",
},
},
devices: {
label: "Tamaño del dispositivo",
+12 -7
View File
@@ -426,17 +426,22 @@ export const fr: TranslationResources = {
enterUrl: "EntrezURL",
openDevTools: "Outils de développement du navigateur ouvert",
cancelSelector: "Annuler le sélecteur d'élément",
annotateElement: "Annoter l'élément",
screenshotElement: "Capturer l'élément",
screenshotCopied: "Capture d'écran copiée dans le presse-papiers",
elementCopied: "Élément copié dans le presse-papiers",
screenshotFailed: "Impossible de copier la capture",
selectElement: "Sélectionner un élément",
grabElement: "Copier l'élément dans le presse-papiers",
grabElementLabel: "élément",
grabFailed: "Impossible de copier l'élément",
},
annotate: {
title: "Annoter l'élément",
placeholder: "Message à l'agent concernant cet élément…",
title: "Envoyer un retour à l'agent",
placeholder: "Décrivez ce qui doit changer…",
submit: "Joindre",
cancel: "Annuler",
intents: {
fix: "Corriger",
change: "Modifier",
question: "Question",
approve: "Approuver",
},
},
devices: {
label: "Taille de l'appareil",
+12 -7
View File
@@ -426,17 +426,22 @@ export const ja: TranslationResources = {
enterUrl: "URLを入力",
openDevTools: "ブラウザ開発ツールを開く",
cancelSelector: "要素セレクターをキャンセル",
annotateElement: "要素に注釈を付ける",
screenshotElement: "要素のスクリーンショット",
screenshotCopied: "スクリーンショットをクリップボードにコピーしました",
elementCopied: "要素をクリップボードにコピーしました",
screenshotFailed: "スクリーンショットをコピーできませんでした",
selectElement: "要素を選択",
grabElement: "要素をクリップボードにコピー",
grabElementLabel: "要素",
grabFailed: "要素をコピーできませんでした",
},
annotate: {
title: "要素に注釈を付ける",
placeholder: "この要素についてエージェントへのメッセージ…",
title: "エージェントにフィードバックを送信",
placeholder: "変更すべき内容を記述…",
submit: "添付",
cancel: "キャンセル",
intents: {
fix: "修正",
change: "変更",
question: "質問",
approve: "承認",
},
},
devices: {
label: "デバイスサイズ",
+12 -7
View File
@@ -426,17 +426,22 @@ export const ptBR: TranslationResources = {
enterUrl: "Inserir URL",
openDevTools: "Abrir ferramentas de desenvolvedor do navegador",
cancelSelector: "Cancelar seletor de elemento",
annotateElement: "Anotar elemento",
screenshotElement: "Capturar elemento",
screenshotCopied: "Captura copiada para a área de transferência",
elementCopied: "Elemento copiado para a área de transferência",
screenshotFailed: "Não foi possível copiar a captura",
selectElement: "Selecionar elemento",
grabElement: "Copiar elemento para a área de transferência",
grabElementLabel: "elemento",
grabFailed: "Não foi possível copiar o elemento",
},
annotate: {
title: "Anotar elemento",
placeholder: "Mensagem ao agente sobre este elemento…",
title: "Enviar feedback ao agente",
placeholder: "Descreva o que deve mudar…",
submit: "Anexar",
cancel: "Cancelar",
intents: {
fix: "Corrigir",
change: "Alterar",
question: "Pergunta",
approve: "Aprovar",
},
},
devices: {
label: "Tamanho do dispositivo",
+12 -7
View File
@@ -426,17 +426,22 @@ export const ru: TranslationResources = {
enterUrl: "Введите URL",
openDevTools: "Открыть инструменты разработки браузера",
cancelSelector: "Отменить выбор элемента",
annotateElement: "Аннотировать элемент",
screenshotElement: "Снимок элемента",
screenshotCopied: "Снимок скопирован в буфер обмена",
elementCopied: "Элемент скопирован в буфер обмена",
screenshotFailed: "Не удалось скопировать снимок",
selectElement: "Выберите элемент",
grabElement: "Скопировать элемент в буфер обмена",
grabElementLabel: "элемент",
grabFailed: "Не удалось скопировать элемент",
},
annotate: {
title: "Аннотировать элемент",
placeholder: "Сообщение агенту об этом элементе…",
title: "Отправить отзыв агенту",
placeholder: "Опишите, что нужно изменить…",
submit: "Прикрепить",
cancel: "Отмена",
intents: {
fix: "Исправить",
change: "Изменить",
question: "Вопрос",
approve: "Одобрить",
},
},
devices: {
label: "Размер устройства",
+12 -7
View File
@@ -422,17 +422,22 @@ export const zhCN: TranslationResources = {
enterUrl: "输入 URL",
openDevTools: "打开浏览器开发者工具",
cancelSelector: "取消元素选择器",
annotateElement: "标注元素",
screenshotElement: "截图元素",
screenshotCopied: "已将截图复制到剪贴板",
elementCopied: "已将元素复制到剪贴板",
screenshotFailed: "无法复制截图",
selectElement: "选择元素",
grabElement: "复制元素到剪贴板",
grabElementLabel: "元素",
grabFailed: "复制元素失败",
},
annotate: {
title: "标注元素",
placeholder: "给智能体关于此元素的留言…",
title: "发送反馈给智能体",
placeholder: "描述需要修改的内容…",
submit: "附加",
cancel: "取消",
intents: {
fix: "修复",
change: "修改",
question: "提问",
approve: "认可",
},
},
devices: {
label: "设备尺寸",
+3 -9
View File
@@ -39,7 +39,6 @@ import {
} from "@/desktop/daemon/desktop-daemon-transport";
import { getDesktopHost } from "@/desktop/host";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { replaceFetchedAgentDirectory } from "@/utils/agent-directory-sync";
import { useSessionStore } from "@/stores/session-store";
import {
@@ -523,15 +522,10 @@ function probeIntervalForConnection(
}
function createDefaultDeps(): HostRuntimeControllerDeps {
const browserHostAvailable =
const desktopBrowserAutomationAvailable =
typeof getDesktopHost()?.browser?.executeAutomationCommand === "function";
const browserAutomationCapabilities = browserHostAvailable
? {
[CLIENT_CAPS.browserHost]: {
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
hostKind: "desktop app",
},
}
const browserAutomationCapabilities = desktopBrowserAutomationAvailable
? { [CLIENT_CAPS.desktopBrowserAutomation]: true }
: undefined;
return {
@@ -2,7 +2,7 @@ import type { MutableDaemonConfig } from "@getpaseo/protocol/messages";
export const BROWSER_TOOLS_TITLE = "Browser tools";
export const BROWSER_TOOLS_WARNING =
"Allow agents to access and control Paseo browser tabs, including logged-in browser state. Only enable this for agents you trust.";
"Allow agents to access and control Paseo desktop browser tabs, including logged-in browser state. Only enable this for agents you trust.";
export interface BrowserToolsCardState {
isVisible: boolean;
@@ -1,5 +1,4 @@
import AsyncStorage from "@react-native-async-storage/async-storage";
import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { create } from "zustand";
import { createJSONStorage, persist } from "zustand/middleware";
import {
@@ -23,14 +22,10 @@ interface BrowserStoreState extends BrowserIndexState {
}
function createBrowserId(): string {
let browserId: string;
if (typeof globalThis.crypto?.randomUUID === "function") {
browserId = globalThis.crypto.randomUUID();
} else {
const randomSuffix = Math.random().toString(16).slice(2) || "0";
browserId = `${Date.now()}-${randomSuffix}`;
return globalThis.crypto.randomUUID();
}
return BrowserAutomationBrowserIdSchema.parse(browserId);
return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
}
export const useBrowserStore = create<BrowserStoreState>()(
+4 -4
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/cli",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"description": "Paseo CLI - control your AI coding agents from the command line",
"bin": {
"paseo": "bin/paseo"
@@ -27,9 +27,9 @@
},
"dependencies": {
"@clack/prompts": "^1.0.0",
"@getpaseo/client": "0.1.104-beta.3",
"@getpaseo/protocol": "0.1.104-beta.3",
"@getpaseo/server": "0.1.104-beta.3",
"@getpaseo/client": "0.1.103",
"@getpaseo/protocol": "0.1.103",
"@getpaseo/server": "0.1.103",
"chalk": "^5.3.0",
"commander": "^12.0.0",
"mime-types": "^2.1.35",
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/client",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"description": "Paseo client SDK package",
"files": [
"dist",
@@ -35,8 +35,8 @@
"test": "vitest run"
},
"dependencies": {
"@getpaseo/protocol": "0.1.104-beta.3",
"@getpaseo/relay": "0.1.104-beta.3",
"@getpaseo/protocol": "0.1.103",
"@getpaseo/relay": "0.1.103",
"zod": "^4.4.3"
},
"devDependencies": {
+5 -22
View File
@@ -2,7 +2,6 @@ import { afterEach, expect, expectTypeOf, test, vi } from "vitest";
import { z } from "zod";
import { DaemonClient, type DaemonTransport, type Logger } from "./daemon-client";
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import {
decodeFileTransferFrame,
encodeFileTransferFrame,
@@ -165,7 +164,7 @@ test("does not infer browser automation capabilities from Electron runtime", asy
capabilities: z.record(z.unknown()),
})
.parse(JSON.parse(assertStr(mock.sent[0])));
expect(hello.capabilities[CLIENT_CAPS.browserHost]).toBeUndefined();
expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBeUndefined();
});
test("advertises consumer-provided browser automation capabilities", async () => {
@@ -175,12 +174,7 @@ test("advertises consumer-provided browser automation capabilities", async () =>
clientId: "browser_capability_unit_test",
transportFactory: () => mock.transport,
reconnect: { enabled: false },
capabilities: {
[CLIENT_CAPS.browserHost]: {
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
hostKind: "desktop app",
},
},
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
});
clients.push(client);
@@ -194,10 +188,7 @@ test("advertises consumer-provided browser automation capabilities", async () =>
capabilities: z.record(z.unknown()),
})
.parse(JSON.parse(assertStr(mock.sent[0])));
expect(hello.capabilities[CLIENT_CAPS.browserHost]).toEqual({
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
hostKind: "desktop app",
});
expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBe(true);
});
const noopLogger: Logger = {
@@ -515,12 +506,7 @@ test("advertises client capabilities in hello", async () => {
logger,
reconnect: { enabled: false },
transportFactory: () => mock.transport,
capabilities: {
browser_host: {
supportedCommands: ["list_tabs"],
hostKind: "desktop app",
},
},
capabilities: { desktop_browser_automation: true },
});
clients.push(client);
@@ -538,10 +524,7 @@ test("advertises client capabilities in hello", async () => {
custom_mode_icons: true,
reasoning_merge_enum: true,
terminal_reflowable_snapshot: true,
browser_host: {
supportedCommands: ["list_tabs"],
hostKind: "desktop app",
},
desktop_browser_automation: true,
},
});
});
+1 -1
View File
@@ -252,7 +252,7 @@ export interface DaemonClientConfig {
};
runtimeMetricsIntervalMs?: number;
runtimeMetricsWindowMs?: number;
capabilities?: Partial<Record<ClientCapability, unknown>>;
capabilities?: Partial<Record<ClientCapability, boolean>>;
}
export interface SendMessageOptions {
@@ -1,2 +0,0 @@
out/
fatal-error.txt
@@ -1,89 +0,0 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>Paseo Capture Harness Target</title>
<style>
html,
body {
width: 1280px;
min-height: 1600px;
margin: 0;
background: #ff00ff;
overflow: auto;
font-family: Arial, Helvetica, sans-serif;
}
.label {
position: absolute;
left: 48px;
top: 48px;
color: #ffffff;
font-size: 72px;
font-weight: 800;
line-height: 1;
text-shadow:
5px 5px 0 #000000,
-2px -2px 0 #000000;
}
.sub {
position: absolute;
left: 54px;
top: 150px;
color: #000000;
background: #ffffff;
font-size: 34px;
font-weight: 700;
padding: 10px 18px;
}
.block {
position: absolute;
left: 56px;
top: 220px;
width: 360px;
height: 92px;
background: linear-gradient(
90deg,
#000000 0 25%,
#ffffff 25% 50%,
#000000 50% 75%,
#ffffff 75% 100%
);
border: 8px solid #000000;
}
.bottom {
position: absolute;
left: 48px;
top: 1370px;
color: #ffffff;
font-size: 60px;
font-weight: 800;
text-shadow: 5px 5px 0 #000000;
}
</style>
</head>
<body>
<div class="label">PASEO CAPTURE HARNESS</div>
<div class="sub">LOCAL MAGENTA TARGET 1280x800 VIEWPORT</div>
<div class="block"></div>
<div class="bottom">FULL PAGE MARKER AT Y=1370</div>
<script>
const params = new URLSearchParams(window.location.search);
const label = params.get("label");
const sub = params.get("sub");
const bottom = params.get("bottom");
if (label) {
document.querySelector(".label").textContent = label;
}
if (sub) {
document.querySelector(".sub").textContent = sub;
}
if (bottom) {
document.querySelector(".bottom").textContent = bottom;
}
</script>
</body>
</html>
@@ -1,93 +0,0 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>Paseo Delayed Capture Harness Target</title>
<style>
html,
body {
width: 1280px;
min-height: 1600px;
margin: 0;
background: #202020;
overflow: auto;
font-family: Arial, Helvetica, sans-serif;
}
.content {
opacity: 0;
}
body.ready {
background: #ff00ff;
}
body.ready .content {
opacity: 1;
}
.label {
position: absolute;
left: 48px;
top: 48px;
color: #ffffff;
font-size: 72px;
font-weight: 800;
line-height: 1;
text-shadow:
5px 5px 0 #000000,
-2px -2px 0 #000000;
}
.sub {
position: absolute;
left: 54px;
top: 150px;
color: #000000;
background: #ffffff;
font-size: 34px;
font-weight: 700;
padding: 10px 18px;
}
.block {
position: absolute;
left: 56px;
top: 220px;
width: 360px;
height: 92px;
background: linear-gradient(
90deg,
#000000 0 25%,
#ffffff 25% 50%,
#000000 50% 75%,
#ffffff 75% 100%
);
border: 8px solid #000000;
}
.bottom {
position: absolute;
left: 48px;
top: 1370px;
color: #ffffff;
font-size: 60px;
font-weight: 800;
text-shadow: 5px 5px 0 #000000;
}
</style>
<script>
setTimeout(() => {
document.body.classList.add("ready");
}, 1500);
</script>
</head>
<body>
<div class="content">
<div class="label">PASEO CAPTURE HARNESS</div>
<div class="sub">DELAYED FIRST PAINT TARGET</div>
<div class="block"></div>
<div class="bottom">FULL PAGE MARKER AT Y=1370</div>
</div>
</body>
</html>
-376
View File
@@ -1,376 +0,0 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>Paseo Browser Capture Harness</title>
<style>
html,
body {
width: 100%;
height: 100%;
margin: 0;
background: #202020;
overflow: hidden;
font-family: Arial, sans-serif;
}
#paseo-browser-resident-webviews {
position: fixed;
left: 0;
top: 0;
width: 1px;
height: 1px;
overflow: hidden;
opacity: 1;
pointer-events: none;
display: block;
visibility: visible;
}
.capture-harness-webview {
display: inline-flex;
flex: 0 0 auto;
width: 1280px;
height: 800px;
border: 0;
background: transparent;
}
</style>
</head>
<body>
<div id="paseo-browser-resident-webviews" aria-hidden="true"></div>
<script>
const RESIDENT_VIEWPORT_WIDTH = 1280;
const RESIDENT_VIEWPORT_HEIGHT = 800;
const host = document.getElementById("paseo-browser-resident-webviews");
const params = new URLSearchParams(window.location.search);
const requestedWebviewCount = params.has("webviewCount")
? Number(params.get("webviewCount"))
: 2;
const webviewCount = Number.isFinite(requestedWebviewCount)
? Math.max(0, requestedWebviewCount)
: 2;
const webviews = [];
const activeTokens = new Set();
let nextTokenId = 0;
let permanentParkingState = params.get("permanentParkingState") || "";
function appendHarnessWebview(sourceUrl) {
const index = webviews.length;
const webview = document.createElement("webview");
webview.id = `target-webview-${index + 1}`;
webview.className = "capture-harness-webview";
webview.setAttribute("data-paseo-browser-id", `capture-harness-${index + 1}`);
webview.setAttribute("partition", `persist:paseo-capture-harness-${index + 1}`);
webview.setAttribute("allowpopups", "true");
webview.setAttribute("spellcheck", "false");
webview.setAttribute("autosize", "on");
webview.src = sourceUrl;
applyStackedWebviewStyle(webview);
host.appendChild(webview);
webviews.push(webview);
return index;
}
for (let index = 0; index < webviewCount; index += 1) {
appendHarnessWebview(params.get("targetUrl") || "bright.html");
}
function applyHostParking() {
host.style.position = "fixed";
host.style.left = "-20000px";
host.style.top = "0";
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
host.style.overflow = "hidden";
host.style.opacity = "0";
host.style.pointerEvents = "none";
host.style.zIndex = "";
host.style.clipPath = "";
host.style.visibility = "";
host.style.transform = "";
}
function applyWebviewBaseStyle(webview) {
webview.style.display = "inline-flex";
webview.style.flex = "0 0 auto";
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
webview.style.border = "0";
webview.style.background = "transparent";
}
function applyLegacyVerticalWebviewStyle(webview, index) {
applyWebviewBaseStyle(webview);
webview.style.position = "";
webview.style.left = "";
webview.style.top = "";
webview.style.marginTop = index === 0 ? "0" : "4px";
webview.style.zIndex = "";
}
function applyStackedWebviewStyle(webview, zIndex = "0") {
applyWebviewBaseStyle(webview);
webview.style.position = "absolute";
webview.style.left = "0";
webview.style.top = "0";
webview.style.marginTop = "0";
webview.style.zIndex = zIndex;
}
function applyParking() {
permanentParkingState = "";
applyHostParking();
webviews.forEach((webview) => {
applyStackedWebviewStyle(webview);
});
}
function applyLegacyVerticalParking() {
applyHostParking();
webviews.forEach((webview, index) => {
applyLegacyVerticalWebviewStyle(webview, index);
});
}
function applyCapturePrep() {
permanentParkingState = "";
host.style.position = "fixed";
host.style.left = "0";
host.style.top = "0";
host.style.width = "1px";
host.style.height = "1px";
host.style.overflow = "hidden";
host.style.opacity = "1";
host.style.pointerEvents = "none";
host.style.zIndex = "1";
host.style.clipPath = "";
host.style.visibility = "";
host.style.transform = "";
}
function applyHostPermanentBase() {
host.style.position = "fixed";
host.style.left = "0";
host.style.top = "0";
host.style.width = "1px";
host.style.height = "1px";
host.style.overflow = "hidden";
host.style.opacity = "1";
host.style.pointerEvents = "none";
host.style.display = "block";
host.style.zIndex = "";
host.style.clipPath = "";
host.style.visibility = "visible";
host.style.transform = "";
host.style.transformOrigin = "";
}
function applyPermanentWebviewBase(webview, index) {
applyStackedWebviewStyle(webview);
webview.style.opacity = "";
webview.style.transform = "";
webview.style.pointerEvents = "";
webview.style.zIndex = "0";
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
webview.style.left = "0";
webview.style.top = "0";
}
function applyPermanentParkingState(stateName) {
permanentParkingState = stateName;
activeTokens.clear();
applyHostPermanentBase();
webviews.forEach(applyPermanentWebviewBase);
if (stateName === "p1-overflow-1x1") {
return state();
}
if (stateName === "p2-clip-path-1x1") {
host.style.width = "1px";
host.style.height = "1px";
host.style.overflow = "visible";
host.style.clipPath = "inset(1px)";
return state();
}
if (stateName === "p3-opacity-0") {
host.style.opacity = "0";
return state();
}
if (stateName === "p4-transform-scale-0") {
host.style.transform = "scale(0)";
return state();
}
if (stateName === "p4-transform-scale-0001") {
host.style.transform = "scale(0.001)";
return state();
}
if (stateName === "p5-webview-0x0") {
webviews.forEach((webview) => {
webview.style.width = "0";
webview.style.height = "0";
});
return state();
}
if (stateName === "p5-webview-1x1") {
webviews.forEach((webview) => {
webview.style.width = "1px";
webview.style.height = "1px";
});
return state();
}
if (stateName === "p6-z-index-negative") {
host.style.zIndex = "-1";
return state();
}
if (stateName === "p7-opacity-001") {
host.style.opacity = "0.01";
return state();
}
throw new Error(`Unknown permanent parking state: ${stateName}`);
}
function applyTargetStacking(targetIndex) {
webviews.forEach((webview, index) => {
applyStackedWebviewStyle(webview, index === targetIndex ? "2" : "0");
});
}
function waitFrames(count) {
return new Promise((resolve) => {
function step(remaining) {
if (remaining <= 0) {
resolve();
return;
}
requestAnimationFrame(() => step(remaining - 1));
}
step(count);
});
}
function state(targetIndex = 0) {
const hostRect = host.getBoundingClientRect();
const targetWebview = webviews[targetIndex] || webviews[0];
const webviewRect = targetWebview?.getBoundingClientRect();
return {
hostStyle: {
left: host.style.left,
top: host.style.top,
width: host.style.width,
height: host.style.height,
overflow: host.style.overflow,
opacity: host.style.opacity,
pointerEvents: host.style.pointerEvents,
zIndex: host.style.zIndex,
clipPath: host.style.clipPath,
transform: host.style.transform,
},
hostRect: {
x: hostRect.x,
y: hostRect.y,
width: hostRect.width,
height: hostRect.height,
},
webviewRect: {
x: webviewRect?.x ?? 0,
y: webviewRect?.y ?? 0,
width: webviewRect?.width ?? 0,
height: webviewRect?.height ?? 0,
},
webviews: webviews.map((webview) => {
const rect = webview.getBoundingClientRect();
return {
id: webview.id,
zIndex: webview.style.zIndex,
position: webview.style.position,
opacity: webview.style.opacity,
x: rect.x,
y: rect.y,
width: rect.width,
height: rect.height,
};
}),
};
}
window.captureHarness = {
async waitForFrames(count = 2) {
await waitFrames(count);
return state();
},
webContentsIds() {
return webviews.map((webview) =>
typeof webview.getWebContentsId === "function" ? webview.getWebContentsId() : null,
);
},
addWebview(sourceUrl) {
return appendHarnessWebview(sourceUrl || params.get("targetUrl") || "bright.html");
},
addPermanentWebview(sourceUrl, stateName) {
const index = appendHarnessWebview(sourceUrl || params.get("targetUrl") || "bright.html");
applyPermanentParkingState(stateName || permanentParkingState);
return index;
},
async applyPermanentParkingState(stateName) {
const nextState = applyPermanentParkingState(stateName);
await waitFrames(2);
return nextState;
},
async prepareForPixelCapture(targetIndex = 0) {
const token = `capture-${++nextTokenId}`;
activeTokens.add(token);
applyCapturePrep();
applyTargetStacking(targetIndex);
await waitFrames(2);
webviews[targetIndex].getBoundingClientRect();
return { token, state: state(targetIndex) };
},
async prepareLegacyVerticalPixelCapture(targetIndex = 0) {
const token = `legacy-capture-${++nextTokenId}`;
activeTokens.add(token);
applyLegacyVerticalParking();
applyCapturePrep();
await waitFrames(2);
webviews[targetIndex].getBoundingClientRect();
return { token, state: state(targetIndex) };
},
async restorePixelCapture(token) {
activeTokens.delete(token);
if (activeTokens.size === 0) {
applyParking();
}
await waitFrames(2);
return state();
},
async restoreLegacyVerticalParking(token) {
activeTokens.delete(token);
applyLegacyVerticalParking();
await waitFrames(2);
return state();
},
async restoreParking() {
activeTokens.clear();
applyParking();
await waitFrames(2);
return state();
},
};
if (permanentParkingState) {
applyPermanentParkingState(permanentParkingState);
}
</script>
</body>
</html>
File diff suppressed because it is too large Load Diff
-7
View File
@@ -1,7 +0,0 @@
#!/bin/sh
set -eu
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd)
exec "$REPO_ROOT/node_modules/.bin/electron" "$SCRIPT_DIR/main.js"
+1 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/desktop",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"private": true,
"description": "Paseo desktop app (Electron wrapper)",
"homepage": "https://paseo.sh",
@@ -17,7 +17,6 @@
"scripts": {
"build": "npm --prefix ../.. run build:server:clean && npm run build:main && electron-builder --config electron-builder.yml",
"build:main": "tsc -p tsconfig.json",
"capture-harness": "./capture-harness/run.sh",
"dev": "./scripts/dev.sh",
"dev:win": "powershell ./scripts/dev.ps1",
"test": "vitest run",
@@ -1,198 +0,0 @@
import type { SnapshotPage } from "./snapshot-engine.js";
export interface ActionablePoint {
x: number;
y: number;
}
export interface ActionableTarget {
point: ActionablePoint;
rect: {
x: number;
y: number;
width: number;
height: number;
};
}
export type ActionabilityResult =
| { ok: true; target: ActionableTarget }
| { ok: false; reason: "stale_ref" | "timeout"; detail?: string };
const DEFAULT_ACTIONABILITY_TIMEOUT_MS = 5_000;
export async function waitForActionableTarget(input: {
page: SnapshotPage;
elementExpression: string;
editable?: boolean;
timeoutMs?: number;
}): Promise<ActionabilityResult> {
const result = await input.page.executeJavaScript(
buildActionabilityScript({
elementExpression: input.elementExpression,
editable: input.editable === true,
timeoutMs: input.timeoutMs ?? DEFAULT_ACTIONABILITY_TIMEOUT_MS,
}),
);
return readActionabilityResult(result);
}
function readActionabilityResult(value: unknown): ActionabilityResult {
if (!value || typeof value !== "object") {
return { ok: false, reason: "timeout" };
}
const record = value as Record<string, unknown>;
if (record.ok === true && isActionableTarget(record.target)) {
return { ok: true, target: record.target };
}
if (record.ok === false) {
const reason = record.reason;
if (reason === "stale_ref" || reason === "timeout") {
return {
ok: false,
reason,
...(typeof record.detail === "string" ? { detail: record.detail } : {}),
};
}
}
return { ok: false, reason: "timeout" };
}
function isActionableTarget(value: unknown): value is ActionableTarget {
if (!value || typeof value !== "object") {
return false;
}
const record = value as Record<string, unknown>;
return isPoint(record.point) && isRect(record.rect);
}
function isPoint(value: unknown): value is ActionablePoint {
if (!value || typeof value !== "object") {
return false;
}
const record = value as Record<string, unknown>;
return isFiniteNumber(record.x) && isFiniteNumber(record.y);
}
function isRect(value: unknown): value is ActionableTarget["rect"] {
if (!value || typeof value !== "object") {
return false;
}
const record = value as Record<string, unknown>;
return (
isFiniteNumber(record.x) &&
isFiniteNumber(record.y) &&
isFiniteNumber(record.width) &&
isFiniteNumber(record.height)
);
}
function isFiniteNumber(value: unknown): value is number {
return typeof value === "number" && Number.isFinite(value);
}
function buildActionabilityScript(input: {
elementExpression: string;
editable: boolean;
timeoutMs: number;
}): string {
return String.raw`(async () => {
const deadline = performance.now() + ${JSON.stringify(input.timeoutMs)};
const requiresEditable = ${JSON.stringify(input.editable)};
const nextFrame = () => new Promise((resolve) => requestAnimationFrame(resolve));
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const nearlyEqual = (a, b) => Math.abs(a - b) < 0.25;
const sameRect = (a, b) =>
nearlyEqual(a.x, b.x) &&
nearlyEqual(a.y, b.y) &&
nearlyEqual(a.width, b.width) &&
nearlyEqual(a.height, b.height);
const rectPayload = (rect) => ({
x: rect.x,
y: rect.y,
width: rect.width,
height: rect.height,
});
const centerPoint = (rect) => ({
x: Math.min(Math.max(rect.left + rect.width / 2, 0), Math.max(window.innerWidth - 1, 0)),
y: Math.min(Math.max(rect.top + rect.height / 2, 0), Math.max(window.innerHeight - 1, 0)),
});
const isDisabled = (element) => {
if (element.closest?.('[aria-disabled="true"]')) return true;
if ('disabled' in element && element.disabled) return true;
const fieldset = element.closest?.('fieldset[disabled]');
return Boolean(fieldset);
};
const isEditable = (element) => {
if (element.isContentEditable) return true;
const tag = element.tagName?.toLowerCase();
if (tag === 'textarea' || tag === 'select') return !element.readOnly && !isDisabled(element);
if (tag !== 'input') return false;
const type = (element.getAttribute('type') || 'text').toLowerCase();
if (['button', 'checkbox', 'color', 'file', 'hidden', 'image', 'radio', 'range', 'reset', 'submit'].includes(type)) return false;
return !element.readOnly && !isDisabled(element);
};
const isVisible = (element, rect) => {
const style = getComputedStyle(element);
return (
rect.width > 0 &&
rect.height > 0 &&
style.visibility !== 'hidden' &&
style.display !== 'none' &&
Number(style.opacity || '1') !== 0
);
};
const hitTargetReceivesEvents = (element, point) => {
const hit = document.elementFromPoint(point.x, point.y);
return Boolean(hit && (hit === element || element.contains(hit)));
};
const resolveElement = () => (${input.elementExpression});
let detail = 'not actionable';
while (performance.now() <= deadline) {
const element = resolveElement();
if (!element || !element.isConnected) {
return { ok: false, reason: 'stale_ref', detail: 'ref no longer resolves' };
}
const rect = element.getBoundingClientRect();
if (!isVisible(element, rect)) {
detail = 'not visible';
await sleep(25);
continue;
}
if (isDisabled(element)) {
detail = 'disabled';
await sleep(25);
continue;
}
if (requiresEditable && !isEditable(element)) {
detail = 'not editable';
await sleep(25);
continue;
}
element.scrollIntoView?.({ block: 'center', inline: 'center' });
await nextFrame();
const firstRect = element.getBoundingClientRect();
await nextFrame();
const secondRect = element.getBoundingClientRect();
if (!sameRect(firstRect, secondRect)) {
detail = 'moving';
continue;
}
const point = centerPoint(secondRect);
if (!hitTargetReceivesEvents(element, point)) {
detail = 'covered';
await sleep(25);
continue;
}
return { ok: true, target: { point, rect: rectPayload(secondRect) } };
}
return { ok: false, reason: 'timeout', detail };
})()`;
}
@@ -1,311 +0,0 @@
export const ARIA_SNAPSHOT_SCRIPT_MARKER = "__PASEO_ARIA_SNAPSHOT__";
// Adapted from Playwright's injected ARIA snapshot model.
// Copyright (c) Microsoft Corporation. Licensed under the Apache License, Version 2.0.
export const ARIA_SNAPSHOT_SCRIPT = String.raw`(() => {
const MARKER = ${JSON.stringify(ARIA_SNAPSHOT_SCRIPT_MARKER)};
const MAX_NODES = 1500;
const MAX_REFS = 500;
const MAX_TEXT_LENGTH = 80000;
const ACTIONABLE_ROLES = new Set([
'button',
'checkbox',
'combobox',
'link',
'menuitem',
'option',
'radio',
'searchbox',
'slider',
'spinbutton',
'switch',
'tab',
'textbox',
'treeitem'
]);
function normalizeText(value) {
return String(value || '').replace(/[\u200b\u00ad]/g, '').replace(/[\r\n\s\t]+/g, ' ').trim();
}
function visibilityFor(element) {
if (!(element instanceof Element)) return false;
const style = window.getComputedStyle(element);
if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) return false;
const rect = element.getBoundingClientRect();
return rect.width > 0 && rect.height > 0 ? 'box' : 'boxless';
}
function explicitRole(element) {
const role = element.getAttribute('role');
if (!role || role === 'presentation' || role === 'none') return null;
return role.split(/\s+/)[0].toLowerCase();
}
function implicitRole(element) {
const tag = element.tagName.toLowerCase();
if (/^h[1-6]$/.test(tag)) return 'heading';
if (tag === 'a' && element.hasAttribute('href')) return 'link';
if (tag === 'button') return 'button';
if (tag === 'select') return 'combobox';
if (tag === 'textarea') return 'textbox';
if (element instanceof HTMLElement && element.isContentEditable) return 'textbox';
if (tag === 'summary') return 'button';
if (tag === 'main') return 'main';
if (tag === 'nav') return 'navigation';
if (tag === 'header') return 'banner';
if (tag === 'footer') return 'contentinfo';
if (tag === 'section') return element.getAttribute('aria-label') ? 'region' : null;
if (tag === 'ul' || tag === 'ol') return 'list';
if (tag === 'li') return 'listitem';
if (tag === 'table') return 'table';
if (tag === 'tr') return 'row';
if (tag === 'th') return 'columnheader';
if (tag === 'td') return 'cell';
if (tag === 'iframe') return 'iframe';
if (tag === 'input') {
const type = (element.getAttribute('type') || 'text').toLowerCase();
if (type === 'checkbox') return 'checkbox';
if (type === 'radio') return 'radio';
if (type === 'button' || type === 'submit' || type === 'reset') return 'button';
if (type === 'range') return 'slider';
if (type === 'number') return 'spinbutton';
if (type === 'search') return 'searchbox';
if (type === 'hidden') return null;
return 'textbox';
}
return null;
}
function roleFor(element) {
return explicitRole(element) || implicitRole(element);
}
function labelText(element) {
if (!(element instanceof HTMLElement)) return '';
if (element.id) {
const escapedId = window.CSS && typeof window.CSS.escape === 'function'
? window.CSS.escape(element.id)
: String(element.id).replace(/"/g, '\\"');
const label = document.querySelector('label[for="' + escapedId + '"]');
if (label) return normalizeText(label.textContent);
}
const closestLabel = element.closest('label');
return closestLabel ? normalizeText(closestLabel.textContent) : '';
}
function nameFor(element, role) {
const tag = element.tagName.toLowerCase();
const labelledBy = element.getAttribute('aria-labelledby');
if (labelledBy) {
const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.textContent || '').join(' ');
const normalized = normalizeText(text);
if (normalized) return normalized;
}
const pieces = [
element.getAttribute('aria-label'),
labelText(element),
element.getAttribute('alt'),
element.getAttribute('title'),
tag === 'input' || tag === 'textarea' ? element.getAttribute('placeholder') : null,
tag === 'input' || tag === 'textarea' ? element.value : null,
role === 'button' || role === 'link' || role === 'heading' || ACTIONABLE_ROLES.has(role)
? element.textContent
: null
];
return normalizeText(pieces.find((piece) => normalizeText(piece).length > 0) || '');
}
function fingerprintNameFor(element, role, name) {
const tag = element.tagName.toLowerCase();
if (tag !== 'input' && tag !== 'textarea') return name;
const mutableValue = normalizeText(element.value);
if (!mutableValue || name !== mutableValue) return name;
return '';
}
function inheritedDisabled(element) {
if (!(element instanceof Element)) return false;
if (element.hasAttribute('disabled') || element.getAttribute('aria-disabled') === 'true') return true;
if (element.closest('fieldset[disabled]')) return true;
return element.closest('[aria-disabled="true"]') !== null;
}
function isActionable(element, role) {
if (!role || !ACTIONABLE_ROLES.has(role)) return false;
if (visibilityFor(element) !== 'box') return false;
if (inheritedDisabled(element)) return false;
return true;
}
function fingerprintFor(element, role, name) {
return {
role,
name: fingerprintNameFor(element, role, name),
tagName: element.tagName.toLowerCase(),
type: element.getAttribute('type') || '',
ariaLabel: element.getAttribute('aria-label') || ''
};
}
function fingerprintMatches(element, fingerprint) {
const role = roleFor(element) || 'generic';
const name = nameFor(element, role);
const current = fingerprintFor(element, role, name);
return current.role === fingerprint.role &&
current.name === fingerprint.name &&
current.tagName === fingerprint.tagName &&
current.type === fingerprint.type &&
current.ariaLabel === fingerprint.ariaLabel;
}
function ensureRuntime() {
const runtime = {
refs: new Map(),
resolve(ref, fingerprint) {
const element = this.refs.get(ref);
if (!element || !element.isConnected || !fingerprintMatches(element, fingerprint)) {
return { ok: false, reason: 'stale_ref' };
}
return { ok: true, element };
}
};
Object.defineProperty(window, '__PASEO_BROWSER_AUTOMATION__', {
configurable: true,
enumerable: false,
value: runtime
});
return runtime;
}
function stateAttributes(element, role) {
const attrs = [];
if (role === 'heading') {
const tag = element.tagName.toLowerCase();
const level = /^h[1-6]$/.test(tag) ? Number(tag.slice(1)) : Number(element.getAttribute('aria-level') || 0);
if (level) attrs.push('level=' + level);
}
if (element.matches?.('input[type="checkbox"], input[type="radio"]')) {
attrs.push('checked=' + (element.checked ? 'true' : 'false'));
}
if (element.getAttribute('aria-checked')) attrs.push('checked=' + element.getAttribute('aria-checked'));
if (element.getAttribute('aria-expanded')) attrs.push('expanded=' + element.getAttribute('aria-expanded'));
if (element.getAttribute('aria-pressed')) attrs.push('pressed=' + element.getAttribute('aria-pressed'));
if (element.getAttribute('aria-selected')) attrs.push('selected=' + element.getAttribute('aria-selected'));
return attrs;
}
function textNode(text) {
return { kind: 'text', text };
}
function elementNode(element, role, name) {
return {
kind: 'role',
role: role || 'generic',
name,
tagName: element.tagName.toLowerCase(),
attributes: stateAttributes(element, role),
children: []
};
}
let nodeCount = 0;
let refCount = 0;
let iframeCount = 0;
let maxDepth = 0;
let truncated = false;
let textBudget = MAX_TEXT_LENGTH;
const runtime = ensureRuntime();
function countNode(depth) {
if (nodeCount >= MAX_NODES) {
truncated = true;
return false;
}
nodeCount += 1;
maxDepth = Math.max(maxDepth, depth);
return true;
}
function cappedText(text) {
if (text.length <= textBudget) {
textBudget -= text.length;
return text;
}
truncated = true;
const capped = text.slice(0, Math.max(0, textBudget));
textBudget = 0;
return capped;
}
function visitNode(domNode, depth) {
if (!countNode(depth)) return null;
if (domNode.nodeType === Node.TEXT_NODE) {
const text = cappedText(normalizeText(domNode.textContent));
if (!text) return null;
return textNode(text);
}
if (!(domNode instanceof Element)) return null;
const visibility = visibilityFor(domNode);
if (!visibility) return null;
if (domNode.getAttribute('aria-hidden') === 'true') return null;
const role = roleFor(domNode);
const name = role ? nameFor(domNode, role) : '';
const children = [];
for (const child of Array.from(domNode.childNodes)) {
const childSnapshot = visitNode(child, depth + 1);
if (childSnapshot) children.push(childSnapshot);
if (truncated) break;
}
if (domNode.tagName.toLowerCase() === 'iframe') {
iframeCount += 1;
}
if (visibility === 'boxless') {
return children.length > 0 ? { kind: 'group', children } : null;
}
if (!role && children.length === 0) return null;
const snapshotNode = role ? elementNode(domNode, role, name) : { kind: 'group', children: [] };
snapshotNode.children = children;
if (role && isActionable(domNode, role) && refCount < MAX_REFS) {
const ref = '@e' + (refCount + 1);
const fingerprint = fingerprintFor(domNode, role, name);
refCount += 1;
runtime.refs.set(ref, domNode);
snapshotNode.ref = ref;
snapshotNode.fingerprint = fingerprint;
} else if (role && isActionable(domNode, role)) {
truncated = true;
}
return snapshotNode;
}
const root = {
kind: 'role',
role: 'document',
name: normalizeText(document.title),
tagName: 'document',
attributes: [],
children: []
};
for (const child of Array.from(document.body ? document.body.childNodes : document.documentElement.childNodes)) {
const childSnapshot = visitNode(child, 1);
if (childSnapshot) root.children.push(childSnapshot);
if (truncated) break;
}
return JSON.stringify({
marker: MARKER,
root,
refs: Array.from(runtime.refs.entries()).map(([ref, element]) => {
const role = roleFor(element) || 'generic';
const name = nameFor(element, role);
return { ref, fingerprint: fingerprintFor(element, role, name) };
}),
truncated,
stats: { nodeCount, refCount, textLength: 0, iframeCount, maxDepth }
});
})()`;
@@ -1,28 +0,0 @@
export type CdpCommandSender = (
command: string,
params?: Record<string, unknown>,
) => Promise<unknown>;
export class CdpSessionQueue {
private queue: Promise<void> = Promise.resolve();
public async run<T>(task: () => Promise<T>): Promise<T> {
const previous = this.queue;
let releaseCurrent = () => {};
const current = new Promise<void>((resolve) => {
releaseCurrent = resolve;
});
const tail = previous.catch(() => {}).then(() => current);
this.queue = tail;
await previous.catch(() => {});
try {
return await task();
} finally {
releaseCurrent();
if (this.queue === tail) {
this.queue = Promise.resolve();
}
}
}
}
@@ -1,89 +0,0 @@
import type { BrowserAutomationDialogEvent } from "@getpaseo/protocol/browser-automation/rpc-schemas";
export const MAX_DIALOGS_PER_COMMAND = 20;
export interface JavaScriptDialogOpening {
type?: unknown;
message?: unknown;
defaultPrompt?: unknown;
}
interface DialogPolicy {
readonly action: BrowserAutomationDialogEvent["action"];
readonly accept: boolean;
}
const DIALOG_POLICIES: Record<BrowserAutomationDialogEvent["type"], DialogPolicy> = {
alert: { action: "accepted", accept: true },
confirm: { action: "dismissed", accept: false },
prompt: { action: "dismissed", accept: false },
beforeunload: { action: "dismissed", accept: false },
};
export function handledDialogEvent(opening: JavaScriptDialogOpening): BrowserAutomationDialogEvent {
const type = normalizeDialogType(opening.type);
const policy = DIALOG_POLICIES[type];
return {
type,
message: typeof opening.message === "string" ? opening.message : String(opening.message ?? ""),
...(typeof opening.defaultPrompt === "string" ? { defaultValue: opening.defaultPrompt } : {}),
action: policy.action,
timestamp: Date.now(),
};
}
export function dialogAcceptValue(type: BrowserAutomationDialogEvent["type"]): boolean {
return DIALOG_POLICIES[type].accept;
}
export function promptShimInstallScript(): string {
return String.raw`(() => {
const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__";
const state = window[stateKey] || { prompts: [], installed: false };
window[stateKey] = state;
if (state.installed) return true;
const originalPrompt = window.prompt;
Object.defineProperty(state, "originalPrompt", { configurable: true, value: originalPrompt });
const promptShim = (message = "", defaultValue = "") => {
state.prompts.push({
type: "prompt",
message: String(message ?? ""),
defaultValue: String(defaultValue ?? ""),
action: "dismissed",
timestamp: Date.now(),
});
return null;
};
Object.defineProperty(state, "promptShim", { configurable: true, value: promptShim });
window.prompt = promptShim;
state.installed = true;
return true;
})()`;
}
export function promptShimDrainScript(): string {
return String.raw`(() => {
const state = window.__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__;
if (!state || !Array.isArray(state.prompts)) return [];
return state.prompts.splice(0);
})()`;
}
export function promptShimRestoreScript(): string {
return String.raw`(() => {
const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__";
const state = window[stateKey];
if (!state || !state.installed) return true;
if (window.prompt === state.promptShim && typeof state.originalPrompt === "function") {
window.prompt = state.originalPrompt;
}
delete window[stateKey];
return true;
})()`;
}
function normalizeDialogType(value: unknown): BrowserAutomationDialogEvent["type"] {
return value === "alert" || value === "confirm" || value === "prompt" || value === "beforeunload"
? value
: "alert";
}
@@ -1,552 +1,18 @@
import type { Rectangle } from "electron";
import { describe, expect, test, vi } from "vitest";
import type { TabImage } from "./service.js";
import { adaptWebContents } from "./ipc.js";
import { describe, expect, it } from "vitest";
class FakeImage implements TabImage {
public toPNG(): Uint8Array {
return new Uint8Array([137, 80, 78, 71]);
}
import { sanitizeDownloadFileName } from "./ipc.js";
public getSize(): { width: number; height: number } {
return { width: 640, height: 480 };
}
}
class FakeDebugger {
public attachedProtocolVersions: string[] = [];
public commands: Array<{ command: string; params: Record<string, unknown> }> = [];
public blockCommands = false;
public readonly blockedCommandNames = new Set<string>();
public readonly failedCommandNames = new Set<string>();
public readonly promptDialogs: unknown[] = [];
public failPromptDrain = false;
private messageListener:
| ((event: unknown, method: string, params?: Record<string, unknown>) => void)
| null = null;
private readonly blockedCommands: Array<() => void> = [];
public isAttached(): boolean {
return this.attachedProtocolVersions.length > 0;
}
public attach(protocolVersion?: string): void {
this.attachedProtocolVersions.push(protocolVersion ?? "");
}
public async sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown> {
this.commands.push({ command, params: params ?? {} });
if (this.failedCommandNames.has(command)) {
throw new Error(`${command} failed`);
}
if (this.blockCommands || this.blockedCommandNames.has(command)) {
await new Promise<void>((resolve) => {
this.blockedCommands.push(resolve);
});
}
if (command === "Runtime.evaluate" && typeof params?.expression === "string") {
if (params.expression.includes("state.prompts.splice(0)")) {
if (this.failPromptDrain) {
throw new Error("execution context destroyed");
}
return { result: { value: this.promptDialogs.splice(0) } };
}
return { result: { value: true } };
}
return { ok: true };
}
public on(
event: "message",
listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
): void {
expect(event).toBe("message");
this.messageListener = listener;
}
public emitMessage(method: string, params?: Record<string, unknown>): void {
if (!this.messageListener) {
throw new Error("Debugger message listener was not registered");
}
this.messageListener({}, method, params);
}
public finishNextCommand(): void {
const resolve = this.blockedCommands.shift();
if (!resolve) {
throw new Error("No command is blocked");
}
resolve();
}
}
type ConsoleMessageListener = (
event: unknown,
level: unknown,
message: unknown,
line: unknown,
sourceId: unknown,
) => void;
class FakeWebContents {
public readonly debugger = new FakeDebugger();
public readonly captures: Array<{
rect: Rectangle | undefined;
options: { stayHidden?: boolean } | undefined;
}> = [];
public readonly invalidations: string[] = [];
private consoleMessageListener: ConsoleMessageListener | null = null;
private destroyedListener: (() => void) | null = null;
public destroyed = false;
public constructor(public readonly id: number) {}
public getURL(): string {
return "https://example.com";
}
public getTitle(): string {
return "Example";
}
public canGoBack(): boolean {
return false;
}
public canGoForward(): boolean {
return false;
}
public isLoading(): boolean {
return false;
}
public isDestroyed(): boolean {
return this.destroyed;
}
public async executeJavaScript(): Promise<unknown> {
return null;
}
public async loadURL(): Promise<void> {}
public goBack(): void {}
public goForward(): void {}
public reload(): void {}
public async capturePage(
rect?: Rectangle,
options?: { stayHidden?: boolean },
): Promise<TabImage> {
this.captures.push({ rect, options });
return new FakeImage();
}
public invalidate(): void {
this.invalidations.push("invalidate");
}
public on(event: "console-message", listener: ConsoleMessageListener): void {
expect(event).toBe("console-message");
this.consoleMessageListener = listener;
}
public once(event: "destroyed", listener: () => void): void {
expect(event).toBe("destroyed");
this.destroyedListener = listener;
}
public emitConsoleMessage(input: {
level: unknown;
message: unknown;
line: unknown;
sourceId: unknown;
}): void {
if (!this.consoleMessageListener) {
throw new Error("Console listener was not registered");
}
this.consoleMessageListener({}, input.level, input.message, input.line, input.sourceId);
}
public destroy(): void {
this.destroyed = true;
this.destroyedListener?.();
}
}
describe("browser automation IPC adapter", () => {
test("delegates viewport capture to the guest without a renderer prep bridge", async () => {
const contents = new FakeWebContents(20);
const tab = adaptWebContents(contents);
const image = await tab.capturePage({ stayHidden: false });
tab.invalidate();
expect(image.getSize()).toEqual({ width: 640, height: 480 });
expect(contents.captures).toEqual([{ rect: undefined, options: { stayHidden: false } }]);
expect(contents.invalidations).toEqual(["invalidate"]);
});
test("collects console messages until the guest is destroyed", () => {
const contents = new FakeWebContents(21);
const tab = adaptWebContents(contents);
contents.emitConsoleMessage({
level: "warning",
message: "hello",
line: 12,
sourceId: "https://example.com/app.js",
});
expect(tab.getConsoleMessages?.()).toEqual([
{
level: "warning",
message: "hello",
line: 12,
source: "https://example.com/app.js",
timestamp: expect.any(Number),
},
]);
contents.destroy();
expect(tab.getConsoleMessages?.()).toEqual([]);
});
test("attaches the debugger before sending a CDP command", async () => {
const contents = new FakeWebContents(22);
const tab = adaptWebContents(contents);
const result = await tab.sendDebugCommand?.("Page.captureScreenshot", {
format: "png",
});
expect(result).toEqual({ ok: true });
expect(contents.debugger.attachedProtocolVersions).toEqual(["1.3"]);
expect(contents.debugger.commands).toEqual([
{ command: "Page.captureScreenshot", params: { format: "png" } },
]);
});
test("serializes CDP commands per guest contents", async () => {
const contents = new FakeWebContents(23);
contents.debugger.blockCommands = true;
const tab = adaptWebContents(contents);
const first = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mouseMoved" });
const second = tab.sendDebugCommand?.("Page.captureScreenshot", { format: "png" });
await flushMicrotasks();
expect(contents.debugger.commands).toEqual([
{ command: "Input.dispatchMouseEvent", params: { type: "mouseMoved" } },
]);
contents.debugger.finishNextCommand();
await flushMicrotasks();
expect(contents.debugger.commands).toEqual([
{ command: "Input.dispatchMouseEvent", params: { type: "mouseMoved" } },
{ command: "Page.captureScreenshot", params: { format: "png" } },
]);
contents.debugger.finishNextCommand();
await expect(first).resolves.toEqual({ ok: true });
await expect(second).resolves.toEqual({ ok: true });
});
test("handles JavaScript dialogs through the per-tab CDP queue", async () => {
const contents = new FakeWebContents(24);
const tab = adaptWebContents(contents);
const captured = tab.captureDialogs?.(async () => {
contents.debugger.blockCommands = true;
const input = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mouseReleased" });
await flushMicrotasks();
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
type: "confirm",
message: "Delete item?",
});
await flushMicrotasks();
expect(contents.debugger.commands).toEqual([
{ command: "Page.enable", params: {} },
{
command: "Runtime.evaluate",
params: { expression: expect.any(String), returnByValue: true },
},
{ command: "Input.dispatchMouseEvent", params: { type: "mouseReleased" } },
{ command: "Page.handleJavaScriptDialog", params: { accept: false } },
]);
contents.debugger.blockCommands = false;
contents.debugger.finishNextCommand();
await input;
await flushMicrotasks();
return "done";
});
await expect(captured).resolves.toEqual({
result: "done",
dialogs: [
{
type: "confirm",
message: "Delete item?",
action: "dismissed",
timestamp: expect.any(Number),
},
],
});
expect(contents.debugger.commands).toEqual([
{ command: "Page.enable", params: {} },
{
command: "Runtime.evaluate",
params: { expression: expect.any(String), returnByValue: true },
},
{ command: "Input.dispatchMouseEvent", params: { type: "mouseReleased" } },
{ command: "Page.handleJavaScriptDialog", params: { accept: false } },
{
command: "Runtime.evaluate",
params: { expression: expect.any(String), returnByValue: true },
},
{
command: "Runtime.evaluate",
params: { expression: expect.any(String), returnByValue: true },
},
]);
});
test("handles JavaScript dialogs while the triggering CDP input command is still in flight", async () => {
const contents = new FakeWebContents(25);
const tab = adaptWebContents(contents);
const captured = tab.captureDialogs?.(async () => {
contents.debugger.blockedCommandNames.add("Input.dispatchMouseEvent");
const input = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mousePressed" });
await flushMicrotasks();
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
type: "alert",
message: "Saved",
});
await flushMicrotasks();
expect(contents.debugger.commands).toEqual([
{ command: "Page.enable", params: {} },
{
command: "Runtime.evaluate",
params: { expression: expect.any(String), returnByValue: true },
},
{ command: "Input.dispatchMouseEvent", params: { type: "mousePressed" } },
{ command: "Page.handleJavaScriptDialog", params: { accept: true } },
]);
contents.debugger.blockedCommandNames.clear();
contents.debugger.finishNextCommand();
await input;
return "done";
});
await expect(captured).resolves.toEqual({
result: "done",
dialogs: [
{
type: "alert",
message: "Saved",
action: "accepted",
timestamp: expect.any(Number),
},
],
});
expect(contents.debugger.commands.at(-1)).toEqual({
command: "Runtime.evaluate",
params: {
expression: expect.stringContaining("delete window[stateKey]"),
returnByValue: true,
},
});
});
test("keeps the prompt shim installed until overlapping captures finish", async () => {
const contents = new FakeWebContents(26);
const tab = adaptWebContents(contents);
const firstStarted = deferred<void>();
const secondStarted = deferred<void>();
const finishFirst = deferred<void>();
const finishSecond = deferred<void>();
const first = tab.captureDialogs?.(async () => {
firstStarted.resolve();
await finishFirst.promise;
return "first";
});
await firstStarted.promise;
const second = tab.captureDialogs?.(async () => {
secondStarted.resolve();
await finishSecond.promise;
return "second";
});
await secondStarted.promise;
contents.debugger.promptDialogs.push(
{
type: "prompt",
message: "First?",
defaultValue: "one",
action: "dismissed",
timestamp: 1,
},
{
type: "prompt",
message: "Second?",
defaultValue: "two",
action: "dismissed",
timestamp: 2,
},
);
finishFirst.resolve();
await expect(first).resolves.toEqual({
result: "first",
dialogs: [
{
type: "prompt",
message: "First?",
defaultValue: "one",
action: "dismissed",
timestamp: 1,
},
{
type: "prompt",
message: "Second?",
defaultValue: "two",
action: "dismissed",
timestamp: 2,
},
],
});
describe("browser automation IPC", () => {
it("strips directories from agent-supplied download filenames", () => {
expect(
contents.debugger.commands.some(
(entry) =>
entry.command === "Runtime.evaluate" &&
typeof entry.params.expression === "string" &&
entry.params.expression.includes("delete window[stateKey]"),
),
).toBe(false);
finishSecond.resolve();
await expect(second).resolves.toEqual({
result: "second",
dialogs: [
{
type: "prompt",
message: "First?",
defaultValue: "one",
action: "dismissed",
timestamp: 1,
},
{
type: "prompt",
message: "Second?",
defaultValue: "two",
action: "dismissed",
timestamp: 2,
},
],
});
expect(contents.debugger.commands.at(-1)).toEqual({
command: "Runtime.evaluate",
params: {
expression: expect.stringContaining("delete window[stateKey]"),
returnByValue: true,
},
});
sanitizeDownloadFileName({
url: "https://example.com/fallback.txt",
fileName: "../../.ssh/authorized_keys",
}),
).toBe("authorized_keys");
});
test("leaves JavaScript dialogs alone when no capture is active", async () => {
const contents = new FakeWebContents(28);
const tab = adaptWebContents(contents);
await expect(tab.captureDialogs?.(async () => "done")).resolves.toEqual({
result: "done",
dialogs: [],
});
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
type: "confirm",
message: "Unsaved changes?",
});
await flushMicrotasks();
expect(contents.debugger.commands).not.toContainEqual({
command: "Page.handleJavaScriptDialog",
params: { accept: false },
});
});
test("treats prompt shim drain failures after navigation as no dialogs", async () => {
const contents = new FakeWebContents(29);
contents.debugger.failPromptDrain = true;
const tab = adaptWebContents(contents);
await expect(tab.captureDialogs?.(async () => "navigated")).resolves.toEqual({
result: "navigated",
dialogs: [],
});
expect(contents.debugger.commands).toEqual([
{ command: "Page.enable", params: {} },
{
command: "Runtime.evaluate",
params: { expression: expect.any(String), returnByValue: true },
},
{
command: "Runtime.evaluate",
params: { expression: expect.any(String), returnByValue: true },
},
{
command: "Runtime.evaluate",
params: {
expression: expect.stringContaining("delete window[stateKey]"),
returnByValue: true,
},
},
]);
});
test("runs the command without dialog capture when CDP setup fails", async () => {
const contents = new FakeWebContents(30);
contents.debugger.failedCommandNames.add("Page.enable");
const tab = adaptWebContents(contents);
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
await expect(tab.captureDialogs?.(async () => "done")).resolves.toEqual({
result: "done",
dialogs: [],
});
expect(warn).toHaveBeenCalledWith(
"[browser-automation] Dialog capture unavailable; running command without it",
{ contentsId: 30, error: expect.any(Error) },
);
warn.mockRestore();
it("falls back to a safe filename when the URL has no basename", () => {
expect(sanitizeDownloadFileName({ url: "https://example.com/" })).toBe("download");
});
});
async function flushMicrotasks(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
await new Promise((resolve) => setTimeout(resolve, 0));
}
function deferred<T>() {
let resolve!: (value: T | PromiseLike<T>) => void;
const promise = new Promise<T>((resolvePromise) => {
resolve = resolvePromise;
});
return { promise, resolve };
}
@@ -1,85 +1,35 @@
import type { Rectangle } from "electron";
import { mkdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import type { WebContents } from "electron";
import { ipcMain } from "electron";
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type {
BrowserAutomationConsoleLogEntry,
BrowserAutomationDialogEvent,
BrowserAutomationCookieEntry,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
import { CdpSessionQueue } from "./cdp-session-queue.js";
import {
dialogAcceptValue,
handledDialogEvent,
MAX_DIALOGS_PER_COMMAND,
promptShimDrainScript,
promptShimInstallScript,
promptShimRestoreScript,
} from "./dialog-handling.js";
import type { TabContents, BrowserRegistry } from "./service.js";
import { executeAutomationCommand } from "./service.js";
import {
listRegisteredPaseoBrowserIds,
listRegisteredPaseoBrowserIdsForWorkspace,
getPaseoBrowserWebContents,
getWorkspaceActivePaseoBrowserWebContents,
getWorkspaceActivePaseoBrowserId,
getAgentActivePaseoBrowserId,
getPaseoBrowserWorkspaceId,
} from "../browser-webviews/index.js";
const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
const cdpQueuesByContentsId = new Map<number, CdpSessionQueue>();
const dialogMonitorsByContentsId = new Map<number, DialogMonitor>();
const observedContentsIds = new Set<number>();
interface IpcHandlerRegistry {
handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
}
interface WebContentsDebugger {
isAttached(): boolean;
attach(protocolVersion?: string): void;
sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown>;
on?(
event: "message",
listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
): void;
}
interface ConsoleMessageEmitter {
on(
event: "console-message",
listener: (
event: unknown,
level: unknown,
message: unknown,
line: unknown,
sourceId: unknown,
) => void,
): void;
once(event: "destroyed", listener: () => void): void;
}
interface BrowserAutomationWebContents extends ConsoleMessageEmitter {
readonly id: number;
readonly debugger: WebContentsDebugger;
getURL(): string;
getTitle(): string;
canGoBack(): boolean;
canGoForward(): boolean;
isLoading(): boolean;
isDestroyed(): boolean;
executeJavaScript(code: string): Promise<unknown>;
loadURL(url: string): Promise<void>;
goBack(): void;
goForward(): void;
reload(): void;
capturePage(rect?: Rectangle, options?: { stayHidden?: boolean }): Promise<TabImage>;
invalidate(): void;
}
export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents {
function adaptWebContents(contents: WebContents): TabContents {
observeConsoleMessages(contents);
const cdpQueue = getCdpQueue(contents.id);
const dialogMonitor = getDialogMonitor(contents, cdpQueue);
return {
id: contents.id,
getURL: () => contents.getURL(),
@@ -93,31 +43,67 @@ export function adaptWebContents(contents: BrowserAutomationWebContents): TabCon
goBack: () => contents.goBack(),
goForward: () => contents.goForward(),
reload: () => contents.reload(),
capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
invalidate: () => contents.invalidate(),
capturePage: () => contents.capturePage(),
getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
captureDialogs: (task) => dialogMonitor.capture(task),
sendDebugCommand: (command: string, params?: Record<string, unknown>) =>
cdpQueue.run(async () => {
if (!contents.debugger.isAttached()) {
contents.debugger.attach("1.3");
}
return contents.debugger.sendCommand(command, params ?? {});
}),
getCookies: async (url: string) =>
(await contents.session.cookies.get({ url })).map(normalizeCookie),
sendDebugCommand: async (command: string, params?: Record<string, unknown>) => {
if (!contents.debugger.isAttached()) {
contents.debugger.attach("1.3");
}
return contents.debugger.sendCommand(command, params ?? {});
},
printToPDF: async (options?: Record<string, unknown>) => contents.printToPDF(options ?? {}),
downloadURL: (input) => downloadWithContents(contents, input),
};
}
function getCdpQueue(contentsId: number): CdpSessionQueue {
const existing = cdpQueuesByContentsId.get(contentsId);
if (existing) {
return existing;
}
const queue = new CdpSessionQueue();
cdpQueuesByContentsId.set(contentsId, queue);
return queue;
function downloadWithContents(
contents: WebContents,
input: { url: string; fileName?: string },
): Promise<{ filePath: string; totalBytes?: number; state: string }> {
const downloadDir = join(tmpdir(), "paseo-browser-downloads");
mkdirSync(downloadDir, { recursive: true });
const filePath = join(downloadDir, sanitizeDownloadFileName(input));
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
contents.session.off("will-download", onDownload);
reject(new Error(`Timed out waiting for browser download: ${input.url}`));
}, 30_000);
function onDownload(_event: Electron.Event, item: Electron.DownloadItem): void {
if (item.getURL() !== input.url) {
return;
}
clearTimeout(timeout);
contents.session.off("will-download", onDownload);
item.setSavePath(filePath);
item.once("done", (_doneEvent, state) => {
resolve({ filePath, totalBytes: item.getTotalBytes(), state });
});
}
contents.session.on("will-download", onDownload);
contents.downloadURL(input.url);
});
}
function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
export function sanitizeDownloadFileName(input: { url: string; fileName?: string }): string {
const requestedName = input.fileName ?? basename(new URL(input.url).pathname);
return basename(requestedName) || "download";
}
function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry {
return {
name: cookie.name,
value: cookie.value,
...(cookie.domain ? { domain: cookie.domain } : {}),
...(cookie.path ? { path: cookie.path } : {}),
secure: cookie.secure,
httpOnly: cookie.httpOnly,
...(typeof cookie.expirationDate === "number" ? { expirationDate: cookie.expirationDate } : {}),
};
}
function observeConsoleMessages(contents: WebContents): void {
if (observedContentsIds.has(contents.id)) {
return;
}
@@ -131,192 +117,6 @@ function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
contents.once("destroyed", () => {
observedContentsIds.delete(contents.id);
consoleMessagesByContentsId.delete(contents.id);
cdpQueuesByContentsId.delete(contents.id);
dialogMonitorsByContentsId.delete(contents.id);
});
}
function getDialogMonitor(
contents: BrowserAutomationWebContents,
cdpQueue: CdpSessionQueue,
): DialogMonitor {
const existing = dialogMonitorsByContentsId.get(contents.id);
if (existing) {
return existing;
}
const monitor = new DialogMonitor(contents, cdpQueue);
dialogMonitorsByContentsId.set(contents.id, monitor);
return monitor;
}
class DialogMonitor {
private enabled = false;
private listenerRegistered = false;
private readonly activeCollectors: DialogCollector[] = [];
public constructor(
private readonly contents: BrowserAutomationWebContents,
private readonly cdpQueue: CdpSessionQueue,
) {}
public async capture<T>(
task: () => Promise<T>,
): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> {
const collector: DialogCollector = { dialogs: [] };
try {
await this.enable();
await this.installPromptShim();
} catch (error) {
console.warn("[browser-automation] Dialog capture unavailable; running command without it", {
contentsId: this.contents.id,
error,
});
return { result: await task(), dialogs: [] };
}
this.activeCollectors.push(collector);
try {
const result = await task();
this.recordPromptShimDialogs(await this.drainPromptShim());
return { result, dialogs: collector.dialogs };
} finally {
const index = this.activeCollectors.indexOf(collector);
if (index >= 0) {
this.activeCollectors.splice(index, 1);
}
if (this.activeCollectors.length === 0) {
await this.restorePromptShim();
}
}
}
private async enable(): Promise<void> {
if (this.enabled) {
return;
}
if (!this.contents.debugger.on) {
return;
}
if (!this.listenerRegistered) {
this.listenerRegistered = true;
this.contents.debugger.on("message", (_event, method, params) => {
if (method !== "Page.javascriptDialogOpening") {
return;
}
if (this.activeCollectors.length === 0) {
return;
}
void this.handleOpening(params ?? {});
});
}
await this.sendDebugCommand("Page.enable");
this.enabled = true;
}
private async handleOpening(params: Record<string, unknown>): Promise<void> {
const event = handledDialogEvent(params);
for (const collector of this.activeCollectors) {
this.recordDialogs(collector, [event]);
}
await this.sendDialogResponseCommand("Page.handleJavaScriptDialog", {
accept: dialogAcceptValue(event.type),
});
}
private async installPromptShim(): Promise<void> {
await this.sendDebugCommand("Runtime.evaluate", {
expression: promptShimInstallScript(),
returnByValue: true,
});
}
private async drainPromptShim(): Promise<BrowserAutomationDialogEvent[]> {
try {
const result = (await this.sendDebugCommand("Runtime.evaluate", {
expression: promptShimDrainScript(),
returnByValue: true,
})) as { result?: { value?: unknown } };
return parsePromptShimDialogs(result.result?.value);
} catch {
return [];
}
}
private async restorePromptShim(): Promise<void> {
try {
await this.sendDebugCommand("Runtime.evaluate", {
expression: promptShimRestoreScript(),
returnByValue: true,
});
} catch {
// Navigation can destroy the execution context before cleanup runs; the next page has no shim.
}
}
private recordDialogs(collector: DialogCollector, dialogs: BrowserAutomationDialogEvent[]): void {
for (const dialog of dialogs) {
if (collector.dialogs.length >= MAX_DIALOGS_PER_COMMAND) {
return;
}
collector.dialogs.push(dialog);
}
}
private recordPromptShimDialogs(dialogs: BrowserAutomationDialogEvent[]): void {
for (const collector of this.activeCollectors) {
this.recordDialogs(collector, dialogs);
}
}
private async sendDebugCommand(
command: string,
params?: Record<string, unknown>,
): Promise<unknown> {
return this.cdpQueue.run(async () => {
if (!this.contents.debugger.isAttached()) {
this.contents.debugger.attach("1.3");
}
return this.contents.debugger.sendCommand(command, params ?? {});
});
}
private async sendDialogResponseCommand(
command: string,
params?: Record<string, unknown>,
): Promise<unknown> {
// Dialogs can block the CDP command that opened them, so the unblocker must not wait behind
// the per-tab command queue.
if (!this.contents.debugger.isAttached()) {
this.contents.debugger.attach("1.3");
}
return this.contents.debugger.sendCommand(command, params ?? {});
}
}
interface DialogCollector {
dialogs: BrowserAutomationDialogEvent[];
}
function parsePromptShimDialogs(value: unknown): BrowserAutomationDialogEvent[] {
if (!Array.isArray(value)) {
return [];
}
return value.flatMap((entry): BrowserAutomationDialogEvent[] => {
if (!entry || typeof entry !== "object") {
return [];
}
const record = entry as Record<string, unknown>;
if (record.type !== "prompt" || record.action !== "dismissed") {
return [];
}
return [
{
type: "prompt",
message: typeof record.message === "string" ? record.message : "",
...(typeof record.defaultValue === "string" ? { defaultValue: record.defaultValue } : {}),
action: "dismissed",
timestamp: typeof record.timestamp === "number" ? record.timestamp : Date.now(),
},
];
});
}
@@ -346,7 +146,12 @@ function createRegistry(): BrowserRegistry {
return contents ? adaptWebContents(contents) : null;
},
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
getWorkspaceActiveTabContents(workspaceId: string): TabContents | null {
const contents = getWorkspaceActivePaseoBrowserWebContents(workspaceId);
return contents ? adaptWebContents(contents) : null;
},
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,
getAgentActiveBrowserId: getAgentActivePaseoBrowserId,
};
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -3,162 +3,77 @@ import { BrowserSnapshotEngine, type SnapshotPage } from "./snapshot-engine.js";
class SnapshotFixture implements SnapshotPage {
public currentUrl = "https://example.com/form";
public actionResult: unknown = { ok: true };
public alreadyTruncated = false;
public snapshotNodes: unknown[] = [
{
kind: "role",
role: "heading",
name: "Settings",
tagName: "h1",
attributes: ["level=1"],
children: [],
},
{ kind: "text", text: "Connected as Maya" },
{
kind: "role",
role: "button",
name: "Save changes",
tagName: "button",
attributes: [],
ref: "@e1",
fingerprint: {
role: "button",
name: "Save changes",
tagName: "button",
type: "",
ariaLabel: "",
},
children: [],
},
];
public actionResult: unknown = true;
public getURL(): string {
return this.currentUrl;
}
public async executeJavaScript(code: string): Promise<unknown> {
if (code.includes("__PASEO_ARIA_SNAPSHOT__")) {
return JSON.stringify({
marker: "__PASEO_ARIA_SNAPSHOT__",
root: {
kind: "role",
role: "document",
name: "Fixture",
tagName: "document",
attributes: [],
children: this.snapshotNodes,
if (code.includes("CANDIDATE_SELECTOR")) {
return JSON.stringify([
{
role: "textbox",
tagName: "input",
text: "Name",
selector: "#name",
attributes: { id: "name", type: "text" },
},
refs: [
{
ref: "@e1",
fingerprint: {
role: "button",
name: "Save changes",
tagName: "button",
type: "",
ariaLabel: "",
},
},
],
truncated: this.alreadyTruncated,
stats: { nodeCount: 4, refCount: 1, textLength: 0, iframeCount: 0, maxDepth: 1 },
});
{
role: "button",
tagName: "button",
text: "Drop",
selector: "#drop",
attributes: { id: "drop" },
},
]);
}
return this.actionResult;
}
}
describe("BrowserSnapshotEngine", () => {
it("renders a hierarchical ARIA YAML snapshot with static text and actionable refs", async () => {
const page = new SnapshotFixture();
const engine = new BrowserSnapshotEngine();
await expect(engine.snapshot({ browserId: "browser-1", page })).resolves.toEqual({
format: "aria-yaml",
snapshot: [
'- document "Fixture"',
' - heading "Settings" [level=1]',
' - text: "Connected as Maya"',
' - button "Save changes" [ref=@e1]',
].join("\n"),
truncated: false,
stats: { nodeCount: 4, refCount: 1, textLength: 119, iframeCount: 0, maxDepth: 1 },
});
});
it("builds a runtime ref expression with the snapshot fingerprint", async () => {
it("treats a false result from a ref action script as a stale ref", async () => {
const page = new SnapshotFixture();
const engine = new BrowserSnapshotEngine();
await engine.snapshot({ browserId: "browser-1", page });
page.currentUrl = "https://example.com/form?panel=advanced";
page.actionResult = false;
expect(engine.runtimeElementExpression({ browserId: "browser-1", ref: "@e1" })).toContain(
'"name":"Save changes"',
);
});
it("treats missing host-side ref metadata as a stale ref", async () => {
const page = new SnapshotFixture();
const engine = new BrowserSnapshotEngine();
await engine.snapshot({ browserId: "browser-1", page });
expect(engine.runtimeElementExpression({ browserId: "browser-1", ref: "@e2" })).toEqual({
await expect(engine.click({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
ok: false,
reason: "missing_ref",
reason: "stale_ref",
});
await expect(engine.focus({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
ok: false,
reason: "stale_ref",
});
});
it("marks rendered output truncation explicitly and deterministically", async () => {
it("treats a false result from optional ref text/key actions as a stale ref", async () => {
const page = new SnapshotFixture();
page.snapshotNodes = [
{
kind: "text",
text: "A".repeat(81_000),
},
];
const engine = new BrowserSnapshotEngine();
await engine.snapshot({ browserId: "browser-1", page });
const snapshot = await engine.snapshot({ browserId: "browser-1", page });
page.actionResult = false;
expect(snapshot.truncated).toBe(true);
expect(snapshot.snapshot.endsWith('- text: "Snapshot truncated."')).toBe(true);
expect(snapshot.stats.textLength).toBeLessThanOrEqual(80_000);
await expect(
engine.typeText({ browserId: "browser-1", page, ref: "@e1", text: "Ada" }),
).resolves.toEqual({ ok: false, reason: "stale_ref" });
await expect(
engine.keypress({ browserId: "browser-1", page, ref: "@e1", key: "Enter" }),
).resolves.toEqual({ ok: false, reason: "stale_ref" });
});
it("keeps the last rendered node when a short snapshot was capped by node count", async () => {
it("treats a false result from drag as a stale ref", async () => {
const page = new SnapshotFixture();
page.alreadyTruncated = true;
page.snapshotNodes = [
{
kind: "role",
role: "button",
name: "Final action",
tagName: "button",
attributes: [],
ref: "@e1",
fingerprint: {
role: "button",
name: "Final action",
tagName: "button",
type: "",
ariaLabel: "",
},
children: [],
},
];
const engine = new BrowserSnapshotEngine();
await engine.snapshot({ browserId: "browser-1", page });
const snapshot = await engine.snapshot({ browserId: "browser-1", page });
page.actionResult = false;
expect(snapshot.truncated).toBe(true);
expect(snapshot.snapshot).toBe(
[
'- document "Fixture"',
' - button "Final action" [ref=@e1]',
'- text: "Snapshot truncated."',
].join("\n"),
);
await expect(
engine.drag({ browserId: "browser-1", page, sourceRef: "@e1", targetRef: "@e2" }),
).resolves.toEqual({ ok: false, reason: "stale_ref" });
});
});
@@ -1,60 +1,24 @@
import { ARIA_SNAPSHOT_SCRIPT, ARIA_SNAPSHOT_SCRIPT_MARKER } from "./aria-snapshot-script.js";
export interface SnapshotPage {
getURL(): string;
executeJavaScript(code: string): Promise<unknown>;
}
export interface BrowserAriaSnapshot {
format: "aria-yaml";
snapshot: string;
truncated: boolean;
stats: BrowserAriaSnapshotStats;
}
export interface BrowserAriaSnapshotStats {
nodeCount: number;
refCount: number;
textLength: number;
iframeCount?: number;
maxDepth?: number;
}
interface SnapshotNode {
kind: "role" | "text" | "group";
role?: string;
name?: string;
text?: string;
tagName?: string;
attributes?: string[];
ref?: string;
fingerprint?: BrowserRefFingerprint;
children?: SnapshotNode[];
}
interface BrowserRefFingerprint {
role: string;
name: string;
tagName: string;
type: string;
ariaLabel: string;
}
interface RawAriaSnapshot {
marker: string;
root: SnapshotNode;
refs: BrowserRefMetadata[];
truncated: boolean;
stats: BrowserAriaSnapshotStats;
}
interface BrowserRefMetadata {
export interface BrowserSnapshotElement extends RawSnapshotElement {
ref: string;
fingerprint: BrowserRefFingerprint;
}
interface RawSnapshotElement {
role: string;
tagName: string;
text: string;
selector: string;
attributes: Record<string, string>;
}
interface BrowserRefState {
refs: Map<string, BrowserRefMetadata>;
nextRefNumber: number;
url: string;
refs: Map<string, RawSnapshotElement>;
}
export type BrowserRefActionResult =
@@ -62,31 +26,45 @@ export type BrowserRefActionResult =
| { ok: false; reason: "stale_ref" | "missing_ref" };
type BrowserRefFailure = Extract<BrowserRefActionResult, { ok: false }>;
type BrowserRefResolveResult = { ok: true; metadata: BrowserRefMetadata } | BrowserRefFailure;
const TRUNCATION_MARKER = '- text: "Snapshot truncated."';
const MAX_RENDERED_TEXT_LENGTH = 80_000;
type BrowserRefResolveResult = { ok: true; element: RawSnapshotElement } | BrowserRefFailure;
export class BrowserSnapshotEngine {
private readonly statesByBrowserId = new Map<string, BrowserRefState>();
async snapshot(input: { browserId: string; page: SnapshotPage }): Promise<BrowserAriaSnapshot> {
const rawSnapshot = parseAriaSnapshot(await input.page.executeJavaScript(ARIA_SNAPSHOT_SCRIPT));
const rendered = renderSnapshot(rawSnapshot.root);
const capped = capRenderedSnapshot(rendered, rawSnapshot.truncated);
this.statesByBrowserId.set(input.browserId, {
refs: new Map(rawSnapshot.refs.map((ref) => [ref.ref, ref])),
});
return {
format: "aria-yaml",
snapshot: capped.snapshot,
truncated: capped.truncated,
stats: {
...rawSnapshot.stats,
refCount: rawSnapshot.refs.length,
textLength: capped.snapshot.length,
},
async snapshot(input: {
browserId: string;
page: SnapshotPage;
}): Promise<BrowserSnapshotElement[]> {
const rawElements = parseRawSnapshotElements(
await input.page.executeJavaScript(SNAPSHOT_SCRIPT),
);
const state = {
nextRefNumber: 1,
url: input.page.getURL(),
refs: new Map<string, RawSnapshotElement>(),
};
const elements = rawElements.map((element) => {
const ref = `@e${state.nextRefNumber++}`;
state.refs.set(ref, element);
return {
ref,
role: element.role,
tagName: element.tagName,
text: element.text,
selector: element.selector,
attributes: element.attributes,
};
});
this.statesByBrowserId.set(input.browserId, state);
return elements;
}
async click(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildClickScript(selector));
}
async fill(input: {
@@ -95,7 +73,64 @@ export class BrowserSnapshotEngine {
ref: string;
value: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (ref) => buildFillScript(ref, input.value));
return this.runRefScript(input, (selector) => buildFillScript(selector, input.value));
}
async typeText(input: {
browserId: string;
page: SnapshotPage;
ref?: string;
text: string;
}): Promise<BrowserRefActionResult> {
const selector = this.resolveOptionalRef(input);
if (!selector.ok) {
return selector;
}
const result = await input.page.executeJavaScript(
buildTypeScript(selector.selector, input.text),
);
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
async keypress(input: {
browserId: string;
page: SnapshotPage;
ref?: string;
key: string;
}): Promise<BrowserRefActionResult> {
const selector = this.resolveOptionalRef(input);
if (!selector.ok) {
return selector;
}
const result = await input.page.executeJavaScript(
buildKeypressScript(selector.selector, input.key),
);
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
async focus(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildFocusScript(selector));
}
async clear(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildClearScript(selector));
}
async check(input: {
browserId: string;
page: SnapshotPage;
ref: string;
checked: boolean;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildCheckScript(selector, input.checked));
}
async select(input: {
@@ -104,214 +139,123 @@ export class BrowserSnapshotEngine {
ref: string;
value: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (ref) => buildSelectScript(ref, input.value));
return this.runRefScript(input, (selector) => buildSelectScript(selector, input.value));
}
async hover(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildHoverScript(selector));
}
async drag(input: {
browserId: string;
page: SnapshotPage;
sourceRef: string;
targetRef: string;
}): Promise<BrowserRefActionResult> {
const source = this.resolveRef({
browserId: input.browserId,
page: input.page,
ref: input.sourceRef,
});
if (!source.ok) {
return source;
}
const target = this.resolveRef({
browserId: input.browserId,
page: input.page,
ref: input.targetRef,
});
if (!target.ok) {
return target;
}
const result = await input.page.executeJavaScript(
buildDragScript(source.element.selector, target.element.selector),
);
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
clearBrowser(browserId: string): void {
this.statesByBrowserId.delete(browserId);
}
runtimeElementExpression(input: { browserId: string; ref: string }): string | BrowserRefFailure {
selectorForRef(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): { ok: true; selector: string } | BrowserRefFailure {
const resolved = this.resolveRef(input);
if (!resolved.ok) {
return resolved;
}
return buildRuntimeElementExpression(resolved.metadata);
return { ok: true, selector: resolved.element.selector };
}
private async runRefScript(
input: { browserId: string; page: SnapshotPage; ref: string },
buildScript: (ref: BrowserRefMetadata) => string,
buildScript: (selector: string) => string,
): Promise<BrowserRefActionResult> {
const resolved = this.resolveRef(input);
if (!resolved.ok) {
return resolved;
}
const result = await input.page.executeJavaScript(buildScript(resolved.metadata));
return readActionResult(result, true);
const result = await input.page.executeJavaScript(buildScript(resolved.element.selector));
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
private resolveRef(input: { browserId: string; ref: string }): BrowserRefResolveResult {
private resolveRef(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): BrowserRefResolveResult {
const state = this.statesByBrowserId.get(input.browserId);
if (!state) {
if (!state || state.url !== input.page.getURL()) {
return { ok: false, reason: "stale_ref" };
}
const metadata = state.refs.get(input.ref);
if (!metadata) {
const element = state.refs.get(input.ref);
if (!element) {
return { ok: false, reason: "missing_ref" };
}
return { ok: true, metadata };
return { ok: true, element };
}
}
function readActionResult(value: unknown, staleWhenFalse: boolean): BrowserRefActionResult {
if (value === false && staleWhenFalse) {
return { ok: false, reason: "stale_ref" };
}
if (isActionFailure(value)) {
return { ok: false, reason: value.reason };
}
return { ok: true };
}
function isActionFailure(value: unknown): value is BrowserRefFailure {
if (!value || typeof value !== "object") {
return false;
}
const reason = (value as Record<string, unknown>).reason;
return reason === "stale_ref" || reason === "missing_ref";
}
function parseAriaSnapshot(value: unknown): RawAriaSnapshot {
const parsed = typeof value === "string" ? JSON.parse(value) : value;
if (!parsed || typeof parsed !== "object") {
return emptySnapshot();
}
const record = parsed as Record<string, unknown>;
if (record.marker !== ARIA_SNAPSHOT_SCRIPT_MARKER) {
return emptySnapshot();
}
const root = parseSnapshotNode(record.root);
return {
marker: ARIA_SNAPSHOT_SCRIPT_MARKER,
root: root ?? emptySnapshot().root,
refs: parseRefs(record.refs),
truncated: record.truncated === true,
stats: parseStats(record.stats),
};
}
function emptySnapshot(): RawAriaSnapshot {
return {
marker: ARIA_SNAPSHOT_SCRIPT_MARKER,
root: { kind: "role", role: "document", name: "", tagName: "document", children: [] },
refs: [],
truncated: false,
stats: { nodeCount: 0, refCount: 0, textLength: 0, iframeCount: 0, maxDepth: 0 },
};
}
function parseSnapshotNode(value: unknown): SnapshotNode | null {
if (!value || typeof value !== "object") {
return null;
}
const record = value as Record<string, unknown>;
const kind = record.kind;
if (kind !== "role" && kind !== "text" && kind !== "group") {
return null;
}
return {
kind,
...(readString(record.role) ? { role: readString(record.role) ?? undefined } : {}),
...(readString(record.name) ? { name: readString(record.name) ?? undefined } : {}),
...(readString(record.text) ? { text: readString(record.text) ?? undefined } : {}),
...(readString(record.tagName) ? { tagName: readString(record.tagName) ?? undefined } : {}),
...(readString(record.ref) ? { ref: readString(record.ref) ?? undefined } : {}),
...(parseFingerprint(record.fingerprint)
? { fingerprint: parseFingerprint(record.fingerprint) ?? undefined }
: {}),
attributes: readStringArray(record.attributes),
children: Array.isArray(record.children)
? record.children.flatMap((child): SnapshotNode[] => {
const parsed = parseSnapshotNode(child);
return parsed ? [parsed] : [];
})
: [],
};
}
function parseRefs(value: unknown): BrowserRefMetadata[] {
if (!Array.isArray(value)) {
return [];
}
return value.flatMap((item): BrowserRefMetadata[] => {
if (!item || typeof item !== "object") {
return [];
private resolveOptionalRef(input: {
browserId: string;
page: SnapshotPage;
ref?: string;
}): { ok: true; selector: string | undefined } | BrowserRefFailure {
if (!input.ref) {
return { ok: true, selector: undefined };
}
const record = item as Record<string, unknown>;
const ref = readString(record.ref);
const fingerprint = parseFingerprint(record.fingerprint);
return ref && fingerprint ? [{ ref, fingerprint }] : [];
});
const resolved = this.resolveRef({
browserId: input.browserId,
page: input.page,
ref: input.ref,
});
if (!resolved.ok) {
return resolved;
}
return { ok: true, selector: resolved.element.selector };
}
}
function parseFingerprint(value: unknown): BrowserRefFingerprint | null {
if (!value || typeof value !== "object") {
return null;
}
const record = value as Record<string, unknown>;
const role = readString(record.role);
const name = readString(record.name);
const tagName = readString(record.tagName);
const type = readString(record.type);
const ariaLabel = readString(record.ariaLabel);
if (role === null || name === null || tagName === null || type === null || ariaLabel === null) {
return null;
}
return { role, name, tagName, type, ariaLabel };
}
function parseStats(value: unknown): BrowserAriaSnapshotStats {
if (!value || typeof value !== "object") {
return { nodeCount: 0, refCount: 0, textLength: 0 };
}
const record = value as Record<string, unknown>;
return {
nodeCount: readNumber(record.nodeCount) ?? 0,
refCount: readNumber(record.refCount) ?? 0,
textLength: readNumber(record.textLength) ?? 0,
...(readNumber(record.iframeCount) !== null
? { iframeCount: readNumber(record.iframeCount) ?? undefined }
: {}),
...(readNumber(record.maxDepth) !== null
? { maxDepth: readNumber(record.maxDepth) ?? undefined }
: {}),
};
}
function renderSnapshot(root: SnapshotNode): string {
const lines = renderNode(root, 0);
return lines.length > 0 ? lines.join("\n") : "- document";
}
function renderNode(node: SnapshotNode, depth: number): string[] {
if (node.kind === "group") {
return (node.children ?? []).flatMap((child) => renderNode(child, depth));
}
const indent = " ".repeat(depth);
if (node.kind === "text") {
return [`${indent}- text: ${JSON.stringify(node.text ?? "")}`];
}
const attrs = [...(node.attributes ?? [])];
if (node.ref) {
attrs.push(`ref=${node.ref}`);
}
const suffix = attrs.length > 0 ? ` [${attrs.join(" ")}]` : "";
const ownLine = `${indent}- ${node.role ?? "generic"}${node.name ? ` ${JSON.stringify(node.name)}` : ""}${suffix}`;
const childLines = (node.children ?? []).flatMap((child) => renderNode(child, depth + 1));
return [ownLine, ...childLines];
}
function capRenderedSnapshot(
snapshot: string,
alreadyTruncated: boolean,
): { snapshot: string; truncated: boolean } {
if (snapshot.length <= MAX_RENDERED_TEXT_LENGTH && !alreadyTruncated) {
return { snapshot, truncated: false };
}
if (snapshot.length + 1 + TRUNCATION_MARKER.length <= MAX_RENDERED_TEXT_LENGTH) {
return { snapshot: `${snapshot}\n${TRUNCATION_MARKER}`, truncated: true };
}
const availableLength = MAX_RENDERED_TEXT_LENGTH - TRUNCATION_MARKER.length - 1;
const capped = snapshot.slice(0, Math.max(0, availableLength)).replace(/\n[^\n]*$/, "");
return { snapshot: `${capped}\n${TRUNCATION_MARKER}`, truncated: true };
}
function buildFillScript(metadata: BrowserRefMetadata, value: string): string {
function buildClickScript(selector: string): string {
return String.raw`(() => {
const resolved = ${buildResolveExpression(metadata)};
if (!resolved.ok) return resolved;
const element = resolved.element;
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView({ block: 'center', inline: 'center' });
element.click();
return true;
})()`;
}
function buildFillScript(selector: string, value: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView({ block: 'center', inline: 'center' });
element.focus();
const nextValue = ${JSON.stringify(value)};
@@ -319,19 +263,102 @@ function buildFillScript(metadata: BrowserRefMetadata, value: string): string {
element.value = nextValue;
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return { ok: true };
return true;
}
element.textContent = nextValue;
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: nextValue }));
return { ok: true };
return true;
})()`;
}
function buildSelectScript(metadata: BrowserRefMetadata, value: string): string {
function buildTypeScript(selector: string | undefined, text: string): string {
return String.raw`(() => {
const resolved = ${buildResolveExpression(metadata)};
if (!resolved.ok) return resolved;
const element = resolved.element;
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
const text = ${JSON.stringify(text)};
if ('value' in element) {
element.value = String(element.value || '') + text;
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
element.textContent = String(element.textContent || '') + text;
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
return true;
})()`;
}
function buildKeypressScript(selector: string | undefined, key: string): string {
return String.raw`(() => {
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
if (!element) return false;
element.focus?.();
const key = ${JSON.stringify(key)};
const eventInit = { bubbles: true, cancelable: true, key };
element.dispatchEvent(new KeyboardEvent('keydown', eventInit));
element.dispatchEvent(new KeyboardEvent('keypress', eventInit));
element.dispatchEvent(new KeyboardEvent('keyup', eventInit));
return true;
})()`;
}
function buildFocusScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
return document.activeElement === element;
})()`;
}
function buildClearScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
if ('value' in element) {
element.value = '';
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
element.textContent = '';
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
return true;
})()`;
}
function buildCheckScript(selector: string, checked: boolean): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
const nextChecked = ${JSON.stringify(checked)};
if ('checked' in element) {
element.checked = nextChecked;
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
if (element.getAttribute('role') === 'checkbox' || element.getAttribute('role') === 'radio') {
element.setAttribute('aria-checked', String(nextChecked));
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
return false;
})()`;
}
function buildSelectScript(selector: string, value: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
const nextValue = ${JSON.stringify(value)};
@@ -339,34 +366,205 @@ function buildSelectScript(metadata: BrowserRefMetadata, value: string): string
element.value = nextValue;
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return { ok: true };
return true;
}
return false;
})()`;
}
function buildRuntimeElementExpression(metadata: BrowserRefMetadata): string {
function buildHoverScript(selector: string): string {
return String.raw`(() => {
const resolved = ${buildResolveExpression(metadata)};
if (!resolved.ok) return null;
return resolved.element;
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
const rect = element.getBoundingClientRect();
const eventInit = {
bubbles: true,
cancelable: true,
clientX: rect.left + rect.width / 2,
clientY: rect.top + rect.height / 2,
screenX: window.screenX + rect.left + rect.width / 2,
screenY: window.screenY + rect.top + rect.height / 2,
view: window,
};
element.dispatchEvent(new MouseEvent('mouseover', eventInit));
element.dispatchEvent(new MouseEvent('mouseenter', eventInit));
element.dispatchEvent(new MouseEvent('mousemove', eventInit));
return true;
})()`;
}
function buildResolveExpression(metadata: BrowserRefMetadata): string {
return `window.__PASEO_BROWSER_AUTOMATION__?.resolve(${JSON.stringify(metadata.ref)}, ${JSON.stringify(metadata.fingerprint)}) ?? { ok: false, reason: 'stale_ref' }`;
function buildDragScript(sourceSelector: string, targetSelector: string): string {
return String.raw`(() => {
const source = document.querySelector(${JSON.stringify(sourceSelector)});
const target = document.querySelector(${JSON.stringify(targetSelector)});
if (!source || !target) return false;
source.scrollIntoView?.({ block: 'center', inline: 'center' });
target.scrollIntoView?.({ block: 'center', inline: 'center' });
const data = new DataTransfer();
const sourceRect = source.getBoundingClientRect();
const targetRect = target.getBoundingClientRect();
function eventInit(rect) {
return {
bubbles: true,
cancelable: true,
clientX: rect.left + rect.width / 2,
clientY: rect.top + rect.height / 2,
screenX: window.screenX + rect.left + rect.width / 2,
screenY: window.screenY + rect.top + rect.height / 2,
dataTransfer: data,
view: window,
};
}
source.dispatchEvent(new MouseEvent('mousedown', eventInit(sourceRect)));
source.dispatchEvent(new DragEvent('dragstart', eventInit(sourceRect)));
target.dispatchEvent(new DragEvent('dragenter', eventInit(targetRect)));
target.dispatchEvent(new DragEvent('dragover', eventInit(targetRect)));
target.dispatchEvent(new DragEvent('drop', eventInit(targetRect)));
source.dispatchEvent(new DragEvent('dragend', eventInit(sourceRect)));
target.dispatchEvent(new MouseEvent('mouseup', eventInit(targetRect)));
return true;
})()`;
}
function parseRawSnapshotElements(value: unknown): RawSnapshotElement[] {
const parsed = typeof value === "string" ? JSON.parse(value) : value;
if (!Array.isArray(parsed)) {
return [];
}
return parsed.flatMap((item): RawSnapshotElement[] => {
if (!item || typeof item !== "object") {
return [];
}
const record = item as Record<string, unknown>;
const selector = readString(record.selector);
if (!selector) {
return [];
}
return [
{
role: readString(record.role) || "generic",
tagName: (readString(record.tagName) || "element").toLowerCase(),
text: readString(record.text) || "",
selector,
attributes: readAttributes(record.attributes),
},
];
});
}
function readString(value: unknown): string | null {
return typeof value === "string" ? value : null;
}
function readStringArray(value: unknown): string[] {
return Array.isArray(value)
? value.filter((item): item is string => typeof item === "string")
: [];
function readAttributes(value: unknown): Record<string, string> {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return {};
}
const result: Record<string, string> = {};
for (const [key, attributeValue] of Object.entries(value)) {
if (typeof attributeValue === "string") {
result[key] = attributeValue;
}
}
return result;
}
function readNumber(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
const SNAPSHOT_SCRIPT = String.raw`(() => {
const MAX_ELEMENTS = 200;
const CANDIDATE_SELECTOR = [
'a[href]',
'button',
'input',
'textarea',
'select',
'summary',
'[role]',
'[tabindex]:not([tabindex="-1"])',
'[contenteditable=""]',
'[contenteditable="true"]'
].join(',');
function cssEscape(value) {
if (window.CSS && typeof window.CSS.escape === 'function') {
return window.CSS.escape(value);
}
return String(value).replace(/[^a-zA-Z0-9_-]/g, '\\$&');
}
function isVisible(element) {
const style = window.getComputedStyle(element);
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
return false;
}
const rect = element.getBoundingClientRect();
return rect.width > 0 && rect.height > 0;
}
function roleFor(element) {
const explicit = element.getAttribute('role');
if (explicit) return explicit;
const tag = element.tagName.toLowerCase();
if (tag === 'a') return 'link';
if (tag === 'button') return 'button';
if (tag === 'select') return 'combobox';
if (tag === 'textarea') return 'textbox';
if (tag === 'summary') return 'button';
if (tag === 'input') {
const type = (element.getAttribute('type') || 'text').toLowerCase();
if (type === 'checkbox') return 'checkbox';
if (type === 'radio') return 'radio';
if (type === 'button' || type === 'submit' || type === 'reset') return 'button';
return 'textbox';
}
return 'generic';
}
function textFor(element) {
const tag = element.tagName.toLowerCase();
const pieces = [
element.getAttribute('aria-label'),
element.getAttribute('alt'),
element.getAttribute('title'),
tag === 'input' ? element.getAttribute('placeholder') : null,
tag === 'input' || tag === 'textarea' ? element.value : null,
element.innerText,
element.textContent
];
const text = pieces.find((piece) => typeof piece === 'string' && piece.trim().length > 0);
return (text || '').replace(/\s+/g, ' ').trim().slice(0, 300);
}
function selectorFor(element) {
if (element.id) return '#' + cssEscape(element.id);
const parts = [];
let current = element;
while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) {
const tag = current.tagName.toLowerCase();
const parent = current.parentElement;
if (!parent) break;
const siblings = Array.from(parent.children).filter((sibling) => sibling.tagName === current.tagName);
const index = siblings.indexOf(current) + 1;
parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag);
current = parent;
}
return parts.length > 0 ? parts.join(' > ') : element.tagName.toLowerCase();
}
return JSON.stringify(Array.from(document.querySelectorAll(CANDIDATE_SELECTOR))
.filter(isVisible)
.slice(0, MAX_ELEMENTS)
.map((element) => ({
role: roleFor(element),
tagName: element.tagName.toLowerCase(),
text: textFor(element),
selector: selectorFor(element),
attributes: {
...(element.id ? { id: element.id } : {}),
...(element.getAttribute('name') ? { name: element.getAttribute('name') } : {}),
...(element.getAttribute('type') ? { type: element.getAttribute('type') } : {}),
...(element.getAttribute('href') ? { href: element.getAttribute('href') } : {}),
...(element.getAttribute('aria-label') ? { 'aria-label': element.getAttribute('aria-label') } : {})
}
})));
})()`;
@@ -1,38 +0,0 @@
import { describe, expect, test } from "vitest";
import { dispatchTrustedKey } from "./trusted-input.js";
describe("trusted browser input", () => {
test("Space dispatches a real space key event", async () => {
const commands: Array<{ command: string; params?: Record<string, unknown> }> = [];
await dispatchTrustedKey(async (command, params) => {
commands.push({ command, ...(params ? { params } : {}) });
return {};
}, "Space");
expect(commands).toEqual([
{
command: "Input.dispatchKeyEvent",
params: {
type: "keyDown",
key: " ",
code: "Space",
windowsVirtualKeyCode: 32,
nativeVirtualKeyCode: 32,
text: " ",
unmodifiedText: " ",
},
},
{
command: "Input.dispatchKeyEvent",
params: {
type: "keyUp",
key: " ",
code: "Space",
windowsVirtualKeyCode: 32,
nativeVirtualKeyCode: 32,
},
},
]);
});
});
@@ -1,217 +0,0 @@
import type { ActionablePoint } from "./actionability.js";
import type { CdpCommandSender } from "./cdp-session-queue.js";
export type MouseButton = "left" | "right" | "middle";
export type InputModifier = "Alt" | "Control" | "Meta" | "Shift";
export interface ClickInputOptions {
button?: MouseButton;
doubleClick?: boolean;
modifiers?: InputModifier[];
}
const MODIFIER_MASKS: Record<InputModifier, number> = {
Alt: 1,
Control: 2,
Meta: 4,
Shift: 8,
};
const SPECIAL_KEY_DEFINITIONS: Record<
string,
{ key: string; code: string; windowsVirtualKeyCode: number; text?: string }
> = {
Enter: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13, text: "\r" },
Space: { key: " ", code: "Space", windowsVirtualKeyCode: 32, text: " " },
Tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9, text: "\t" },
Escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
Backspace: { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 },
Delete: { key: "Delete", code: "Delete", windowsVirtualKeyCode: 46 },
ArrowUp: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
ArrowDown: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
ArrowLeft: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
ArrowRight: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
Home: { key: "Home", code: "Home", windowsVirtualKeyCode: 36 },
End: { key: "End", code: "End", windowsVirtualKeyCode: 35 },
PageUp: { key: "PageUp", code: "PageUp", windowsVirtualKeyCode: 33 },
PageDown: { key: "PageDown", code: "PageDown", windowsVirtualKeyCode: 34 },
};
export async function dispatchTrustedClick(
send: CdpCommandSender,
point: ActionablePoint,
options: ClickInputOptions = {},
): Promise<void> {
const button = options.button ?? "left";
const modifiers = modifierMask(options.modifiers);
await send("Input.dispatchMouseEvent", {
type: "mouseMoved",
x: point.x,
y: point.y,
button: "none",
modifiers,
});
if (options.doubleClick) {
await dispatchTrustedMouseClick(send, point, button, modifiers, 1);
await dispatchTrustedMouseClick(send, point, button, modifiers, 2);
return;
}
await dispatchTrustedMouseClick(send, point, button, modifiers, 1);
}
async function dispatchTrustedMouseClick(
send: CdpCommandSender,
point: ActionablePoint,
button: MouseButton,
modifiers: number,
clickCount: number,
): Promise<void> {
await send("Input.dispatchMouseEvent", {
type: "mousePressed",
x: point.x,
y: point.y,
button,
buttons: mouseButtonMask(button),
clickCount,
modifiers,
});
await send("Input.dispatchMouseEvent", {
type: "mouseReleased",
x: point.x,
y: point.y,
button,
buttons: 0,
clickCount,
modifiers,
});
}
export async function dispatchTrustedHover(
send: CdpCommandSender,
point: ActionablePoint,
): Promise<void> {
await send("Input.dispatchMouseEvent", {
type: "mouseMoved",
x: point.x,
y: point.y,
button: "none",
});
}
export async function dispatchTrustedDrag(
send: CdpCommandSender,
source: ActionablePoint,
target: ActionablePoint,
): Promise<void> {
await send("Input.dispatchMouseEvent", {
type: "mouseMoved",
x: source.x,
y: source.y,
button: "none",
});
await send("Input.dispatchMouseEvent", {
type: "mousePressed",
x: source.x,
y: source.y,
button: "left",
buttons: 1,
clickCount: 1,
});
await send("Input.dispatchMouseEvent", {
type: "mouseMoved",
x: (source.x + target.x) / 2,
y: (source.y + target.y) / 2,
button: "left",
buttons: 1,
});
await send("Input.dispatchMouseEvent", {
type: "mouseMoved",
x: target.x,
y: target.y,
button: "left",
buttons: 1,
});
await send("Input.dispatchMouseEvent", {
type: "mouseReleased",
x: target.x,
y: target.y,
button: "left",
buttons: 0,
clickCount: 1,
});
}
export async function dispatchTrustedScroll(
send: CdpCommandSender,
point: ActionablePoint,
deltaX: number,
deltaY: number,
): Promise<void> {
await send("Input.dispatchMouseEvent", {
type: "mouseWheel",
x: point.x,
y: point.y,
deltaX,
deltaY,
});
}
export async function dispatchTrustedText(send: CdpCommandSender, text: string): Promise<void> {
if (text.length === 0) {
return;
}
await send("Input.insertText", { text });
}
export async function dispatchTrustedKey(send: CdpCommandSender, key: string): Promise<void> {
const definition = keyDefinition(key);
await send("Input.dispatchKeyEvent", {
type: "keyDown",
key: definition.key,
code: definition.code,
windowsVirtualKeyCode: definition.windowsVirtualKeyCode,
nativeVirtualKeyCode: definition.windowsVirtualKeyCode,
...(definition.text ? { text: definition.text, unmodifiedText: definition.text } : {}),
});
await send("Input.dispatchKeyEvent", {
type: "keyUp",
key: definition.key,
code: definition.code,
windowsVirtualKeyCode: definition.windowsVirtualKeyCode,
nativeVirtualKeyCode: definition.windowsVirtualKeyCode,
});
}
function keyDefinition(key: string): {
key: string;
code: string;
windowsVirtualKeyCode: number;
text?: string;
} {
const special = SPECIAL_KEY_DEFINITIONS[key];
if (special) {
return special;
}
const text = key.length === 1 ? key : "";
const upper = text.toUpperCase();
return {
key,
code: upper ? `Key${upper}` : key,
windowsVirtualKeyCode: upper ? upper.charCodeAt(0) : 0,
...(text ? { text, unmodifiedText: text } : {}),
};
}
function modifierMask(modifiers: InputModifier[] | undefined): number {
return (modifiers ?? []).reduce((mask, modifier) => mask | MODIFIER_MASKS[modifier], 0);
}
function mouseButtonMask(button: MouseButton): number {
if (button === "right") {
return 2;
}
if (button === "middle") {
return 4;
}
return 1;
}
@@ -1,44 +0,0 @@
import { describe, expect, test } from "vitest";
import { getPaseoBrowserIdForWebContents, registerPaseoBrowserWebContents } from "./index.js";
class FakeRegisteredWebContents {
public readonly backgroundThrottlingCalls: boolean[] = [];
private destroyedListener: (() => void) | null = null;
private destroyed = false;
public constructor(public readonly id: number) {}
public isDestroyed(): boolean {
return this.destroyed;
}
public setBackgroundThrottling(allowed: boolean): void {
this.backgroundThrottlingCalls.push(allowed);
}
public once(event: "destroyed", listener: () => void): void {
expect(event).toBe("destroyed");
this.destroyedListener = listener;
}
public destroy(): void {
this.destroyed = true;
this.destroyedListener?.();
}
}
describe("registerPaseoBrowserWebContents", () => {
test("disables guest background throttling once when the webview is registered", () => {
const contents = new FakeRegisteredWebContents(9001);
registerPaseoBrowserWebContents(contents, "browser-throttle");
expect(contents.backgroundThrottlingCalls).toEqual([false]);
expect(getPaseoBrowserIdForWebContents(contents)).toBe("browser-throttle");
contents.destroy();
expect(getPaseoBrowserIdForWebContents(contents)).toBeNull();
expect(contents.backgroundThrottlingCalls).toEqual([false]);
});
});
@@ -11,16 +11,6 @@ export type { BrowserWorkspaceRegistration };
const browserRegistry = new PaseoBrowserWebviewRegistry();
interface BrowserWebContentsIdentity {
readonly id: number;
isDestroyed(): boolean;
}
interface RegisteredBrowserWebContents extends BrowserWebContentsIdentity {
setBackgroundThrottling(allowed: boolean): void;
once(event: "destroyed", listener: () => void): void;
}
function getBrowserIdFromWebviewPartition(partition: string | undefined): string | null {
const prefix = "persist:paseo-browser-";
if (!partition?.startsWith(prefix)) {
@@ -46,20 +36,14 @@ export function listRegisteredPaseoBrowserIds(): string[] {
.filter((browserId) => getPaseoBrowserWebContents(browserId));
}
export function registerPaseoBrowserWebContents(
contents: RegisteredBrowserWebContents,
browserId: string,
): void {
contents.setBackgroundThrottling(false);
export function registerPaseoBrowserWebContents(contents: WebContents, browserId: string): void {
browserRegistry.registerWebContents({ webContentsId: contents.id, browserId });
contents.once("destroyed", () => {
browserRegistry.unregisterWebContents(contents.id);
});
}
export function getPaseoBrowserIdForWebContents(
contents: BrowserWebContentsIdentity | null,
): string | null {
export function getPaseoBrowserIdForWebContents(contents: WebContents | null): string | null {
if (!contents || contents.isDestroyed()) {
return null;
}
@@ -70,10 +54,6 @@ export function registerPaseoBrowserWorkspace(input: BrowserWorkspaceRegistratio
browserRegistry.registerWorkspace(input);
}
export function unregisterPaseoBrowser(browserId: string): void {
browserRegistry.unregisterBrowser(browserId);
}
export function getPaseoBrowserWorkspaceId(browserId: string): string | null {
return browserRegistry.getWorkspaceId(browserId);
}
@@ -95,6 +75,17 @@ export function getWorkspaceActivePaseoBrowserId(workspaceId: string): string |
return browserRegistry.getWorkspaceActiveBrowserId(workspaceId);
}
export function setAgentActivePaseoBrowserId(input: {
agentId: string;
browserId: string | null;
}): void {
browserRegistry.setAgentActiveBrowser(input);
}
export function getAgentActivePaseoBrowserId(agentId: string): string | null {
return browserRegistry.getAgentActiveBrowserId(agentId);
}
export function getPaseoBrowserWebContents(browserId: string): WebContents | null {
const contentsId = browserRegistry.getWebContentsIdForBrowser(browserId);
if (contentsId === null) {
@@ -108,6 +99,16 @@ export function getPaseoBrowserWebContents(browserId: string): WebContents | nul
return null;
}
export function getWorkspaceActivePaseoBrowserWebContents(workspaceId: string): WebContents | null {
const activeBrowserId = getWorkspaceActivePaseoBrowserId(workspaceId);
return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null;
}
export function getAgentActivePaseoBrowserWebContents(agentId: string): WebContents | null {
const activeBrowserId = getAgentActivePaseoBrowserId(agentId);
return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null;
}
export function getMostRecentWorkspaceActivePaseoBrowserWebContents(): WebContents | null {
const browserId = browserRegistry.getMostRecentWorkspaceActiveBrowserId();
return browserId ? getPaseoBrowserWebContents(browserId) : null;
@@ -8,6 +8,7 @@ describe("PaseoBrowserWebviewRegistry", () => {
registry.registerWebContents({ webContentsId: 1, browserId: "browser-a" });
registry.registerWorkspace({ browserId: "browser-a", workspaceId: "workspace-a" });
registry.setWorkspaceActiveBrowser({ workspaceId: "workspace-a", browserId: "browser-a" });
registry.setAgentActiveBrowser({ agentId: "agent-a", browserId: "browser-a" });
registry.registerWebContents({ webContentsId: 2, browserId: "browser-a" });
expect(registry.getBrowserIdForWebContents(1)).toBeNull();
@@ -15,6 +16,7 @@ describe("PaseoBrowserWebviewRegistry", () => {
expect(registry.getWebContentsIdForBrowser("browser-a")).toBe(2);
expect(registry.getWorkspaceId("browser-a")).toBe("workspace-a");
expect(registry.getWorkspaceActiveBrowserId("workspace-a")).toBe("browser-a");
expect(registry.getAgentActiveBrowserId("agent-a")).toBe("browser-a");
});
it("ignores stale destroy events after a duplicate browserId moved", () => {
@@ -8,6 +8,7 @@ export class PaseoBrowserWebviewRegistry {
private readonly webContentsIdsByBrowserId = new Map<string, number>();
private readonly workspaceIdsByBrowserId = new Map<string, string>();
private readonly activeBrowserIdsByWorkspaceId = new Map<string, string>();
private readonly activeBrowserIdsByAgentId = new Map<string, string>();
public registerWebContents(input: { webContentsId: number; browserId: string }): void {
const previousWebContentsId = this.webContentsIdsByBrowserId.get(input.browserId) ?? null;
@@ -51,16 +52,6 @@ export class PaseoBrowserWebviewRegistry {
this.workspaceIdsByBrowserId.set(input.browserId, input.workspaceId);
}
public unregisterBrowser(browserId: string): void {
const webContentsId = this.webContentsIdsByBrowserId.get(browserId) ?? null;
if (webContentsId !== null) {
this.browserIdsByWebContentsId.delete(webContentsId);
this.webContentsIdsByBrowserId.delete(browserId);
}
this.workspaceIdsByBrowserId.delete(browserId);
this.deleteActiveBrowserReferences(browserId);
}
public getWorkspaceId(browserId: string): string | null {
return this.workspaceIdsByBrowserId.get(browserId) ?? null;
}
@@ -89,11 +80,29 @@ export class PaseoBrowserWebviewRegistry {
return Array.from(this.activeBrowserIdsByWorkspaceId.values()).at(-1) ?? null;
}
public setAgentActiveBrowser(input: { agentId: string; browserId: string | null }): void {
if (input.browserId) {
this.activeBrowserIdsByAgentId.delete(input.agentId);
this.activeBrowserIdsByAgentId.set(input.agentId, input.browserId);
return;
}
this.activeBrowserIdsByAgentId.delete(input.agentId);
}
public getAgentActiveBrowserId(agentId: string): string | null {
return this.activeBrowserIdsByAgentId.get(agentId) ?? null;
}
private deleteActiveBrowserReferences(browserId: string): void {
for (const [workspaceId, activeBrowserId] of this.activeBrowserIdsByWorkspaceId) {
if (activeBrowserId === browserId) {
this.activeBrowserIdsByWorkspaceId.delete(workspaceId);
}
}
for (const [agentId, activeBrowserId] of this.activeBrowserIdsByAgentId) {
if (activeBrowserId === browserId) {
this.activeBrowserIdsByAgentId.delete(agentId);
}
}
}
}
+22 -7
View File
@@ -54,9 +54,9 @@ import {
listRegisteredPaseoBrowserIds,
readBrowserIdFromWebviewAttach,
registerBrowserWebviewNavigationGuards,
unregisterPaseoBrowser,
registerPaseoBrowserWorkspace,
registerPaseoBrowserWebContents,
setAgentActivePaseoBrowserId,
setWorkspaceActivePaseoBrowserId,
} from "./features/browser-webviews/index.js";
import { parseOpenProjectPathFromArgv } from "./open-project-routing.js";
@@ -144,6 +144,20 @@ function readActiveBrowserInput(
return { workspaceId: record.workspaceId.trim(), browserId: browserId || null };
}
function readAgentActiveBrowserInput(
input: unknown,
): { agentId: string; browserId: string | null } | null {
if (typeof input !== "object" || input === null || Array.isArray(input)) {
return null;
}
const record = input as Record<string, unknown>;
if (typeof record.agentId !== "string" || record.agentId.trim().length === 0) {
return null;
}
const browserId = typeof record.browserId === "string" ? record.browserId.trim() : null;
return { agentId: record.agentId.trim(), browserId: browserId || null };
}
const pendingBrowserWebviewIds: string[] = [];
function isBrowserRefreshInput(input: Electron.Input): boolean {
@@ -329,12 +343,6 @@ ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: un
}
});
ipcMain.handle("paseo:browser:unregister-workspace-browser", (_event, browserId: unknown) => {
if (typeof browserId === "string" && browserId.trim().length > 0) {
unregisterPaseoBrowser(browserId.trim());
}
});
ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput: unknown) => {
const input = readActiveBrowserInput(rawInput);
if (input) {
@@ -342,6 +350,13 @@ ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput:
}
});
ipcMain.handle("paseo:browser:set-agent-active-browser", (_event, rawInput: unknown) => {
const input = readAgentActiveBrowserInput(rawInput);
if (input) {
setAgentActivePaseoBrowserId(input);
}
});
ipcMain.handle("paseo:browser:open-devtools", (_event, browserId: unknown) => {
if (typeof browserId !== "string" || browserId.trim().length === 0) {
const result = {
+2 -2
View File
@@ -76,10 +76,10 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
browser: {
registerWorkspaceBrowser: (input: { browserId: string; workspaceId: string }) =>
ipcRenderer.invoke("paseo:browser:register-workspace-browser", input),
unregisterWorkspaceBrowser: (browserId: string) =>
ipcRenderer.invoke("paseo:browser:unregister-workspace-browser", browserId),
setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) =>
ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input),
setAgentActiveBrowser: (input: { agentId: string; browserId: string | null }) =>
ipcRenderer.invoke("paseo:browser:set-agent-active-browser", input),
openDevTools: (browserId: string) =>
ipcRenderer.invoke("paseo:browser:open-devtools", browserId),
clearPartition: (browserId: string) =>
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/expo-two-way-audio",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"description": "Native module for two way audio streaming",
"keywords": [
"ExpoTwoWayAudio",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/highlight",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"files": [
"dist",
"!dist/**/*.map"
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/protocol",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"description": "Paseo shared protocol schemas and wire types",
"files": [
"dist",
@@ -1,41 +0,0 @@
import { z } from "zod";
import {
BROWSER_AUTOMATION_COMMAND_NAMES,
type BrowserAutomationCommandName,
} from "./rpc-schemas.js";
const KNOWN_BROWSER_AUTOMATION_COMMAND_NAMES = new Set<string>(BROWSER_AUTOMATION_COMMAND_NAMES);
export const BrowserAutomationHostCapabilitySchema = z
.object({
supportedCommands: z.array(z.string().min(1)).transform((commands, context) => {
const supportedCommands: BrowserAutomationCommandName[] = [];
const seen = new Set<BrowserAutomationCommandName>();
for (const command of commands) {
if (!isKnownBrowserAutomationCommandName(command) || seen.has(command)) {
continue;
}
seen.add(command);
supportedCommands.push(command);
}
if (supportedCommands.length === 0) {
context.addIssue({
code: "custom",
message: "supportedCommands must include at least one known browser automation command",
});
return z.NEVER;
}
return supportedCommands;
}),
hostKind: z.string().min(1).default("browser host"),
})
.passthrough();
export type BrowserAutomationHostCapability = z.infer<typeof BrowserAutomationHostCapabilitySchema>;
function isKnownBrowserAutomationCommandName(value: string): value is BrowserAutomationCommandName {
return KNOWN_BROWSER_AUTOMATION_COMMAND_NAMES.has(value);
}
File diff suppressed because it is too large Load Diff
@@ -2,65 +2,23 @@ import { z } from "zod";
export const BrowserAutomationErrorCodeSchema = z.enum([
"browser_disabled",
"browser_no_host",
"browser_no_desktop",
"browser_no_tab",
"browser_tab_not_found",
"browser_tab_closed",
"browser_timeout",
"screenshot_no_frame",
"browser_denied",
"browser_unsupported",
"browser_stale_ref",
"browser_unknown_error",
]);
const BROWSER_AUTOMATION_BROWSER_ID_PATTERN =
/^(?:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|\d{13,}-[0-9a-f]+)$/i;
const BROWSER_AUTOMATION_BROWSER_ID_MESSAGE =
"browserId must be a real id returned by browser_new_tab or browser_list_tabs";
const BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE =
"browser_wait requires exactly one of text or url";
export const BROWSER_AUTOMATION_COMMAND_NAMES = [
"list_tabs",
"new_tab",
"snapshot",
"click",
"fill",
"wait",
"type",
"keypress",
"navigate",
"back",
"forward",
"reload",
"screenshot",
"upload",
"select",
"hover",
"drag",
"logs",
"evaluate",
"scroll",
"resize",
"close_tab",
] as const;
export const BrowserAutomationCommandNameSchema = z.enum(BROWSER_AUTOMATION_COMMAND_NAMES);
export const BrowserAutomationBrowserIdSchema = z
.string({ error: () => BROWSER_AUTOMATION_BROWSER_ID_MESSAGE })
.min(1, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE)
.regex(BROWSER_AUTOMATION_BROWSER_ID_PATTERN, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE);
const BrowserAutomationTabTargetSchema = z
.object({
browserId: BrowserAutomationBrowserIdSchema,
})
.strict();
const BrowserAutomationTabTargetSchema = z.object({
workspaceId: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
});
const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/);
const BrowserAutomationMouseButtonSchema = z.enum(["left", "right", "middle"]);
const BrowserAutomationInputModifierSchema = z.enum(["Alt", "Control", "Meta", "Shift"]);
const BrowserAutomationHttpUrlSchema = z
.string()
.url()
@@ -71,31 +29,37 @@ const BrowserAutomationHttpUrlSchema = z
export const BrowserAutomationListTabsCommandSchema = z.object({
command: z.literal("list_tabs"),
args: z.object({}).strict().default({}),
args: z
.object({
workspaceId: z.string().min(1).optional(),
})
.default({}),
});
export const BrowserAutomationNewTabCommandSchema = z.object({
command: z.literal("new_tab"),
args: z
.object({
workspaceId: z.string().min(1).optional(),
url: BrowserAutomationHttpUrlSchema.optional(),
})
.strict()
.default({}),
});
export const BrowserAutomationPageInfoCommandSchema = z.object({
command: z.literal("page_info"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationSnapshotCommandSchema = z.object({
command: z.literal("snapshot"),
args: BrowserAutomationTabTargetSchema,
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationClickCommandSchema = z.object({
command: z.literal("click"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
button: BrowserAutomationMouseButtonSchema.default("left"),
doubleClick: z.boolean().default(false),
modifiers: z.array(BrowserAutomationInputModifierSchema).default([]),
}),
});
@@ -113,8 +77,6 @@ export const BrowserAutomationWaitCommandSchema = z.object({
text: z.string().min(1).optional(),
url: z.string().min(1).optional(),
timeoutMs: z.number().int().positive().max(30_000).optional(),
}).refine((args) => Number(Boolean(args.text)) + Number(Boolean(args.url)) === 1, {
message: BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE,
}),
});
@@ -143,23 +105,42 @@ export const BrowserAutomationNavigateCommandSchema = z.object({
export const BrowserAutomationBackCommandSchema = z.object({
command: z.literal("back"),
args: BrowserAutomationTabTargetSchema,
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationForwardCommandSchema = z.object({
command: z.literal("forward"),
args: BrowserAutomationTabTargetSchema,
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationReloadCommandSchema = z.object({
command: z.literal("reload"),
args: BrowserAutomationTabTargetSchema,
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationScreenshotCommandSchema = z.object({
command: z.literal("screenshot"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({
command: z.literal("full_page_screenshot"),
args: BrowserAutomationTabTargetSchema.default({}),
});
export const BrowserAutomationPdfCommandSchema = z.object({
command: z.literal("pdf"),
args: BrowserAutomationTabTargetSchema.extend({
fullPage: z.boolean().default(false),
landscape: z.boolean().optional(),
printBackground: z.boolean().default(true),
}).default({ printBackground: true }),
});
export const BrowserAutomationDownloadCommandSchema = z.object({
command: z.literal("download"),
args: BrowserAutomationTabTargetSchema.extend({
url: BrowserAutomationHttpUrlSchema,
fileName: z.string().min(1).optional(),
}),
});
@@ -171,6 +152,28 @@ export const BrowserAutomationUploadCommandSchema = z.object({
}),
});
export const BrowserAutomationFocusCommandSchema = z.object({
command: z.literal("focus"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationClearCommandSchema = z.object({
command: z.literal("clear"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
}),
});
export const BrowserAutomationCheckCommandSchema = z.object({
command: z.literal("check"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema,
checked: z.boolean().default(true),
}),
});
export const BrowserAutomationSelectCommandSchema = z.object({
command: z.literal("select"),
args: BrowserAutomationTabTargetSchema.extend({
@@ -198,42 +201,45 @@ export const BrowserAutomationLogsCommandSchema = z.object({
command: z.literal("logs"),
args: BrowserAutomationTabTargetSchema.extend({
maxEntries: z.number().int().positive().max(200).default(50),
}),
}).default({ maxEntries: 50 }),
});
export const BrowserAutomationEvaluateCommandSchema = z.object({
command: z.literal("evaluate"),
export const BrowserAutomationStorageCommandSchema = z.object({
command: z.literal("storage"),
args: BrowserAutomationTabTargetSchema.default({}),
});
const BrowserAutomationViewportInputSchema = z.object({
width: z.number().int().positive(),
height: z.number().int().positive(),
deviceScaleFactor: z.number().positive().optional(),
});
const BrowserAutomationGeolocationInputSchema = z.object({
latitude: z.number().min(-90).max(90),
longitude: z.number().min(-180).max(180),
accuracy: z.number().positive().optional(),
});
export const BrowserAutomationEnvironmentCommandSchema = z.object({
command: z.literal("environment"),
args: BrowserAutomationTabTargetSchema.extend({
function: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
}),
viewport: BrowserAutomationViewportInputSchema.optional(),
geolocation: BrowserAutomationGeolocationInputSchema.optional(),
}).default({}),
});
export const BrowserAutomationScrollCommandSchema = z.object({
command: z.literal("scroll"),
export const BrowserAutomationSetBackgroundCommandSchema = z.object({
command: z.literal("set_background"),
args: BrowserAutomationTabTargetSchema.extend({
ref: BrowserAutomationRefSchema.optional(),
deltaX: z.number(),
deltaY: z.number(),
color: z.string().min(1),
}),
});
export const BrowserAutomationResizeCommandSchema = z.object({
command: z.literal("resize"),
args: BrowserAutomationTabTargetSchema.extend({
width: z.number().int().positive(),
height: z.number().int().positive(),
}),
});
export const BrowserAutomationCloseTabCommandSchema = z.object({
command: z.literal("close_tab"),
args: BrowserAutomationTabTargetSchema,
});
export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsCommandSchema,
BrowserAutomationNewTabCommandSchema,
BrowserAutomationPageInfoCommandSchema,
BrowserAutomationSnapshotCommandSchema,
BrowserAutomationClickCommandSchema,
BrowserAutomationFillCommandSchema,
@@ -245,19 +251,24 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
BrowserAutomationForwardCommandSchema,
BrowserAutomationReloadCommandSchema,
BrowserAutomationScreenshotCommandSchema,
BrowserAutomationFullPageScreenshotCommandSchema,
BrowserAutomationPdfCommandSchema,
BrowserAutomationDownloadCommandSchema,
BrowserAutomationUploadCommandSchema,
BrowserAutomationFocusCommandSchema,
BrowserAutomationClearCommandSchema,
BrowserAutomationCheckCommandSchema,
BrowserAutomationSelectCommandSchema,
BrowserAutomationHoverCommandSchema,
BrowserAutomationDragCommandSchema,
BrowserAutomationLogsCommandSchema,
BrowserAutomationEvaluateCommandSchema,
BrowserAutomationScrollCommandSchema,
BrowserAutomationResizeCommandSchema,
BrowserAutomationCloseTabCommandSchema,
BrowserAutomationStorageCommandSchema,
BrowserAutomationEnvironmentCommandSchema,
BrowserAutomationSetBackgroundCommandSchema,
]);
export const BrowserAutomationTabInfoSchema = z.object({
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
@@ -274,131 +285,164 @@ export const BrowserAutomationListTabsResultSchema = z.object({
export const BrowserAutomationNewTabResultSchema = z.object({
command: z.literal("new_tab"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
workspaceId: z.string().min(1),
url: z.string().min(1),
});
export const BrowserAutomationSnapshotStatsSchema = z
.object({
nodeCount: z.number().int().nonnegative(),
refCount: z.number().int().nonnegative(),
textLength: z.number().int().nonnegative(),
iframeCount: z.number().int().nonnegative().optional(),
maxDepth: z.number().int().nonnegative().optional(),
})
.strict();
export const BrowserAutomationPageInfoResultSchema = z.object({
command: z.literal("page_info"),
tab: BrowserAutomationTabInfoSchema,
});
export const BrowserAutomationSnapshotElementSchema = z.object({
ref: z.string().regex(/^@e\d+$/),
role: z.string(),
tagName: z.string(),
text: z.string(),
selector: z.string(),
attributes: z.record(z.string(), z.string()).default({}),
});
export const BrowserAutomationSnapshotResultSchema = z.object({
command: z.literal("snapshot"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
workspaceId: z.string().min(1).optional(),
url: z.string(),
title: z.string(),
format: z.literal("aria-yaml"),
snapshot: z.string(),
truncated: z.boolean(),
stats: BrowserAutomationSnapshotStatsSchema,
elements: z.array(BrowserAutomationSnapshotElementSchema),
});
export const BrowserAutomationClickResultSchema = z.object({
command: z.literal("click"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationFillResultSchema = z.object({
command: z.literal("fill"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationWaitResultSchema = z.object({
command: z.literal("wait"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
matched: z.enum(["text", "url"]),
});
export const BrowserAutomationTypeResultSchema = z.object({
command: z.literal("type"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationKeypressResultSchema = z.object({
command: z.literal("keypress"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
key: z.string().min(1),
ref: BrowserAutomationRefSchema.optional(),
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationNavigateResultSchema = z.object({
command: z.literal("navigate"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
url: z.string().min(1),
});
export const BrowserAutomationBackResultSchema = z.object({
command: z.literal("back"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
});
export const BrowserAutomationForwardResultSchema = z.object({
command: z.literal("forward"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
});
export const BrowserAutomationReloadResultSchema = z.object({
command: z.literal("reload"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
});
export const BrowserAutomationScreenshotResultSchema = z.object({
command: z.literal("screenshot"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
});
export const BrowserAutomationFullPageScreenshotResultSchema = z.object({
command: z.literal("full_page_screenshot"),
browserId: z.string().min(1),
mimeType: z.literal("image/png"),
dataBase64: z.string().min(1),
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
});
export const BrowserAutomationPdfResultSchema = z.object({
command: z.literal("pdf"),
browserId: z.string().min(1),
mimeType: z.literal("application/pdf"),
dataBase64: z.string().min(1),
});
export const BrowserAutomationDownloadResultSchema = z.object({
command: z.literal("download"),
browserId: z.string().min(1),
url: z.string().min(1),
filePath: z.string().min(1),
totalBytes: z.number().int().nonnegative().optional(),
state: z.string().min(1),
});
export const BrowserAutomationUploadResultSchema = z.object({
command: z.literal("upload"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
filePaths: z.array(z.string().min(1)).min(1),
});
export const BrowserAutomationFocusResultSchema = z.object({
command: z.literal("focus"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationClearResultSchema = z.object({
command: z.literal("clear"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
});
export const BrowserAutomationCheckResultSchema = z.object({
command: z.literal("check"),
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
checked: z.boolean(),
});
export const BrowserAutomationSelectResultSchema = z.object({
command: z.literal("select"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
value: z.string(),
});
export const BrowserAutomationHoverResultSchema = z.object({
command: z.literal("hover"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
ref: BrowserAutomationRefSchema,
x: z.number().optional(),
y: z.number().optional(),
});
export const BrowserAutomationDragResultSchema = z.object({
command: z.literal("drag"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
sourceRef: BrowserAutomationRefSchema,
targetRef: BrowserAutomationRefSchema,
sourceX: z.number().optional(),
sourceY: z.number().optional(),
targetX: z.number().optional(),
targetY: z.number().optional(),
});
export const BrowserAutomationConsoleLogEntrySchema = z.object({
@@ -421,43 +465,64 @@ export const BrowserAutomationNetworkLogEntrySchema = z.object({
export const BrowserAutomationLogsResultSchema = z.object({
command: z.literal("logs"),
browserId: BrowserAutomationBrowserIdSchema,
browserId: z.string().min(1),
console: z.array(BrowserAutomationConsoleLogEntrySchema),
network: z.array(BrowserAutomationNetworkLogEntrySchema),
});
export const BrowserAutomationEvaluateResultSchema = z.object({
command: z.literal("evaluate"),
browserId: BrowserAutomationBrowserIdSchema,
resultJson: z.string(),
truncated: z.boolean(),
export const BrowserAutomationCookieEntrySchema = z.object({
name: z.string(),
value: z.string(),
domain: z.string().optional(),
path: z.string().optional(),
secure: z.boolean().optional(),
httpOnly: z.boolean().optional(),
expirationDate: z.number().optional(),
});
export const BrowserAutomationScrollResultSchema = z.object({
command: z.literal("scroll"),
browserId: BrowserAutomationBrowserIdSchema,
ref: BrowserAutomationRefSchema.optional(),
deltaX: z.number(),
deltaY: z.number(),
x: z.number().optional(),
y: z.number().optional(),
export const BrowserAutomationStorageEntrySchema = z.object({
key: z.string(),
value: z.string(),
});
export const BrowserAutomationResizeResultSchema = z.object({
command: z.literal("resize"),
browserId: BrowserAutomationBrowserIdSchema,
width: z.number().int().positive(),
height: z.number().int().positive(),
export const BrowserAutomationStorageResultSchema = z.object({
command: z.literal("storage"),
browserId: z.string().min(1),
url: z.string(),
cookies: z.array(BrowserAutomationCookieEntrySchema),
localStorage: z.array(BrowserAutomationStorageEntrySchema),
sessionStorage: z.array(BrowserAutomationStorageEntrySchema),
});
export const BrowserAutomationCloseTabResultSchema = z.object({
command: z.literal("close_tab"),
browserId: BrowserAutomationBrowserIdSchema,
export const BrowserAutomationViewportResultSchema = z.object({
width: z.number().int().nonnegative(),
height: z.number().int().nonnegative(),
deviceScaleFactor: z.number().positive(),
});
export const BrowserAutomationGeolocationResultSchema = z.object({
latitude: z.number(),
longitude: z.number(),
accuracy: z.number(),
});
export const BrowserAutomationEnvironmentResultSchema = z.object({
command: z.literal("environment"),
browserId: z.string().min(1),
viewport: BrowserAutomationViewportResultSchema,
geolocation: BrowserAutomationGeolocationResultSchema.optional(),
});
export const BrowserAutomationSetBackgroundResultSchema = z.object({
command: z.literal("set_background"),
browserId: z.string().min(1),
color: z.string().min(1),
});
export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
BrowserAutomationListTabsResultSchema,
BrowserAutomationNewTabResultSchema,
BrowserAutomationPageInfoResultSchema,
BrowserAutomationSnapshotResultSchema,
BrowserAutomationClickResultSchema,
BrowserAutomationFillResultSchema,
@@ -469,15 +534,20 @@ export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
BrowserAutomationForwardResultSchema,
BrowserAutomationReloadResultSchema,
BrowserAutomationScreenshotResultSchema,
BrowserAutomationFullPageScreenshotResultSchema,
BrowserAutomationPdfResultSchema,
BrowserAutomationDownloadResultSchema,
BrowserAutomationUploadResultSchema,
BrowserAutomationFocusResultSchema,
BrowserAutomationClearResultSchema,
BrowserAutomationCheckResultSchema,
BrowserAutomationSelectResultSchema,
BrowserAutomationHoverResultSchema,
BrowserAutomationDragResultSchema,
BrowserAutomationLogsResultSchema,
BrowserAutomationEvaluateResultSchema,
BrowserAutomationScrollResultSchema,
BrowserAutomationResizeResultSchema,
BrowserAutomationCloseTabResultSchema,
BrowserAutomationStorageResultSchema,
BrowserAutomationEnvironmentResultSchema,
BrowserAutomationSetBackgroundResultSchema,
]);
export const BrowserAutomationErrorSchema = z.object({
@@ -486,26 +556,16 @@ export const BrowserAutomationErrorSchema = z.object({
retryable: z.boolean().default(false),
});
export const BrowserAutomationDialogEventSchema = z.object({
type: z.enum(["alert", "confirm", "prompt", "beforeunload"]),
message: z.string(),
defaultValue: z.string().optional(),
action: z.enum(["accepted", "dismissed"]),
promptText: z.string().optional(),
timestamp: z.number(),
export const BrowserAutomationExecuteRequestSchema = z.object({
type: z.literal("browser.automation.execute.request"),
requestId: z.string().min(1),
agentId: z.string().min(1).optional(),
cwd: z.string().min(1).optional(),
workspaceId: z.string().min(1).optional(),
browserId: z.string().min(1).optional(),
command: BrowserAutomationCommandSchema,
});
export const BrowserAutomationExecuteRequestSchema = z
.object({
type: z.literal("browser.automation.execute.request"),
requestId: z.string().min(1),
agentId: z.string().min(1).optional(),
cwd: z.string().min(1).optional(),
workspaceId: z.string().min(1).optional(),
command: BrowserAutomationCommandSchema,
})
.strict();
export const BrowserAutomationExecuteResponseSchema = z.object({
type: z.literal("browser.automation.execute.response"),
payload: z.discriminatedUnion("ok", [
@@ -513,19 +573,16 @@ export const BrowserAutomationExecuteResponseSchema = z.object({
requestId: z.string().min(1),
ok: z.literal(true),
result: BrowserAutomationResultSchema,
dialogs: z.array(BrowserAutomationDialogEventSchema).optional(),
}),
z.object({
requestId: z.string().min(1),
ok: z.literal(false),
error: BrowserAutomationErrorSchema,
dialogs: z.array(BrowserAutomationDialogEventSchema).optional(),
}),
]),
});
export type BrowserAutomationErrorCode = z.infer<typeof BrowserAutomationErrorCodeSchema>;
export type BrowserAutomationCommandName = z.infer<typeof BrowserAutomationCommandNameSchema>;
export type BrowserAutomationCommand = z.infer<typeof BrowserAutomationCommandSchema>;
export type BrowserAutomationResult = z.infer<typeof BrowserAutomationResultSchema>;
export type BrowserAutomationConsoleLogEntry = z.infer<
@@ -534,7 +591,8 @@ export type BrowserAutomationConsoleLogEntry = z.infer<
export type BrowserAutomationNetworkLogEntry = z.infer<
typeof BrowserAutomationNetworkLogEntrySchema
>;
export type BrowserAutomationDialogEvent = z.infer<typeof BrowserAutomationDialogEventSchema>;
export type BrowserAutomationCookieEntry = z.infer<typeof BrowserAutomationCookieEntrySchema>;
export type BrowserAutomationStorageEntry = z.infer<typeof BrowserAutomationStorageEntrySchema>;
export type BrowserAutomationExecuteRequest = z.infer<typeof BrowserAutomationExecuteRequestSchema>;
export type BrowserAutomationExecuteResponse = z.infer<
typeof BrowserAutomationExecuteResponseSchema
+1 -1
View File
@@ -11,7 +11,7 @@ export const CLIENT_CAPS = {
// Old clients use a strict TerminalState schema and would reject the extra fields.
// Drop the gate (always send the flags) when floor >= v0.1.88.
terminalReflowableSnapshot: "terminal_reflowable_snapshot",
browserHost: "browser_host",
desktopBrowserAutomation: "desktop_browser_automation",
} as const;
export type ClientCapability = (typeof CLIENT_CAPS)[keyof typeof CLIENT_CAPS];
@@ -1,6 +1,5 @@
import { describe, expect, test } from "vitest";
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "./browser-automation/rpc-schemas.js";
import { CLIENT_CAPS } from "./client-capabilities.js";
import {
MutableDaemonConfigPatchSchema,
@@ -11,9 +10,7 @@ import {
} from "./messages.js";
describe("browser automation protocol integration", () => {
const browserId = "11111111-1111-4111-8111-111111111111";
test("browser host capability parses supported commands in hello", () => {
test("desktop automation capability parses in hello without narrowing old clients", () => {
expect(
WSHelloMessageSchema.parse({
type: "hello",
@@ -21,108 +18,13 @@ describe("browser automation protocol integration", () => {
clientType: "mobile",
protocolVersion: 1,
capabilities: {
[CLIENT_CAPS.browserHost]: {
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
hostKind: "desktop app",
},
[CLIENT_CAPS.desktopBrowserAutomation]: true,
},
}).capabilities,
).toMatchObject({
[CLIENT_CAPS.browserHost]: {
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
hostKind: "desktop app",
},
[CLIENT_CAPS.desktopBrowserAutomation]: true,
});
});
test("browser host capability requires at least one supported command", () => {
expect(() =>
WSHelloMessageSchema.parse({
type: "hello",
clientId: "client-1",
clientType: "mobile",
protocolVersion: 1,
capabilities: {
[CLIENT_CAPS.browserHost]: {
supportedCommands: [],
hostKind: "desktop app",
},
},
}),
).toThrow();
expect(() =>
WSHelloMessageSchema.parse({
type: "hello",
clientId: "client-2",
clientType: "mobile",
protocolVersion: 1,
capabilities: {
[CLIENT_CAPS.browserHost]: {},
},
}),
).toThrow();
expect(() =>
WSHelloMessageSchema.parse({
type: "hello",
clientId: "client-3",
clientType: "mobile",
protocolVersion: 1,
capabilities: {
[CLIENT_CAPS.browserHost]: {
supportedCommands: ["future_command"],
},
},
}),
).toThrow();
});
test("browser host capability ignores unknown future commands when known commands remain", () => {
expect(
WSHelloMessageSchema.parse({
type: "hello",
clientId: "client-1",
clientType: "mobile",
protocolVersion: 1,
capabilities: {
[CLIENT_CAPS.browserHost]: {
supportedCommands: ["list_tabs", "future_command", "list_tabs"],
hostKind: "desktop app",
},
},
}).capabilities,
).toMatchObject({
[CLIENT_CAPS.browserHost]: {
supportedCommands: ["list_tabs"],
hostKind: "desktop app",
},
});
});
test("browser host capability accepts new tool commands as supported commands", () => {
expect(
WSHelloMessageSchema.parse({
type: "hello",
clientId: "client-1",
clientType: "mobile",
protocolVersion: 1,
capabilities: {
[CLIENT_CAPS.browserHost]: {
supportedCommands: ["evaluate", "scroll", "resize", "close_tab"],
hostKind: "desktop app",
},
},
}).capabilities,
).toMatchObject({
[CLIENT_CAPS.browserHost]: {
supportedCommands: ["evaluate", "scroll", "resize", "close_tab"],
hostKind: "desktop app",
},
});
});
test("hello remains valid when no browser host capability is advertised", () => {
expect(
WSHelloMessageSchema.parse({
type: "hello",
@@ -133,17 +35,17 @@ describe("browser automation protocol integration", () => {
).toBeUndefined();
});
test("daemon to browser host execute request is an outbound session message", () => {
test("daemon to desktop execute request is an outbound session message", () => {
const parsed = SessionOutboundMessageSchema.parse({
type: "browser.automation.execute.request",
requestId: "req-1",
command: { command: "snapshot", args: { browserId } },
command: { command: "page_info", args: { browserId: "browser-1" } },
});
expect(parsed.type).toBe("browser.automation.execute.request");
});
test("browser host to daemon execute response is an inbound session message", () => {
test("desktop to daemon execute response is an inbound session message", () => {
const parsed = SessionInboundMessageSchema.parse({
type: "browser.automation.execute.response",
payload: {
+1 -2
View File
@@ -57,7 +57,6 @@ import {
BrowserAutomationExecuteRequestSchema,
BrowserAutomationExecuteResponseSchema,
} from "./browser-automation/rpc-schemas.js";
import { BrowserAutomationHostCapabilitySchema } from "./browser-automation/capabilities.js";
import {
PaseoConfigRawSchema,
PaseoLifecycleCommandRawSchema,
@@ -4673,7 +4672,7 @@ export const WSHelloMessageSchema = z.object({
[CLIENT_CAPS.reasoningMergeEnum]: z.boolean().optional(),
[CLIENT_CAPS.customModeIcons]: z.boolean().optional(),
[CLIENT_CAPS.terminalReflowableSnapshot]: z.boolean().optional(),
[CLIENT_CAPS.browserHost]: BrowserAutomationHostCapabilitySchema.optional(),
[CLIENT_CAPS.desktopBrowserAutomation]: z.boolean().optional(),
})
.passthrough()
.optional(),
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/relay",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"description": "Paseo relay for bridging daemon and client connections",
"files": [
"dist",
+5 -5
View File
@@ -1,6 +1,6 @@
{
"name": "@getpaseo/server",
"version": "0.1.104-beta.3",
"version": "0.1.103",
"description": "Paseo backend server",
"files": [
"dist/server",
@@ -65,10 +65,10 @@
"@agentclientprotocol/sdk": "^0.17.1",
"@anthropic-ai/claude-agent-sdk": "^0.3.195",
"@anthropic-ai/sdk": "^0.104.2",
"@getpaseo/client": "0.1.104-beta.3",
"@getpaseo/highlight": "0.1.104-beta.3",
"@getpaseo/protocol": "0.1.104-beta.3",
"@getpaseo/relay": "0.1.104-beta.3",
"@getpaseo/client": "0.1.103",
"@getpaseo/highlight": "0.1.103",
"@getpaseo/protocol": "0.1.103",
"@getpaseo/relay": "0.1.103",
"@isaacs/ttlcache": "^2.1.4",
"@modelcontextprotocol/sdk": "^1.20.1",
"@opencode-ai/sdk": "1.14.46",
@@ -47,10 +47,7 @@ interface CreateAgentCommandDependencies {
providerSnapshotManager: ProviderSnapshotManager;
createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn;
// Mints a fresh directory workspace for a cwd and returns its id.
ensureWorkspaceForCreate?: (
cwd: string,
firstAgentContext?: FirstAgentContext,
) => Promise<string>;
ensureWorkspaceForCreate?: (cwd: string) => Promise<string>;
}
export interface CreateAgentFromSessionInput {
@@ -256,7 +253,7 @@ async function resolveMcpCreateAgent(
? createdWorkspaceId
: (input.workspaceId ??
parentAgent?.workspaceId ??
(await ensureWorkspaceForMcpCreate(dependencies, resolvedCwd, input.initialPrompt)));
(await ensureWorkspaceForMcpCreate(dependencies, resolvedCwd)));
const { modeId: resolvedMode, featureValues: resolvedFeatures } =
await dependencies.providerSnapshotManager.resolveCreateConfig({
@@ -303,12 +300,11 @@ async function resolveMcpCreateAgent(
async function ensureWorkspaceForMcpCreate(
dependencies: CreateAgentCommandDependencies,
cwd: string,
initialPrompt: string,
): Promise<string | undefined> {
if (!dependencies.ensureWorkspaceForCreate) {
return undefined;
}
return dependencies.ensureWorkspaceForCreate(cwd, { prompt: initialPrompt });
return dependencies.ensureWorkspaceForCreate(cwd);
}
async function sendInitialPrompt(
@@ -428,7 +424,7 @@ async function resolveMcpCwd(params: {
refName: worktree.refName,
action: worktree.action,
githubPrNumber: worktree.githubPrNumber,
firstAgentContext: { prompt: params.initialPrompt },
...(params.initialPrompt ? { firstAgentContext: { prompt: params.initialPrompt } } : {}),
runSetup: false,
paseoHome: dependencies.paseoHome,
worktreesRoot: dependencies.worktreesRoot,
File diff suppressed because it is too large Load Diff
+36 -2
View File
@@ -5,6 +5,7 @@ import type {
ServerNotification,
ServerRequest,
} from "@modelcontextprotocol/sdk/types.js";
import { z } from "zod";
import { createPaseoToolCatalog, type PaseoToolHostDependencies } from "./tools/paseo-tools.js";
import type { PaseoToolResult } from "./tools/types.js";
@@ -13,6 +14,39 @@ export type AgentMcpServerOptions = PaseoToolHostDependencies;
type McpToolContext = RequestHandlerExtra<ServerRequest, ServerNotification>;
function isZodSchema(value: unknown): value is z.ZodType {
return (
typeof value === "object" &&
value !== null &&
typeof (value as { safeParseAsync?: unknown }).safeParseAsync === "function"
);
}
function relaxMcpOutputSchema(outputSchema: unknown): unknown {
if (!outputSchema) {
return outputSchema;
}
if (isZodSchema(outputSchema)) {
return outputSchema instanceof z.ZodObject ? outputSchema.passthrough() : outputSchema;
}
return z.object(outputSchema as z.ZodRawShape).passthrough();
}
function relaxMcpToolOutputSchema<TConfig extends { outputSchema?: unknown }>(
config: TConfig,
): TConfig {
if (config.outputSchema === undefined) {
return config;
}
return {
...config,
outputSchema: relaxMcpOutputSchema(config.outputSchema),
} as TConfig;
}
function formatStructuredContentForModel(structuredContent: unknown): string {
if (
!structuredContent ||
@@ -81,12 +115,12 @@ export async function createAgentMcpServer(options: AgentMcpServerOptions): Prom
for (const tool of catalog.tools.values()) {
server.registerTool(
tool.name,
{
relaxMcpToolOutputSchema({
title: tool.title,
description: tool.description,
inputSchema: tool.inputSchema,
outputSchema: tool.outputSchema,
},
}),
async (args: unknown, context?: McpToolContext) =>
toMcpToolResult(await catalog.executeTool(tool.name, args, { signal: context?.signal })),
);
@@ -438,7 +438,7 @@ describe("ClaudeAgentClient.fetchCatalog", () => {
}
});
test("exposes Ultra Code on xhigh-capable Claude models", async () => {
test("exposes Ultracode only on Claude models that support it", async () => {
const emptyConfigDir = await fs.mkdtemp(path.join(os.tmpdir(), "paseo-claude-models-empty-"));
try {
const client = new ClaudeAgentClient({
@@ -459,9 +459,8 @@ describe("ClaudeAgentClient.fetchCatalog", () => {
expect(getThinkingIds("claude-opus-4-8[1m]")).toContain("ultracode");
expect(getThinkingIds("claude-opus-4-8")).toContain("ultracode");
expect(getThinkingIds("claude-sonnet-5")).toContain("xhigh");
expect(getThinkingIds("claude-sonnet-5")).toContain("ultracode");
expect(getThinkingIds("claude-opus-4-7[1m]")).toContain("ultracode");
expect(getThinkingIds("claude-opus-4-7")).toContain("ultracode");
expect(getThinkingIds("claude-sonnet-5")).not.toContain("ultracode");
expect(getThinkingIds("claude-opus-4-7")).not.toContain("ultracode");
expect(getThinkingIds("claude-sonnet-4-6")).not.toContain("ultracode");
} finally {
await fs.rm(emptyConfigDir, { recursive: true, force: true });
@@ -598,30 +597,6 @@ describe("ClaudeAgentSession features", () => {
}),
).resolves.toEqual([expect.objectContaining({ id: "fast_mode", value: false })]);
await expect(
client.listFeatures({
provider: "claude",
cwd: process.cwd(),
model: "claude-opus-4-8[1m]",
}),
).resolves.toEqual([expect.objectContaining({ id: "fast_mode", value: false })]);
await expect(
client.listFeatures({
provider: "claude",
cwd: process.cwd(),
model: "claude-opus-4-8-20260101",
}),
).resolves.toEqual([expect.objectContaining({ id: "fast_mode", value: false })]);
await expect(
client.listFeatures({
provider: "claude",
cwd: process.cwd(),
model: "openrouter/anthropic/claude-opus-4-8",
}),
).resolves.toEqual([]);
await expect(
client.listFeatures({
provider: "claude",
@@ -34,7 +34,6 @@ import {
getClaudeModelsWithSettings,
normalizeClaudeRuntimeModelId,
} from "./models.js";
import { CLAUDE_ULTRACODE_THINKING_OPTION_ID } from "./model-manifest.js";
import { parsePartialJsonObject } from "./partial-json.js";
import { ClaudeSidechainTracker } from "./sidechain-tracker.js";
import { buildClaudeFeatures, claudeModelSupportsFastMode } from "./feature-definitions.js";
@@ -374,7 +373,7 @@ interface ClaudeAgentSessionOptions {
}
type ClaudeThinkingEffort = "low" | "medium" | "high" | "xhigh" | "max";
type ClaudeThinkingOption = ClaudeThinkingEffort | typeof CLAUDE_ULTRACODE_THINKING_OPTION_ID;
type ClaudeThinkingOption = ClaudeThinkingEffort | "ultracode";
function resolvePathEnvKey(): "Path" | "PATH" | null {
if (process.env["Path"] !== undefined) return "Path";
@@ -422,7 +421,7 @@ function isClaudeThinkingEffort(value: string | null | undefined): value is Clau
}
function isClaudeThinkingOption(value: string | null | undefined): value is ClaudeThinkingOption {
return value === CLAUDE_ULTRACODE_THINKING_OPTION_ID || isClaudeThinkingEffort(value);
return value === "ultracode" || isClaudeThinkingEffort(value);
}
interface ClaudeOptionsLogSummary {
@@ -2865,7 +2864,7 @@ class ClaudeAgentSession implements AgentSession {
this.config.thinkingOptionId && this.config.thinkingOptionId !== "default"
? this.config.thinkingOptionId
: undefined;
if (thinkingOptionId === CLAUDE_ULTRACODE_THINKING_OPTION_ID) {
if (thinkingOptionId === "ultracode") {
return { thinking: { type: "adaptive" }, effort: "xhigh", ultracode: true };
}
if (thinkingOptionId && isClaudeThinkingEffort(thinkingOptionId)) {
@@ -1,5 +1,10 @@
import type { AgentFeature, AgentFeatureToggle } from "../../agent-sdk-types.js";
import { claudeManifestModelSupportsFastMode } from "./model-manifest.js";
const CLAUDE_FAST_MODE_SUPPORTED_MODEL_PREFIXES = [
"claude-opus-4-8",
"claude-opus-4-7",
"claude-opus-4-6",
] as const;
export const CLAUDE_FAST_MODE_FEATURE: Omit<AgentFeatureToggle, "value"> = {
type: "toggle",
@@ -10,8 +15,20 @@ export const CLAUDE_FAST_MODE_FEATURE: Omit<AgentFeatureToggle, "value"> = {
icon: "zap",
};
function normalizeClaudeModelId(modelId: string | null | undefined): string | null {
const normalized = typeof modelId === "string" ? modelId.trim() : "";
return normalized.length > 0 ? normalized : null;
}
export function claudeModelSupportsFastMode(modelId: string | null | undefined): boolean {
return claudeManifestModelSupportsFastMode(modelId);
const normalizedModelId = normalizeClaudeModelId(modelId);
if (!normalizedModelId) {
return false;
}
return CLAUDE_FAST_MODE_SUPPORTED_MODEL_PREFIXES.some(
(prefix) => normalizedModelId === prefix || normalizedModelId.startsWith(`${prefix}[`),
);
}
export function buildClaudeFeatures(input: {
@@ -1,282 +0,0 @@
import type { AgentModelDefinition, AgentSelectOption } from "../../agent-sdk-types.js";
type ClaudeEffortLevel = "low" | "medium" | "high" | "xhigh" | "max";
interface ClaudeModelManifestEntry {
id: string;
label: string;
description: string;
isDefault?: boolean;
contextWindowMaxTokens?: number;
effortLevels?: readonly ClaudeEffortLevel[];
supportsFastMode?: boolean;
}
const CLAUDE_EFFORT_LEVELS = {
standard: ["low", "medium", "high", "max"],
xhigh: ["low", "medium", "high", "xhigh", "max"],
} as const satisfies Record<string, readonly ClaudeEffortLevel[]>;
const CLAUDE_EFFORT_LABELS = {
low: "Low",
medium: "Medium",
high: "High",
xhigh: "Extra High",
max: "Max",
} as const satisfies Record<ClaudeEffortLevel, string>;
export const CLAUDE_ULTRACODE_THINKING_OPTION_ID = "ultracode";
export const CLAUDE_MODEL_MANIFEST = [
{
id: "claude-fable-5",
label: "Fable 5",
description: "Fable 5 · Most powerful model",
contextWindowMaxTokens: 1_000_000,
effortLevels: CLAUDE_EFFORT_LEVELS.xhigh,
},
{
id: "claude-opus-4-8[1m]",
label: "Opus 4.8 1M",
description: "Opus 4.8 with 1M context window",
contextWindowMaxTokens: 1_000_000,
effortLevels: CLAUDE_EFFORT_LEVELS.xhigh,
supportsFastMode: true,
},
{
id: "claude-opus-4-8",
label: "Opus 4.8",
description: "Opus 4.8 · Latest release",
isDefault: true,
contextWindowMaxTokens: 200_000,
effortLevels: CLAUDE_EFFORT_LEVELS.xhigh,
supportsFastMode: true,
},
{
id: "claude-sonnet-5",
label: "Sonnet 5",
description: "Sonnet 5 · Best for everyday tasks",
contextWindowMaxTokens: 1_000_000,
effortLevels: CLAUDE_EFFORT_LEVELS.xhigh,
},
{
id: "claude-opus-4-7[1m]",
label: "Opus 4.7 1M",
description: "Opus 4.7 with 1M context window",
contextWindowMaxTokens: 1_000_000,
effortLevels: CLAUDE_EFFORT_LEVELS.xhigh,
supportsFastMode: true,
},
{
id: "claude-opus-4-7",
label: "Opus 4.7",
description: "Opus 4.7 · Previous release",
contextWindowMaxTokens: 200_000,
effortLevels: CLAUDE_EFFORT_LEVELS.xhigh,
supportsFastMode: true,
},
{
id: "claude-opus-4-6[1m]",
label: "Opus 4.6 1M",
description: "Opus 4.6 with 1M context window",
contextWindowMaxTokens: 1_000_000,
effortLevels: CLAUDE_EFFORT_LEVELS.standard,
supportsFastMode: true,
},
{
id: "claude-opus-4-6",
label: "Opus 4.6",
description: "Opus 4.6 · Most capable for complex work",
contextWindowMaxTokens: 200_000,
effortLevels: CLAUDE_EFFORT_LEVELS.standard,
supportsFastMode: true,
},
{
id: "claude-sonnet-4-6[1m]",
label: "Sonnet 4.6 1M",
description: "Sonnet 4.6 with 1M context window",
contextWindowMaxTokens: 1_000_000,
effortLevels: CLAUDE_EFFORT_LEVELS.standard,
},
{
id: "claude-sonnet-4-6",
label: "Sonnet 4.6",
description: "Sonnet 4.6 · Best for everyday tasks",
contextWindowMaxTokens: 200_000,
effortLevels: CLAUDE_EFFORT_LEVELS.standard,
},
{
id: "claude-haiku-4-5",
label: "Haiku 4.5",
description: "Haiku 4.5 · Fastest for quick answers",
contextWindowMaxTokens: 200_000,
},
] as const satisfies readonly ClaudeModelManifestEntry[];
function buildThinkingOptions(
effortLevels: readonly ClaudeEffortLevel[] | undefined,
): AgentSelectOption[] | undefined {
if (!effortLevels) {
return undefined;
}
const options: AgentSelectOption[] = effortLevels.map((id) => ({
id,
label: CLAUDE_EFFORT_LABELS[id],
}));
if (effortLevels.includes("xhigh")) {
options.push({ id: CLAUDE_ULTRACODE_THINKING_OPTION_ID, label: "Ultra Code" });
}
return options;
}
export function getClaudeManifestModels(): AgentModelDefinition[] {
return CLAUDE_MODEL_MANIFEST.map((model) => {
const thinkingOptions = buildThinkingOptions(
"effortLevels" in model ? model.effortLevels : undefined,
);
return {
provider: "claude",
id: model.id,
label: model.label,
description: model.description,
...("isDefault" in model && model.isDefault ? { isDefault: true } : {}),
...(model.contextWindowMaxTokens !== undefined
? { contextWindowMaxTokens: model.contextWindowMaxTokens }
: {}),
...(thinkingOptions ? { thinkingOptions } : {}),
};
});
}
export function isClaudeManifestModelId(modelId: string): boolean {
return CLAUDE_MODEL_MANIFEST.some((model) => model.id === modelId);
}
export function claudeManifestModelSupportsFastMode(modelId: string | null | undefined): boolean {
const normalizedModelId = normalizeClaudeManifestModelId(modelId);
if (!normalizedModelId) {
return false;
}
return CLAUDE_MODEL_MANIFEST.some(
(model) =>
model.id === normalizedModelId &&
"supportsFastMode" in model &&
model.supportsFastMode === true,
);
}
/**
* Normalize first-party Claude model IDs for manifest capability checks. Provider-prefixed
* runtime IDs intentionally use normalizeClaudeRuntimeModelId instead.
*/
export function normalizeClaudeManifestModelId(value: string | null | undefined): string | null {
const trimmed = typeof value === "string" ? value.trim() : "";
if (!trimmed) {
return null;
}
if (isClaudeManifestModelId(trimmed)) {
return trimmed;
}
const singleSegmentMatch = trimmed.match(
/^(?:claude[-_ ])?(fable|opus|sonnet|haiku)[-_ ]+(\d+)(\[1m\])?(?:[-_ ]+\d{8})?$/i,
);
if (singleSegmentMatch) {
return normalizeSingleSegmentClaudeModelId(
singleSegmentMatch[1],
singleSegmentMatch[2],
trimmed.toLowerCase().includes("[1m]"),
);
}
const runtimeMatch = trimmed.match(
/^(?:claude[-_ ])?(opus|sonnet|haiku)[-_ ]+(\d+)[-.](\d+)(\[1m\])?(?:[-_ ]+\d{8})?$/i,
);
if (!runtimeMatch) {
return null;
}
return normalizeMajorMinorClaudeModelId(
runtimeMatch[1],
runtimeMatch[2],
runtimeMatch[3],
Boolean(runtimeMatch[4]),
);
}
/**
* Normalize a Claude Code runtime/config model string to a known manifest ID.
* Runtime metadata may include provider prefixes such as Bedrock model IDs; feature
* gates should use normalizeClaudeManifestModelId instead.
*/
export function normalizeClaudeRuntimeModelId(value: string | null | undefined): string | null {
const normalizedManifestModelId = normalizeClaudeManifestModelId(value);
if (normalizedManifestModelId) {
return normalizedManifestModelId;
}
const trimmed = typeof value === "string" ? value.trim() : "";
if (!trimmed) {
return null;
}
const singleSegmentMatch = trimmed.match(
/claude[-_ ](fable|opus|sonnet|haiku)[-_ ]+(\d+)(\[1m\])?/i,
);
if (singleSegmentMatch) {
const normalizedModelId = normalizeSingleSegmentClaudeModelId(
singleSegmentMatch[1],
singleSegmentMatch[2],
Boolean(singleSegmentMatch[3]),
);
if (normalizedModelId) {
return normalizedModelId;
}
}
const runtimeMatch = trimmed.match(
/claude[-_ ](opus|sonnet|haiku)[-_ ]+(\d+)[-.](\d+)(\[1m\])?/i,
);
if (!runtimeMatch) {
return null;
}
return normalizeMajorMinorClaudeModelId(
runtimeMatch[1],
runtimeMatch[2],
runtimeMatch[3],
Boolean(runtimeMatch[4]),
);
}
function normalizeSingleSegmentClaudeModelId(
familyValue: string,
major: string,
hasOneMillionContext: boolean,
): string | null {
const family = familyValue.toLowerCase();
const suffix = hasOneMillionContext ? "[1m]" : "";
const candidates = [`claude-${family}-${major}${suffix}`, `claude-${family}-${major}`];
for (const candidate of candidates) {
if (isClaudeManifestModelId(candidate)) {
return candidate;
}
}
return null;
}
function normalizeMajorMinorClaudeModelId(
familyValue: string,
major: string,
minor: string,
hasOneMillionContext: boolean,
): string | null {
const family = familyValue.toLowerCase();
const suffix = hasOneMillionContext ? "[1m]" : "";
const candidate = `claude-${family}-${major}-${minor}${suffix}`;
return isClaudeManifestModelId(candidate) ? candidate : null;
}
@@ -5,11 +5,6 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { createTestLogger } from "../../../../test-utils/test-logger.js";
import { ClaudeAgentClient } from "./agent.js";
import {
CLAUDE_ULTRACODE_THINKING_OPTION_ID,
claudeManifestModelSupportsFastMode,
normalizeClaudeManifestModelId,
} from "./model-manifest.js";
import { findClaudeModel, getClaudeModels, normalizeClaudeRuntimeModelId } from "./models.js";
const createdClaudeConfigDirs: string[] = [];
@@ -83,41 +78,6 @@ describe("getClaudeModels", () => {
);
});
it("derives thinking options from model effort capabilities", () => {
const models = new Map(getClaudeModels().map((model) => [model.id, model]));
expect(models.get("claude-sonnet-5")?.thinkingOptions?.map((option) => option.id)).toEqual([
"low",
"medium",
"high",
"xhigh",
"max",
CLAUDE_ULTRACODE_THINKING_OPTION_ID,
]);
expect(
models
.get("claude-sonnet-5")
?.thinkingOptions?.find((option) => option.id === CLAUDE_ULTRACODE_THINKING_OPTION_ID)
?.label,
).toBe("Ultra Code");
expect(models.get("claude-opus-4-7")?.thinkingOptions?.map((option) => option.id)).toEqual([
"low",
"medium",
"high",
"xhigh",
"max",
CLAUDE_ULTRACODE_THINKING_OPTION_ID,
]);
expect(models.get("claude-sonnet-4-6")?.thinkingOptions?.map((option) => option.id)).toEqual([
"low",
"medium",
"high",
"max",
]);
expect(models.get("claude-haiku-4-5")?.thinkingOptions).toBeUndefined();
});
it("returns fresh copies each call", () => {
const a = getClaudeModels();
const b = getClaudeModels();
@@ -294,34 +254,11 @@ describe("normalizeClaudeRuntimeModelId", () => {
expect(normalizeClaudeRuntimeModelId("gpt-5")).toBeNull();
expect(normalizeClaudeRuntimeModelId("random")).toBeNull();
});
it("normalizes provider-form runtime model strings", () => {
expect(normalizeClaudeRuntimeModelId("openrouter/anthropic/claude-opus-4-8")).toBe(
"claude-opus-4-8",
);
expect(normalizeClaudeRuntimeModelId("us.anthropic.claude-opus-4-8[1m]")).toBe(
"claude-opus-4-8[1m]",
);
expect(normalizeClaudeRuntimeModelId("us.anthropic.claude-opus-4-8-20260101")).toBe(
"claude-opus-4-8",
);
});
});
describe("findClaudeModel", () => {
it("resolves runtime model IDs to catalog entries", () => {
expect(findClaudeModel("claude-sonnet-5-20260101")?.id).toBe("claude-sonnet-5");
expect(findClaudeModel("claude-sonnet-5[1m]")?.contextWindowMaxTokens).toBe(1_000_000);
expect(findClaudeModel("us.anthropic.claude-opus-4-8[1m]")?.contextWindowMaxTokens).toBe(
1_000_000,
);
});
});
describe("claudeManifestModelSupportsFastMode", () => {
it("keeps fast mode strict to first-party manifest model IDs", () => {
expect(normalizeClaudeManifestModelId("openrouter/anthropic/claude-opus-4-8")).toBeNull();
expect(claudeManifestModelSupportsFastMode("openrouter/anthropic/claude-opus-4-8")).toBe(false);
expect(claudeManifestModelSupportsFastMode("claude-opus-4-8-20260101")).toBe(true);
});
});
@@ -4,10 +4,117 @@ import * as path from "node:path";
import type { Logger } from "pino";
import type { AgentModelDefinition } from "../../agent-sdk-types.js";
import {
getClaudeManifestModels,
normalizeClaudeRuntimeModelId as normalizeClaudeManifestRuntimeModelId,
} from "./model-manifest.js";
const CLAUDE_THINKING_OPTIONS = [
{ id: "low", label: "Low" },
{ id: "medium", label: "Medium" },
{ id: "high", label: "High" },
{ id: "max", label: "Max" },
] as const;
const CLAUDE_EXTENDED_THINKING_OPTIONS = [
{ id: "low", label: "Low" },
{ id: "medium", label: "Medium" },
{ id: "high", label: "High" },
{ id: "xhigh", label: "Extra High" },
{ id: "max", label: "Max" },
] as const;
const CLAUDE_ULTRACODE_THINKING_OPTIONS = [
...CLAUDE_EXTENDED_THINKING_OPTIONS,
{ id: "ultracode", label: "Ultracode" },
] as const;
const CLAUDE_MODELS: AgentModelDefinition[] = [
{
provider: "claude",
id: "claude-fable-5",
label: "Fable 5",
description: "Fable 5 · Most powerful model",
contextWindowMaxTokens: 1_000_000,
thinkingOptions: [...CLAUDE_ULTRACODE_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-opus-4-8[1m]",
label: "Opus 4.8 1M",
description: "Opus 4.8 with 1M context window",
contextWindowMaxTokens: 1_000_000,
thinkingOptions: [...CLAUDE_ULTRACODE_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-opus-4-8",
label: "Opus 4.8",
description: "Opus 4.8 · Latest release",
isDefault: true,
contextWindowMaxTokens: 200_000,
thinkingOptions: [...CLAUDE_ULTRACODE_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-sonnet-5",
label: "Sonnet 5",
description: "Sonnet 5 · Best for everyday tasks",
contextWindowMaxTokens: 1_000_000,
thinkingOptions: [...CLAUDE_EXTENDED_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-opus-4-7[1m]",
label: "Opus 4.7 1M",
description: "Opus 4.7 with 1M context window",
contextWindowMaxTokens: 1_000_000,
thinkingOptions: [...CLAUDE_EXTENDED_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-opus-4-7",
label: "Opus 4.7",
description: "Opus 4.7 · Previous release",
contextWindowMaxTokens: 200_000,
thinkingOptions: [...CLAUDE_EXTENDED_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-opus-4-6[1m]",
label: "Opus 4.6 1M",
description: "Opus 4.6 with 1M context window",
contextWindowMaxTokens: 1_000_000,
thinkingOptions: [...CLAUDE_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-opus-4-6",
label: "Opus 4.6",
description: "Opus 4.6 · Most capable for complex work",
contextWindowMaxTokens: 200_000,
thinkingOptions: [...CLAUDE_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-sonnet-4-6[1m]",
label: "Sonnet 4.6 1M",
description: "Sonnet 4.6 with 1M context window",
contextWindowMaxTokens: 1_000_000,
thinkingOptions: [...CLAUDE_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-sonnet-4-6",
label: "Sonnet 4.6",
description: "Sonnet 4.6 · Best for everyday tasks",
contextWindowMaxTokens: 200_000,
thinkingOptions: [...CLAUDE_THINKING_OPTIONS],
},
{
provider: "claude",
id: "claude-haiku-4-5",
label: "Haiku 4.5",
description: "Haiku 4.5 · Fastest for quick answers",
contextWindowMaxTokens: 200_000,
},
];
const CLAUDE_SETTINGS_MODEL_ENV_KEYS = [
"ANTHROPIC_MODEL",
@@ -18,7 +125,7 @@ const CLAUDE_SETTINGS_MODEL_ENV_KEYS = [
] as const;
export function getClaudeModels(): AgentModelDefinition[] {
return getClaudeManifestModels();
return CLAUDE_MODELS.map((model) => ({ ...model }));
}
export function findClaudeModel(
@@ -28,7 +135,7 @@ export function findClaudeModel(
if (!normalizedModelId) {
return undefined;
}
return getClaudeModels().find((model) => model.id === normalizedModelId);
return CLAUDE_MODELS.find((model) => model.id === normalizedModelId);
}
export async function getClaudeModelsWithSettings(
@@ -129,5 +236,42 @@ function isRecord(value: unknown): value is Record<string, unknown> {
* Handles the `[1m]` suffix that the SDK appends for 1M context sessions.
*/
export function normalizeClaudeRuntimeModelId(value: string | null | undefined): string | null {
return normalizeClaudeManifestRuntimeModelId(value);
const trimmed = typeof value === "string" ? value.trim() : "";
if (!trimmed) {
return null;
}
// Check for exact match first (handles claude-opus-4-6[1m] directly)
if (CLAUDE_MODELS.some((model) => model.id === trimmed)) {
return trimmed;
}
// Match single-segment major-version model names such as claude-fable-5
// and claude-sonnet-5, including dated runtime strings from the SDK.
const singleSegmentMatch = trimmed.match(/(?:claude-)?(fable|opus|sonnet|haiku)[-_ ]+(\d+)/i);
if (singleSegmentMatch) {
const family = singleSegmentMatch[1].toLowerCase();
const major = singleSegmentMatch[2];
const suffix = trimmed.toLowerCase().includes("[1m]") ? "[1m]" : "";
const candidates = [`claude-${family}-${major}${suffix}`, `claude-${family}-${major}`];
for (const candidate of candidates) {
if (CLAUDE_MODELS.some((model) => model.id === candidate)) {
return candidate;
}
}
}
// Match: claude-{family}-{major}-{minor}[1m]? possibly followed by a date suffix
const runtimeMatch = trimmed.match(
/(?:claude-)?(opus|sonnet|haiku)[-_ ]+(\d+)[-.](\d+)(\[1m\])?/i,
);
if (!runtimeMatch) {
return null;
}
const family = runtimeMatch[1].toLowerCase();
const major = runtimeMatch[2];
const minor = runtimeMatch[3];
const suffix = runtimeMatch[4] ?? "";
return `claude-${family}-${major}-${minor}${suffix}`;
}
@@ -2359,126 +2359,6 @@ describe("Codex app-server provider", () => {
rmSync(source, { force: true });
});
test("mcpToolCall image content emits a completed tool call plus assistant markdown image", async () => {
const appServer = createFakeCodexAppServer();
const session = new CodexAppServerAgentSession(
createConfig({ cwd: "/workspace/project" }),
null,
createTestLogger(),
async () => appServer.child,
);
const events: AgentStreamEvent[] = [];
const timelineEvents: Array<Extract<AgentStreamEvent, { type: "timeline" }>> = [];
const timelineItemsReceived = new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
unsubscribe();
reject(new Error("Timed out waiting for MCP image timeline events"));
}, 1000);
const unsubscribe = session.subscribe((event) => {
events.push(event);
if (event.type !== "timeline") {
return;
}
timelineEvents.push(event);
if (timelineEvents.length === 2) {
clearTimeout(timeout);
unsubscribe();
resolve();
}
});
});
try {
const { turnId } = await session.startTurn("capture a browser screenshot");
appServer.child.stdout.write(
`${JSON.stringify({
method: "item/completed",
params: {
item: {
id: "mcp-browser-screenshot",
type: "mcpToolCall",
status: "completed",
server: "paseo",
tool: "browser_screenshot",
arguments: { browserId: "11111111-1111-4111-8111-111111111111" },
result: {
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "image", data: ONE_BY_ONE_PNG_BASE64, mimeType: "image/png" },
],
structuredContent: {
ok: true,
result: {
command: "screenshot",
browserId: "11111111-1111-4111-8111-111111111111",
mimeType: "image/png",
width: 1,
height: 1,
},
},
},
},
},
})}\n`,
);
await timelineItemsReceived;
expect(timelineEvents).toEqual([
{
type: "timeline",
provider: "codex",
turnId,
item: {
type: "tool_call",
callId: "mcp-browser-screenshot",
name: "paseo.browser_screenshot",
status: "completed",
error: null,
detail: {
type: "unknown",
input: { browserId: "11111111-1111-4111-8111-111111111111" },
output: {
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "text", text: "[image]" },
],
structuredContent: {
ok: true,
result: {
command: "screenshot",
browserId: "11111111-1111-4111-8111-111111111111",
mimeType: "image/png",
width: 1,
height: 1,
},
},
},
},
},
},
{
type: "timeline",
provider: "codex",
turnId,
item: expect.objectContaining({ type: "assistant_message" }),
},
]);
const imageEvent = timelineEvents[1];
if (imageEvent.item.type !== "assistant_message") {
throw new Error("Expected assistant image timeline event");
}
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
const source = markdownImageSource(imageEvent.item.text);
expect(source).toMatch(/paseo-attachments[\\/].+\.png$/);
expect(existsSync(source)).toBe(true);
rmSync(source, { force: true });
appServer.assertNoErrors();
} finally {
await session.close();
}
});
test("ignores incomplete imageGeneration thread items without failing the turn", () => {
const session = createSession();
const events: AgentStreamEvent[] = [];
@@ -49,7 +49,6 @@ import { curateAgentActivity } from "../activity-curator.js";
import {
mapCodexToolCallEnvelope,
mapCodexToolCallFromThreadItem,
splitCodexMcpToolResultImages,
} from "./codex/tool-call-mapper.js";
import {
checkProviderLaunchAvailable,
@@ -1697,39 +1696,6 @@ export function threadItemToTimeline(
}
}
function mcpToolResultImagesToTimeline(item: unknown): AgentTimelineItem[] {
const itemRecord = toObjectRecord(item);
if (!itemRecord) {
return [];
}
const normalizedType = normalizeCodexThreadItemType(
typeof itemRecord.type === "string" ? itemRecord.type : undefined,
);
if (normalizedType !== "mcpToolCall") {
return [];
}
const { images } = splitCodexMcpToolResultImages(itemRecord.result);
return images
.map((image) =>
renderProviderImageOutputAsAssistantMarkdown(image, {
materialize: materializeProviderImage,
}),
)
.filter((timelineItem): timelineItem is AgentTimelineItem => timelineItem !== null);
}
function threadItemToTimelineEntries(
item: unknown,
options?: { includeUserMessage?: boolean; cwd?: string | null },
): AgentTimelineItem[] {
const timelineItem = threadItemToTimeline(item, options);
if (!timelineItem) {
return [];
}
return [timelineItem, ...mcpToolResultImagesToTimeline(item)];
}
const CodexThreadReadResponseSchema = z
.object({
thread: z
@@ -1769,7 +1735,8 @@ async function loadCodexThreadHistoryTimeline(params: {
const timeline: PersistedTimelineEntry[] = [];
for (const turn of response.thread.turns) {
for (const item of turn.items) {
for (const timelineItem of threadItemToTimelineEntries(item, { cwd: params.cwd })) {
const timelineItem = threadItemToTimeline(item, { cwd: params.cwd });
if (timelineItem) {
const timestamp =
readCodexHistoryTimestamp(item) ?? readCodexTurnHistoryTimestamp(turn, timelineItem);
timeline.push({
@@ -4881,15 +4848,10 @@ export class CodexAppServerAgentSession implements AgentSession {
}
this.warnOnIncompleteEditToolCall(timelineItem, "item_completed", parsed.item);
}
const imageItems = mcpToolResultImagesToTimeline(parsed.item);
this.emitEvent({ type: "timeline", provider: CODEX_PROVIDER, item: timelineItem });
if (timelineItem.type === "assistant_message") {
this.pendingAssistantMessageBoundary = true;
}
for (const imageItem of imageItems) {
this.emitEvent({ type: "timeline", provider: CODEX_PROVIDER, item: imageItem });
this.pendingAssistantMessageBoundary = true;
}
if (itemId) {
this.emittedItemCompletedIds.add(itemId);
this.emittedItemStartedIds.delete(itemId);
@@ -703,44 +703,6 @@ describe("codex tool-call mapper", () => {
});
});
it("replaces mcp image result blocks with placeholder text in tool output", () => {
const item = expectMapped(
mapCodexToolCallFromThreadItem({
type: "mcpToolCall",
id: "codex-browser-screenshot",
status: "completed",
server: "paseo",
tool: "browser_screenshot",
arguments: { browserId: "11111111-1111-4111-8111-111111111111" },
result: {
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
],
},
}),
);
expect(item).toEqual({
type: "tool_call",
callId: "codex-browser-screenshot",
name: "paseo.browser_screenshot",
status: "completed",
error: null,
detail: {
type: "unknown",
input: { browserId: "11111111-1111-4111-8111-111111111111" },
output: {
content: [
{ type: "text", text: "Captured browser screenshot (1x1)." },
{ type: "text", text: "[image]" },
],
},
},
});
expect(JSON.stringify(item)).not.toContain("iVBORw0KGgo=");
});
it("normalizes codex paseo_voice.speak mcp calls and extracts spoken text", () => {
const item = expectMapped(
mapCodexToolCallFromThreadItem({
@@ -76,16 +76,6 @@ interface CodexResolvedToolCall {
cwd: string | null;
}
export interface CodexMcpToolResultImage {
data: string;
mimeType: string;
}
interface CodexMcpToolResultImagesSplit {
images: CodexMcpToolResultImage[];
output: unknown;
}
function toToolCallTimelineItem(envelope: CodexResolvedToolCall): ToolCallTimelineItem {
const name = envelope.toolKind === "speak" ? ("speak" as const) : envelope.name;
const parsedDetail = deriveCodexToolDetail({
@@ -606,50 +596,6 @@ function buildMcpToolName(server: string | undefined, tool: string): string {
return trimmedTool;
}
function readMcpImageContent(block: unknown): CodexMcpToolResultImage | null {
if (!isRecord(block)) {
return null;
}
if (block.type !== "image") {
return null;
}
if (typeof block.data !== "string" || typeof block.mimeType !== "string") {
return null;
}
return {
data: block.data,
mimeType: block.mimeType,
};
}
export function splitCodexMcpToolResultImages(result: unknown): CodexMcpToolResultImagesSplit {
if (!isRecord(result) || !Array.isArray(result.content)) {
return { images: [], output: result };
}
const images: CodexMcpToolResultImage[] = [];
const content = result.content.map((block) => {
const image = readMcpImageContent(block);
if (!image) {
return block;
}
images.push(image);
return { type: "text", text: "[image]" };
});
if (images.length === 0) {
return { images, output: result };
}
return {
images,
output: {
...result,
content,
},
};
}
function toNullableObject(value: Record<string, unknown>): Record<string, unknown> | null {
return Object.keys(value).length > 0 ? value : null;
}
@@ -909,7 +855,7 @@ function mapMcpToolCallItem(
}
const name = buildMcpToolName(item.server, tool);
const input = item.arguments ?? null;
const { output } = splitCodexMcpToolResultImages(item.result ?? null);
const output = item.result ?? null;
const error = item.error ?? null;
const status = normalizeToolCallStatus(item.status, error, output);
@@ -29,7 +29,6 @@ import {
import { WaitForAgentTracker } from "../wait-for-agent-tracker.js";
import { createAgentCommand, type CreateAgentFromMcpInput } from "../create-agent/create.js";
import type { VoiceCallerContext, VoiceSpeakHandler } from "../../voice-types.js";
import type { FirstAgentContext } from "../../messages.js";
import { expandUserPath, isSameOrDescendantPath, resolvePathFromBase } from "../../path-utils.js";
import type { TerminalManager } from "../../../terminal/terminal-manager.js";
import type { CreatePaseoWorktreeWorkflowFn } from "../../worktree-session.js";
@@ -66,7 +65,6 @@ import {
} from "../lifecycle-command.js";
import type { GitHubService } from "../../../services/github-service.js";
import type { WorkspaceGitService } from "../../workspace-git-service.js";
import type { WorkspaceRegistry } from "../../workspace-registry.js";
import { WorktreeRequestError } from "../../worktree-errors.js";
import {
archiveCommand,
@@ -101,15 +99,11 @@ export interface PaseoToolHostDependencies {
listActiveWorkspaces?: ArchiveDependencies["listActiveWorkspaces"];
archiveWorkspaceRecord?: ArchiveDependencies["archiveWorkspaceRecord"];
emitWorkspaceUpdatesForWorkspaceIds?: ArchiveDependencies["emitWorkspaceUpdatesForWorkspaceIds"];
workspaceRegistry?: Pick<WorkspaceRegistry, "get" | "upsert">;
markWorkspaceArchiving?: ArchiveDependencies["markWorkspaceArchiving"];
clearWorkspaceArchiving?: ArchiveDependencies["clearWorkspaceArchiving"];
createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn;
// Mints a fresh directory workspace for a cwd and returns its id.
ensureWorkspaceForCreate?: (
cwd: string,
firstAgentContext?: FirstAgentContext,
) => Promise<string>;
ensureWorkspaceForCreate?: (cwd: string) => Promise<string>;
browserToolsBroker?: BrowserToolsBroker | null;
paseoHome?: string;
worktreesRoot?: string;
@@ -560,22 +554,6 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
return options.ensureWorkspaceForCreate(resolvedCwd);
}
function resolveWorkspaceIdForRename(requestedWorkspaceId?: string): string {
const explicitWorkspaceId = requestedWorkspaceId?.trim();
if (explicitWorkspaceId) {
return explicitWorkspaceId;
}
if (callerAgentId) {
const callerAgent = resolveCallerAgent();
if (!callerAgent?.workspaceId) {
throw new Error(`Caller agent ${callerAgentId} has no current workspace`);
}
return callerAgent.workspaceId;
}
throw new Error("workspaceId is required outside an agent-scoped session");
}
const buildCallerAgentScheduleConfigExtras = (
callerAgent: NonNullable<ReturnType<typeof resolveCallerAgent>>,
): Record<string, unknown> => {
@@ -1185,9 +1163,7 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
async function resolveCreateAgentToolArgs(args: unknown): Promise<ResolvedCreateAgentToolArgs> {
if (callerAgentId) {
const parsed = agentToAgentCreateAgentArgsSchema.parse(args);
const { cwd, workspaceId, worktree } = await resolveCreateAgentWorkspace(parsed.workspace, {
prompt: parsed.initialPrompt,
});
const { cwd, workspaceId, worktree } = await resolveCreateAgentWorkspace(parsed.workspace);
return {
kind: "agent-scoped",
parsedArgs: parsed,
@@ -1201,9 +1177,7 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
if (parsedArgs.relationship.kind === "subagent") {
throw new Error("relationship subagent requires an agent-scoped tool session");
}
const { cwd, workspaceId, worktree } = await resolveCreateAgentWorkspace(parsedArgs.workspace, {
prompt: parsedArgs.initialPrompt,
});
const { cwd, workspaceId, worktree } = await resolveCreateAgentWorkspace(parsedArgs.workspace);
return {
kind: "top-level",
parsedArgs,
@@ -1317,7 +1291,6 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
async function resolveCreateAgentWorkspace(
workspace: AgentToAgentCreateAgentArgs["workspace"] | TopLevelCreateAgentArgs["workspace"],
firstAgentContext: FirstAgentContext | undefined,
): Promise<{
cwd: string | undefined;
workspaceId: string | undefined;
@@ -1369,7 +1342,7 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
}
return {
cwd,
workspaceId: await options.ensureWorkspaceForCreate(cwd, firstAgentContext),
workspaceId: await options.ensureWorkspaceForCreate(cwd),
worktree: undefined,
};
}
@@ -1808,66 +1781,6 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase
},
);
registerTool(
"rename_workspace",
{
title: "Rename workspace",
description:
"Rename a workspace by setting its user-visible title. Omit workspaceId to rename your current workspace.",
inputSchema: {
workspaceId: z
.string()
.trim()
.min(1)
.optional()
.describe("Workspace id to rename. Omit to rename your current workspace."),
title: z
.string()
.trim()
.min(1, "title is required")
.describe("New user-visible workspace title."),
},
outputSchema: {
success: z.boolean(),
workspaceId: z.string(),
title: z.string(),
},
},
async ({ workspaceId: requestedWorkspaceId, title }) => {
if (!options.workspaceRegistry) {
throw new Error("Workspace registry is required to rename workspaces");
}
if (!options.emitWorkspaceUpdatesForWorkspaceIds) {
throw new Error("Workspace update emitter is required to rename workspaces");
}
const workspaceId = resolveWorkspaceIdForRename(requestedWorkspaceId);
const existing = await options.workspaceRegistry.get(workspaceId);
if (!existing) {
throw new Error(`Workspace ${workspaceId} not found`);
}
if (existing.archivedAt) {
throw new Error(`Workspace ${workspaceId} is archived`);
}
await options.workspaceRegistry.upsert({
...existing,
title,
updatedAt: new Date().toISOString(),
});
await options.emitWorkspaceUpdatesForWorkspaceIds([workspaceId]);
return {
content: [],
structuredContent: ensureValidJson({
success: true,
workspaceId,
title,
}),
};
},
);
registerTool(
"list_terminals",
{
@@ -14,7 +14,7 @@ export interface PaseoToolConfig {
title?: string;
description?: string;
inputSchema?: z.ZodRawShape | z.ZodType;
outputSchema?: z.ZodRawShape;
outputSchema?: z.ZodRawShape | z.ZodType;
}
export interface PaseoToolDefinition extends PaseoToolConfig {
+3 -42
View File
@@ -139,7 +139,7 @@ import type { PushNotificationSender } from "./push/notifications.js";
import { getOrCreateServerId } from "./server-id.js";
import { resolveDaemonVersion } from "./daemon-version.js";
import type { AgentClient, AgentProvider } from "./agent/agent-sdk-types.js";
import type { FirstAgentContext, TerminalProfile } from "@getpaseo/protocol/messages";
import type { TerminalProfile } from "@getpaseo/protocol/messages";
import type {
AgentProviderRuntimeSettingsMap,
ProviderOverride,
@@ -162,10 +162,6 @@ import {
type DaemonAuthConfig,
} from "./auth.js";
import { createWebUiMiddleware } from "./web-ui.js";
import { WorkspaceAutoName } from "./workspace-auto-name.js";
import { createGitMutationService } from "./session/git-mutation/git-mutation-service.js";
import { workspaceIdsOnCheckout } from "./workspace-directory.js";
import { resolveFirstAgentPromptTitle } from "./agent/create-agent-title.js";
const MAX_MCP_DEBUG_BATCH_ITEMS = 10;
const REDACTED_LOG_VALUE = "[redacted]";
@@ -858,21 +854,11 @@ export async function createPaseoDaemon(
const findWorkspaceIdForCwdExternal = async (cwd: string): Promise<string | null> => {
return resolveWorkspaceIdForPath(cwd, await workspaceRegistry.list());
};
const ensureWorkspaceForCreateExternal = async (
cwd: string,
firstAgentContext?: FirstAgentContext,
): Promise<string> => {
const ensureWorkspaceForCreateExternal = async (cwd: string): Promise<string> => {
const workspace = await createLocalCheckoutWorkspace(
{ cwd, title: resolveFirstAgentPromptTitle(firstAgentContext) },
{ cwd },
{ projectRegistry, workspaceRegistry, workspaceGitService },
);
if (firstAgentContext) {
workspaceAutoName.scheduleForDirectory({
workspaceId: workspace.workspaceId,
cwd: workspace.cwd,
firstAgentContext,
});
}
return workspace.workspaceId;
};
const listActiveWorkspacesExternal = async (): Promise<ActiveWorkspaceRef[]> => {
@@ -905,30 +891,9 @@ export async function createPaseoDaemon(
),
);
};
const emitWorkspaceUpdateForCwdExternal = async (cwd: string) => {
const workspaceIds = workspaceIdsOnCheckout(await workspaceRegistry.list(), cwd);
await emitWorkspaceUpdatesExternal(workspaceIds);
};
const emitExternalSessionMessage = (message: SessionOutboundMessage) => {
wsServer?.broadcast(wrapSessionMessage(message));
};
const workspaceAutoName = new WorkspaceAutoName({
agentManager,
workspaceRegistry,
workspaceGitService,
providerSnapshotManager,
readDaemonConfig: () => ({ metadataGeneration: daemonConfigStore.get().metadataGeneration }),
gitMutation: createGitMutationService({
workspaceGitService,
github,
logger,
}),
emitWorkspaceUpdateForCwd: emitWorkspaceUpdateForCwdExternal,
emitWorkspaceUpdateForWorkspaceId: async (workspaceId) => {
await emitWorkspaceUpdatesExternal([workspaceId]);
},
logger,
});
setupAutoArchiveOnMerge({
paseoHome: config.paseoHome,
@@ -976,8 +941,6 @@ export async function createPaseoDaemon(
.map((session) => session.warmWorkspaceGitDataForWorkspace(workspace)) ?? [],
);
},
autoNameWorkspaceBranchForFirstAgent: (autoNameInput) =>
workspaceAutoName.scheduleForWorktree(autoNameInput),
emitWorkspaceUpdateForWorkspaceId: async (workspaceId) => {
await emitWorkspaceUpdatesExternal([workspaceId]);
},
@@ -1013,7 +976,6 @@ export async function createPaseoDaemon(
listActiveWorkspaces: listActiveWorkspacesExternal,
archiveWorkspaceRecord: archiveWorkspaceRecordExternal,
emitWorkspaceUpdatesForWorkspaceIds: emitWorkspaceUpdatesExternal,
workspaceRegistry,
markWorkspaceArchiving: markWorkspaceArchivingExternal,
clearWorkspaceArchiving: clearWorkspaceArchivingExternal,
ensureWorkspaceForCreate: ensureWorkspaceForCreateExternal,
@@ -1245,7 +1207,6 @@ export async function createPaseoDaemon(
daemonConfigStore,
mcpBaseUrl,
{ allowedOrigins, hostnames: configuredHostnames },
workspaceAutoName,
config.auth,
speechService,
terminalManager,
@@ -1,32 +1,16 @@
import { afterEach, describe, expect, test, vi } from "vitest";
import type {
BrowserAutomationCommand,
BrowserAutomationCommandName,
BrowserAutomationExecuteRequest,
BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BrowserToolsBroker, type BrowserHostClient } from "./broker.js";
import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js";
import { StaticBrowserToolsPolicy } from "./policy.js";
const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
const SECOND_BROWSER_ID = "22222222-2222-4222-8222-222222222222";
class FakeBrowserHostClient implements BrowserHostClient {
class FakeDesktopClient implements BrowserToolsDesktopClient {
public readonly receivedRequests: BrowserAutomationExecuteRequest[] = [];
public readonly hostKind: string;
public readonly supportedCommands: readonly BrowserAutomationCommandName[];
public constructor(
public readonly id: string,
options: {
hostKind?: string;
supportedCommands?: readonly BrowserAutomationCommandName[];
} = {},
) {
this.hostKind = options.hostKind ?? "desktop app";
this.supportedCommands = options.supportedCommands ?? [...BROWSER_AUTOMATION_COMMAND_NAMES];
}
public constructor(public readonly id: string) {}
public sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void {
this.receivedRequests.push(request);
@@ -35,30 +19,16 @@ class FakeBrowserHostClient implements BrowserHostClient {
public resolveLatestWith(
broker: BrowserToolsBroker,
responsePayload: BrowserAutomationExecuteResponse["payload"],
): boolean {
const latest = this.receivedRequests.at(-1);
if (!latest) {
throw new Error(`Host ${this.id} has not received a browser request.`);
}
return this.resolveRequestWith(broker, latest, responsePayload);
}
public resolveRequestWith(
broker: BrowserToolsBroker,
request: BrowserAutomationExecuteRequest,
responsePayload: BrowserAutomationExecuteResponse["payload"],
): boolean {
return broker.receiveResponse({
type: "browser.automation.execute.response",
payload: { ...responsePayload, requestId: request.requestId },
payload: responsePayload,
});
}
}
class FailingBrowserHostClient implements BrowserHostClient {
public readonly id = "host-1";
public readonly hostKind = "desktop app";
public readonly supportedCommands = [...BROWSER_AUTOMATION_COMMAND_NAMES];
class FailingDesktopClient implements BrowserToolsDesktopClient {
public readonly id = "desktop-1";
public sendBrowserAutomationRequest(): void {
throw new Error("websocket send failed");
@@ -73,8 +43,8 @@ function createBroker(options: { enabled: boolean; timeoutMs?: number }): Browse
});
}
function snapshotCommand(): BrowserAutomationCommand {
return { command: "snapshot", args: { browserId: BROWSER_ID } };
function pageInfoCommand(): BrowserAutomationCommand {
return { command: "page_info", args: { browserId: "browser-1" } };
}
describe("BrowserToolsBroker", () => {
@@ -85,7 +55,7 @@ describe("BrowserToolsBroker", () => {
test("disabled returns browser_disabled", async () => {
const broker = createBroker({ enabled: false });
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
@@ -96,23 +66,23 @@ describe("BrowserToolsBroker", () => {
});
});
test("no connected browser host returns a retryable browser_no_host error", async () => {
test("no capable desktop returns browser_no_desktop", async () => {
const broker = createBroker({ enabled: true });
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_host",
message: "No browser automation host is connected.",
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
retryable: true,
},
});
});
test("invalid browser requests return structured failures without contacting a host", async () => {
test("invalid browser requests return structured failures without contacting desktop", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeBrowserHostClient("host-1");
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
await expect(
@@ -135,13 +105,13 @@ describe("BrowserToolsBroker", () => {
expect(broker.getPendingRequestCount()).toBe(0);
});
test("capable browser host receives request and returns response", async () => {
test("capable fake desktop receives request and returns response", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeBrowserHostClient("host-1");
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({
command: { command: "list_tabs", args: {} },
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
workspaceId: "workspace-1",
});
@@ -150,7 +120,7 @@ describe("BrowserToolsBroker", () => {
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "list_tabs", args: {} },
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
},
]);
expect(broker.getPendingRequestCount()).toBe(1);
@@ -163,7 +133,7 @@ describe("BrowserToolsBroker", () => {
command: "list_tabs",
tabs: [
{
browserId: BROWSER_ID,
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
@@ -180,7 +150,7 @@ describe("BrowserToolsBroker", () => {
command: "list_tabs",
tabs: [
{
browserId: BROWSER_ID,
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
@@ -193,13 +163,13 @@ describe("BrowserToolsBroker", () => {
expect(broker.getPendingRequestCount()).toBe(0);
});
test("single browser host receives snapshot requests", async () => {
test("desktop receives snapshot requests", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeBrowserHostClient("host-1");
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
workspaceId: "workspace-1",
});
@@ -208,7 +178,7 @@ describe("BrowserToolsBroker", () => {
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
},
]);
@@ -217,14 +187,11 @@ describe("BrowserToolsBroker", () => {
ok: true,
result: {
command: "snapshot",
browserId: BROWSER_ID,
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
format: "aria-yaml",
snapshot: "- document",
truncated: false,
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
elements: [],
},
});
@@ -233,580 +200,21 @@ describe("BrowserToolsBroker", () => {
ok: true,
result: {
command: "snapshot",
browserId: BROWSER_ID,
browserId: "browser-1",
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
format: "aria-yaml",
snapshot: "- document",
truncated: false,
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
elements: [],
},
});
});
test("new tabs target the most recently registered host and tab commands stay with that host", async () => {
const broker = createBroker({ enabled: true });
const firstHost = new FakeBrowserHostClient("host-1");
const recentHost = new FakeBrowserHostClient("host-2");
broker.registerClient(firstHost);
broker.registerClient(recentHost);
const newTabPromise = broker.execute({
command: { command: "new_tab", args: { url: "https://example.com" } },
workspaceId: "workspace-1",
});
expect(firstHost.receivedRequests).toEqual([]);
expect(recentHost.receivedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "new_tab", args: { url: "https://example.com" } },
},
]);
recentHost.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "new_tab",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
},
});
await expect(newTabPromise).resolves.toMatchObject({
ok: true,
result: { command: "new_tab", browserId: BROWSER_ID },
});
const snapshotPromise = broker.execute({
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
workspaceId: "workspace-1",
});
expect(firstHost.receivedRequests).toEqual([]);
expect(recentHost.receivedRequests.at(-1)).toEqual({
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
});
recentHost.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "snapshot",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
format: "aria-yaml",
snapshot: "- document",
truncated: false,
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
},
});
await expect(snapshotPromise).resolves.toMatchObject({
ok: true,
result: { command: "snapshot", browserId: BROWSER_ID },
});
});
test("list tabs aggregates all hosts and seeds browser id affinity", async () => {
const broker = createBroker({ enabled: true });
const firstHost = new FakeBrowserHostClient("host-1");
const secondHost = new FakeBrowserHostClient("host-2");
broker.registerClient(firstHost);
broker.registerClient(secondHost);
const listPromise = broker.execute({
command: { command: "list_tabs", args: {} },
workspaceId: "workspace-1",
});
expect(firstHost.receivedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-1:host-1",
workspaceId: "workspace-1",
command: { command: "list_tabs", args: {} },
},
]);
expect(secondHost.receivedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-1:host-2",
workspaceId: "workspace-1",
command: { command: "list_tabs", args: {} },
},
]);
firstHost.resolveLatestWith(broker, {
requestId: "req-1:host-1",
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://one.example",
title: "One",
},
],
},
});
secondHost.resolveLatestWith(broker, {
requestId: "req-1:host-2",
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: SECOND_BROWSER_ID,
workspaceId: "workspace-1",
url: "https://two.example",
title: "Two",
},
],
},
});
await expect(listPromise).resolves.toEqual({
requestId: "req-1",
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://one.example",
title: "One",
isActive: false,
isLoading: false,
},
{
browserId: SECOND_BROWSER_ID,
workspaceId: "workspace-1",
url: "https://two.example",
title: "Two",
isActive: false,
isLoading: false,
},
],
},
});
const firstSnapshot = broker.execute({
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
workspaceId: "workspace-1",
});
const secondSnapshot = broker.execute({
command: { command: "snapshot", args: { browserId: SECOND_BROWSER_ID } },
workspaceId: "workspace-1",
requestId: "req-2",
});
expect(firstHost.receivedRequests.at(-1)?.command).toEqual({
command: "snapshot",
args: { browserId: BROWSER_ID },
});
expect(secondHost.receivedRequests.at(-1)?.command).toEqual({
command: "snapshot",
args: { browserId: SECOND_BROWSER_ID },
});
firstHost.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "snapshot",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://one.example",
title: "One",
format: "aria-yaml",
snapshot: "- document",
truncated: false,
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
},
});
secondHost.resolveLatestWith(broker, {
requestId: "req-2",
ok: true,
result: {
command: "snapshot",
browserId: SECOND_BROWSER_ID,
workspaceId: "workspace-1",
url: "https://two.example",
title: "Two",
format: "aria-yaml",
snapshot: "- document",
truncated: false,
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
},
});
await expect(firstSnapshot).resolves.toMatchObject({
ok: true,
result: { command: "snapshot", browserId: BROWSER_ID },
});
await expect(secondSnapshot).resolves.toMatchObject({
ok: true,
result: { command: "snapshot", browserId: SECOND_BROWSER_ID },
});
});
test.each([
{
name: "scroll",
command: { command: "scroll", args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 } },
result: { command: "scroll", browserId: BROWSER_ID, deltaX: 0, deltaY: 400 },
},
{
name: "resize",
command: { command: "resize", args: { browserId: BROWSER_ID, width: 1024, height: 768 } },
result: { command: "resize", browserId: BROWSER_ID, width: 1024, height: 768 },
},
{
name: "close_tab",
command: { command: "close_tab", args: { browserId: BROWSER_ID } },
result: { command: "close_tab", browserId: BROWSER_ID },
},
] satisfies Array<{
name: string;
command: BrowserAutomationCommand;
result: BrowserAutomationExecuteResponse["payload"] extends infer Payload
? Payload extends { ok: true }
? Payload["result"]
: never
: never;
}>)("routes $name to the host that owns the browser id", async ({ command, result }) => {
const broker = createBroker({ enabled: true });
const other = new FakeBrowserHostClient("host-1");
const owner = new FakeBrowserHostClient("host-2");
broker.registerClient(other);
broker.registerClient(owner);
const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } });
owner.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "new_tab",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://one.example",
},
});
await newTabPromise;
const resultPromise = broker.execute({ command, requestId: "req-command" });
expect(owner.receivedRequests.at(-1)).toEqual({
type: "browser.automation.execute.request",
requestId: "req-command",
command,
});
expect(other.receivedRequests).toEqual([]);
owner.resolveLatestWith(broker, {
requestId: "req-command",
ok: true,
result,
});
await expect(resultPromise).resolves.toEqual({
requestId: "req-command",
ok: true,
result,
});
});
test("successful close_tab clears browser id host affinity", async () => {
const broker = createBroker({ enabled: true });
const other = new FakeBrowserHostClient("host-1");
const owner = new FakeBrowserHostClient("host-2");
broker.registerClient(other);
broker.registerClient(owner);
const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } });
owner.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "new_tab",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://one.example",
},
});
await newTabPromise;
const closePromise = broker.execute({
command: { command: "close_tab", args: { browserId: BROWSER_ID } },
requestId: "req-close",
});
owner.resolveLatestWith(broker, {
requestId: "req-close",
ok: true,
result: { command: "close_tab", browserId: BROWSER_ID },
});
await expect(closePromise).resolves.toMatchObject({
ok: true,
result: { command: "close_tab", browserId: BROWSER_ID },
});
await expect(
broker.execute({
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
requestId: "req-after-close",
}),
).resolves.toEqual({
requestId: "req-after-close",
ok: false,
error: {
code: "browser_tab_not_found",
message: `Browser tab ${BROWSER_ID} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`,
retryable: false,
},
});
expect(owner.receivedRequests.at(-1)?.command).toEqual({
command: "close_tab",
args: { browserId: BROWSER_ID },
});
expect(other.receivedRequests).toEqual([]);
});
test("failed list tabs aggregation does not seed browser id affinity", async () => {
const broker = createBroker({ enabled: true });
const firstHost = new FakeBrowserHostClient("host-1");
const secondHost = new FakeBrowserHostClient("host-2");
broker.registerClient(firstHost);
broker.registerClient(secondHost);
const listPromise = broker.execute({
command: { command: "list_tabs", args: {} },
workspaceId: "workspace-1",
});
firstHost.resolveLatestWith(broker, {
requestId: "req-1:host-1",
ok: true,
result: {
command: "list_tabs",
tabs: [
{
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://one.example",
title: "One",
},
],
},
});
secondHost.resolveLatestWith(broker, {
requestId: "req-1:host-2",
ok: false,
error: {
code: "browser_timeout",
message: "Host did not answer list_tabs.",
retryable: true,
},
});
await expect(listPromise).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_timeout",
message: "Host did not answer list_tabs.",
retryable: true,
},
});
await expect(
broker.execute({
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
workspaceId: "workspace-1",
}),
).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: `Browser tab ${BROWSER_ID} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`,
retryable: false,
},
});
expect(firstHost.receivedRequests).toHaveLength(1);
expect(secondHost.receivedRequests).toHaveLength(1);
});
test("unsupported commands are rejected before sending to the routed host", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeBrowserHostClient("host-1", {
supportedCommands: ["list_tabs"],
hostKind: "desktop app",
});
broker.registerClient(client);
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unsupported",
message: 'Browser automation command "snapshot" is not supported by the desktop app.',
retryable: false,
},
});
await expect(
broker.execute({
command: {
command: "evaluate",
args: { browserId: BROWSER_ID, function: "() => document.title" },
},
}),
).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unsupported",
message: 'Browser automation command "evaluate" is not supported by the desktop app.',
retryable: false,
},
});
await expect(
broker.execute({
command: { command: "scroll", args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 } },
}),
).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_unsupported",
message: 'Browser automation command "scroll" is not supported by the desktop app.',
retryable: false,
},
});
expect(client.receivedRequests).toEqual([]);
});
test("unregistering a host strands its browser ids instead of routing them to another host", async () => {
const broker = createBroker({ enabled: true });
const owner = new FakeBrowserHostClient("host-1");
const unregisterOwner = broker.registerClient(owner);
const newTabPromise = broker.execute({
command: { command: "new_tab", args: {} },
workspaceId: "workspace-1",
});
owner.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "new_tab",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
},
});
await newTabPromise;
unregisterOwner();
const replacement = new FakeBrowserHostClient("host-2");
broker.registerClient(replacement);
await expect(
broker.execute({
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
workspaceId: "workspace-1",
}),
).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_host",
message: `The app hosting browser tab ${BROWSER_ID} disconnected.`,
retryable: true,
},
});
expect(replacement.receivedRequests).toEqual([]);
});
test("reconnecting the same host reclaims its stranded browser ids", async () => {
const broker = createBroker({ enabled: true });
const owner = new FakeBrowserHostClient("host-1");
const unregisterOwner = broker.registerClient(owner);
const newTabPromise = broker.execute({
command: { command: "new_tab", args: {} },
workspaceId: "workspace-1",
});
owner.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "new_tab",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
},
});
await newTabPromise;
unregisterOwner();
const reconnectedOwner = new FakeBrowserHostClient("host-1");
broker.registerClient(reconnectedOwner);
const snapshotPromise = broker.execute({
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
workspaceId: "workspace-1",
});
expect(reconnectedOwner.receivedRequests).toEqual([
{
type: "browser.automation.execute.request",
requestId: "req-1",
workspaceId: "workspace-1",
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
},
]);
reconnectedOwner.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: {
command: "snapshot",
browserId: BROWSER_ID,
workspaceId: "workspace-1",
url: "https://example.com",
title: "Example",
format: "aria-yaml",
snapshot: "- document",
truncated: false,
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
},
});
await expect(snapshotPromise).resolves.toMatchObject({
ok: true,
result: { command: "snapshot", browserId: BROWSER_ID },
});
});
test("timeout resolves browser_timeout and clears pending state", async () => {
vi.useFakeTimers();
const broker = createBroker({ enabled: true, timeoutMs: 50 });
broker.registerClient(new FakeBrowserHostClient("host-1"));
broker.registerClient(new FakeDesktopClient("desktop-1"));
const resultPromise = broker.execute({ command: snapshotCommand() });
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
await vi.advanceTimersByTimeAsync(50);
@@ -825,10 +233,10 @@ describe("BrowserToolsBroker", () => {
test("disconnect resolves retryable failure and clears pending request", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeBrowserHostClient("host-1");
const client = new FakeDesktopClient("desktop-1");
const unregister = broker.registerClient(client);
const resultPromise = broker.execute({ command: snapshotCommand() });
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
unregister();
@@ -837,57 +245,19 @@ describe("BrowserToolsBroker", () => {
requestId: "req-1",
ok: false,
error: {
code: "browser_no_host",
message: "The browser automation host disconnected before responding.",
code: "browser_no_desktop",
message: "The desktop browser automation client disconnected before responding.",
retryable: true,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
});
test("replacing a host registration resolves pending requests from the old registration", async () => {
test("desktop send failure resolves structured failure and clears pending request", async () => {
const broker = createBroker({ enabled: true });
const oldHost = new FakeBrowserHostClient("host-1");
const newHost = new FakeBrowserHostClient("host-1");
broker.registerClient(oldHost);
broker.registerClient(new FailingDesktopClient());
const pendingResult = broker.execute({ command: snapshotCommand() });
expect(oldHost.receivedRequests).toHaveLength(1);
expect(broker.getPendingRequestCount()).toBe(1);
broker.registerClient(newHost);
await expect(pendingResult).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
code: "browser_no_host",
message: "The browser automation host disconnected before responding.",
retryable: true,
},
});
expect(broker.getPendingRequestCount()).toBe(0);
const listResult = broker.execute({ command: { command: "list_tabs", args: {} } });
expect(newHost.receivedRequests).toHaveLength(1);
newHost.resolveLatestWith(broker, {
requestId: "req-1",
ok: true,
result: { command: "list_tabs", tabs: [] },
});
await expect(listResult).resolves.toEqual({
requestId: "req-1",
ok: true,
result: { command: "list_tabs", tabs: [] },
});
});
test("browser host send failure resolves structured failure and clears pending request", async () => {
const broker = createBroker({ enabled: true });
broker.registerClient(new FailingBrowserHostClient());
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
requestId: "req-1",
ok: false,
error: {
@@ -901,17 +271,17 @@ describe("BrowserToolsBroker", () => {
test("explicit browser failure response propagates typed error", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeBrowserHostClient("host-1");
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({ command: snapshotCommand() });
const resultPromise = broker.execute({ command: pageInfoCommand() });
client.resolveLatestWith(broker, {
requestId: "req-1",
ok: false,
error: {
code: "browser_tab_not_found",
message: `Browser tab ${BROWSER_ID} was not found.`,
message: "Browser tab browser-1 was not found.",
retryable: false,
},
});
@@ -921,7 +291,7 @@ describe("BrowserToolsBroker", () => {
ok: false,
error: {
code: "browser_tab_not_found",
message: `Browser tab ${BROWSER_ID} was not found.`,
message: "Browser tab browser-1 was not found.",
retryable: false,
},
});
@@ -930,10 +300,10 @@ describe("BrowserToolsBroker", () => {
test("invalid browser response resolves a structured failure and clears pending state", async () => {
const broker = createBroker({ enabled: true });
const client = new FakeBrowserHostClient("host-1");
const client = new FakeDesktopClient("desktop-1");
broker.registerClient(client);
const resultPromise = broker.execute({ command: snapshotCommand() });
const resultPromise = broker.execute({ command: pageInfoCommand() });
expect(broker.getPendingRequestCount()).toBe(1);
expect(
@@ -3,17 +3,14 @@ import {
BrowserAutomationExecuteRequestSchema,
BrowserAutomationExecuteResponseSchema,
type BrowserAutomationCommand,
type BrowserAutomationCommandName,
type BrowserAutomationExecuteRequest,
type BrowserAutomationExecuteResponse,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { browserToolsFailure, type BrowserToolsResponsePayload } from "./errors.js";
import type { BrowserToolsPolicy } from "./policy.js";
export interface BrowserHostClient {
export interface BrowserToolsDesktopClient {
id: string;
hostKind: string;
supportedCommands: readonly BrowserAutomationCommandName[];
sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void | Promise<void>;
}
@@ -22,23 +19,17 @@ export interface BrowserToolsExecuteInput {
agentId?: string;
cwd?: string;
workspaceId?: string;
browserId?: string;
requestId?: string;
timeoutMs?: number;
}
interface PendingBrowserToolsRequest {
clientId: string;
rememberAffinity: boolean;
timeout: ReturnType<typeof setTimeout>;
resolve: (payload: BrowserToolsResponsePayload) => void;
}
interface RegisteredBrowserHost {
client: BrowserHostClient;
registeredAt: number;
supportedCommands: ReadonlySet<BrowserAutomationCommandName>;
}
export interface BrowserToolsBrokerOptions {
policy: BrowserToolsPolicy;
defaultTimeoutMs?: number;
@@ -51,11 +42,8 @@ export class BrowserToolsBroker {
private readonly policy: BrowserToolsPolicy;
private readonly defaultTimeoutMs: number;
private readonly createRequestId: () => string;
private readonly clients = new Map<string, RegisteredBrowserHost>();
private readonly clients = new Map<string, BrowserToolsDesktopClient>();
private readonly pending = new Map<string, PendingBrowserToolsRequest>();
private readonly browserHostByBrowserId = new Map<string, string>();
private readonly strandedBrowserHostByBrowserId = new Map<string, string>();
private registrationSequence = 0;
public constructor(options: BrowserToolsBrokerOptions) {
this.policy = options.policy;
@@ -63,31 +51,16 @@ export class BrowserToolsBroker {
this.createRequestId = options.createRequestId ?? (() => `browser_${randomUUID()}`);
}
public registerClient(client: BrowserHostClient): () => void {
this.unregisterClient(client.id);
const registeredAt = ++this.registrationSequence;
this.clients.set(client.id, {
client,
registeredAt,
supportedCommands: new Set(client.supportedCommands),
});
return () => this.unregisterClient(client.id, registeredAt);
public registerClient(client: BrowserToolsDesktopClient): () => void {
this.clients.set(client.id, client);
return () => this.unregisterClient(client.id);
}
public unregisterClient(clientId: string, registeredAt?: number): void {
const current = this.clients.get(clientId);
if (!current || (registeredAt !== undefined && current.registeredAt !== registeredAt)) {
public unregisterClient(clientId: string): void {
const deleted = this.clients.delete(clientId);
if (!deleted) {
return;
}
this.clients.delete(clientId);
for (const [browserId, ownerClientId] of this.browserHostByBrowserId) {
if (ownerClientId !== clientId) {
continue;
}
this.browserHostByBrowserId.delete(browserId);
this.strandedBrowserHostByBrowserId.set(browserId, clientId);
}
for (const [requestId, pending] of this.pending) {
if (pending.clientId !== clientId) {
@@ -98,8 +71,8 @@ export class BrowserToolsBroker {
pending.resolve(
browserToolsFailure({
requestId,
code: "browser_no_host",
message: "The browser automation host disconnected before responding.",
code: "browser_no_desktop",
message: "The desktop browser automation client disconnected before responding.",
retryable: true,
}),
);
@@ -125,12 +98,23 @@ export class BrowserToolsBroker {
});
}
const client = this.selectClient();
if (!client) {
return browserToolsFailure({
requestId,
code: "browser_no_desktop",
message: "No desktop browser automation client is connected.",
retryable: true,
});
}
const request = BrowserAutomationExecuteRequestSchema.safeParse({
type: "browser.automation.execute.request",
requestId,
...(input.agentId ? { agentId: input.agentId } : {}),
...(input.cwd ? { cwd: input.cwd } : {}),
...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
...(input.browserId ? { browserId: input.browserId } : {}),
command: input.command,
});
@@ -142,29 +126,8 @@ export class BrowserToolsBroker {
});
}
if (request.data.command.command === "list_tabs") {
return this.executeListTabs({
request: request.data,
timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs,
});
}
const host = this.selectHostForCommand(request.data.command, requestId);
if (!host.ok) {
return host.payload;
}
const unsupported = this.unsupportedCommandFailure({
host: host.value,
commandName: request.data.command.command,
requestId,
});
if (unsupported) {
return unsupported;
}
return this.sendRequest({
host: host.value,
client,
request: request.data,
timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs,
});
@@ -202,225 +165,23 @@ export class BrowserToolsBroker {
this.pending.delete(parsed.data.payload.requestId);
clearTimeout(pending.timeout);
if (pending.rememberAffinity) {
this.rememberBrowserHostForPayload(pending.clientId, parsed.data.payload);
}
pending.resolve(parsed.data.payload);
return true;
}
private async executeListTabs(params: {
request: BrowserAutomationExecuteRequest;
timeoutMs: number;
}): Promise<BrowserToolsResponsePayload> {
const hosts = Array.from(this.clients.values());
if (hosts.length === 0) {
return this.noBrowserHostFailure(params.request.requestId);
}
for (const host of hosts) {
const unsupported = this.unsupportedCommandFailure({
host,
commandName: "list_tabs",
requestId: params.request.requestId,
});
if (unsupported) {
return unsupported;
}
}
if (hosts.length === 1) {
return this.sendRequest({
host: hosts[0],
request: params.request,
timeoutMs: params.timeoutMs,
});
}
const hostResponses = await Promise.all(
hosts.map(async (host) => ({
host,
payload: await this.sendRequest({
host,
request: {
...params.request,
requestId: `${params.request.requestId}:${host.client.id}`,
},
rememberAffinity: false,
timeoutMs: params.timeoutMs,
}),
})),
);
const failed = hostResponses.find(({ payload }) => !payload.ok);
if (failed) {
return withBrowserToolsRequestId(failed.payload, params.request.requestId);
}
for (const { host, payload } of hostResponses) {
this.rememberBrowserHostForPayload(host.client.id, payload);
}
return {
requestId: params.request.requestId,
ok: true,
result: {
command: "list_tabs",
tabs: hostResponses.flatMap(({ payload }) =>
payload.ok && payload.result.command === "list_tabs" ? payload.result.tabs : [],
),
},
};
}
private selectHostForCommand(
command: BrowserAutomationCommand,
requestId: string,
):
| { ok: true; value: RegisteredBrowserHost }
| { ok: false; payload: BrowserToolsResponsePayload } {
if (command.command === "new_tab") {
const host = this.selectMostRecentlyRegisteredHost();
return host
? { ok: true, value: host }
: { ok: false, payload: this.noBrowserHostFailure(requestId) };
}
const browserId = getBrowserIdForCommand(command);
if (!browserId) {
const host = this.selectMostRecentlyRegisteredHost();
return host
? { ok: true, value: host }
: { ok: false, payload: this.noBrowserHostFailure(requestId) };
}
const ownerClientId = this.browserHostByBrowserId.get(browserId);
if (ownerClientId) {
const host = this.clients.get(ownerClientId);
if (host) {
return { ok: true, value: host };
}
return {
ok: false,
payload: this.strandedBrowserTabFailure({ requestId, browserId }),
};
}
const strandedOwnerClientId = this.strandedBrowserHostByBrowserId.get(browserId);
if (strandedOwnerClientId) {
const reconnectedHost = this.clients.get(strandedOwnerClientId);
if (reconnectedHost) {
this.strandedBrowserHostByBrowserId.delete(browserId);
this.browserHostByBrowserId.set(browserId, strandedOwnerClientId);
return { ok: true, value: reconnectedHost };
}
return {
ok: false,
payload: this.strandedBrowserTabFailure({ requestId, browserId }),
};
}
if (this.clients.size === 1) {
const host = this.selectMostRecentlyRegisteredHost();
if (host) {
return { ok: true, value: host };
}
}
if (this.clients.size === 0) {
return { ok: false, payload: this.noBrowserHostFailure(requestId) };
}
return {
ok: false,
payload: browserToolsFailure({
requestId,
code: "browser_tab_not_found",
message: `Browser tab ${browserId} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`,
}),
};
}
private selectMostRecentlyRegisteredHost(): RegisteredBrowserHost | null {
let selected: RegisteredBrowserHost | null = null;
for (const host of this.clients.values()) {
selected = host;
}
return selected;
}
private unsupportedCommandFailure(params: {
host: RegisteredBrowserHost;
commandName: BrowserAutomationCommandName;
requestId: string;
}): BrowserToolsResponsePayload | null {
if (params.host.supportedCommands.has(params.commandName)) {
return null;
}
return browserToolsFailure({
requestId: params.requestId,
code: "browser_unsupported",
message: `Browser automation command "${params.commandName}" is not supported by the ${describeBrowserHost(params.host)}.`,
});
}
private noBrowserHostFailure(requestId: string): BrowserToolsResponsePayload {
return browserToolsFailure({
requestId,
code: "browser_no_host",
message: "No browser automation host is connected.",
retryable: true,
});
}
private strandedBrowserTabFailure(params: {
requestId: string;
browserId: string;
}): BrowserToolsResponsePayload {
return browserToolsFailure({
requestId: params.requestId,
code: "browser_no_host",
message: `The app hosting browser tab ${params.browserId} disconnected.`,
retryable: true,
});
}
private rememberBrowserHostForPayload(
clientId: string,
payload: BrowserToolsResponsePayload,
): void {
if (!payload.ok) {
return;
}
if (payload.result.command === "list_tabs") {
for (const tab of payload.result.tabs) {
this.browserHostByBrowserId.set(tab.browserId, clientId);
this.strandedBrowserHostByBrowserId.delete(tab.browserId);
}
return;
}
if (payload.result.command === "close_tab") {
this.browserHostByBrowserId.delete(payload.result.browserId);
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
return;
}
if ("browserId" in payload.result) {
this.browserHostByBrowserId.set(payload.result.browserId, clientId);
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
private selectClient(): BrowserToolsDesktopClient | null {
for (const client of this.clients.values()) {
return client;
}
return null;
}
private sendRequest(params: {
host: RegisteredBrowserHost;
client: BrowserToolsDesktopClient;
request: BrowserAutomationExecuteRequest;
rememberAffinity?: boolean;
timeoutMs: number;
}): Promise<BrowserToolsResponsePayload> {
const { host, request, timeoutMs } = params;
const client = host.client;
const { client, request, timeoutMs } = params;
return new Promise<BrowserToolsResponsePayload>((resolve) => {
const timeout = setTimeout(() => {
@@ -439,7 +200,6 @@ export class BrowserToolsBroker {
this.pending.set(request.requestId, {
clientId: client.id,
rememberAffinity: params.rememberAffinity ?? true,
timeout,
resolve,
});
@@ -467,25 +227,6 @@ export class BrowserToolsBroker {
}
}
function getBrowserIdForCommand(command: BrowserAutomationCommand): string | null {
if (command.command === "list_tabs" || command.command === "new_tab") {
return null;
}
return command.args.browserId;
}
function describeBrowserHost(host: RegisteredBrowserHost): string {
const hostKind = host.client.hostKind.trim();
return hostKind || "browser host";
}
function withBrowserToolsRequestId(
payload: BrowserToolsResponsePayload,
requestId: string,
): BrowserToolsResponsePayload {
return { ...payload, requestId } as BrowserToolsResponsePayload;
}
function resolveSendFailure(params: {
requestId: string;
pending: Map<string, PendingBrowserToolsRequest>;
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -587,8 +587,8 @@ describe("PersistedConfigSchema logging config", () => {
expect(parsed.log?.format).toBe("json");
});
test("rejects unknown logging config fields", () => {
const result = PersistedConfigSchema.safeParse({
test("strips unknown logging config fields", () => {
const parsed = PersistedConfigSchema.parse({
log: {
console: {
level: "info",
@@ -597,7 +597,7 @@ describe("PersistedConfigSchema logging config", () => {
},
});
expect(result.success).toBe(false);
expect(parsed.log?.console).toEqual({ level: "info" });
});
});
@@ -669,6 +669,42 @@ describe("loadPersistedConfig", () => {
}
});
test("loads a config with unrecognized persisted fields", () => {
const home = createTempHome();
const configPath = path.join(home, "config.json");
try {
writeFileSync(
configPath,
`${JSON.stringify(
{
version: 1,
daemon: {
listen: "127.0.0.1:6767",
futureDaemonSetting: { enabled: true },
relay: {
enabled: true,
futureRelaySetting: "enabled",
},
},
futureRootSetting: true,
},
null,
2,
)}\n`,
);
const config = loadPersistedConfig(home);
expect(config.daemon?.listen).toBe("127.0.0.1:6767");
expect(config.daemon?.relay?.enabled).toBe(true);
expect((config.daemon as Record<string, unknown>)?.futureDaemonSetting).toBeUndefined();
expect((config.daemon?.relay as Record<string, unknown>)?.futureRelaySetting).toBeUndefined();
expect((config as Record<string, unknown>).futureRootSetting).toBeUndefined();
} finally {
rmSync(home, { recursive: true, force: true });
}
});
test("loads a config that still uses the removed providers.openai.voice block", () => {
const home = createTempHome();
const configPath = path.join(home, "config.json");
+28 -28
View File
@@ -25,7 +25,7 @@ const LogConfigSchema = z
level: LogLevelSchema.optional(),
format: LogFormatSchema.optional(),
})
.strict()
.strip()
.optional(),
file: z
@@ -37,20 +37,20 @@ const LogConfigSchema = z
maxSize: z.string().min(1).optional(),
maxFiles: z.number().int().positive().optional(),
})
.strict()
.strip()
.optional(),
})
.strict()
.strip()
.optional(),
})
.strict();
.strip();
const OpenAiSpeechEndpointSchema = z
.object({
apiKey: z.string().trim().min(1).optional(),
baseUrl: z.string().trim().min(1).optional(),
})
.strict();
.strip();
const OpenAiProviderSchema = z
.object({
@@ -59,26 +59,26 @@ const OpenAiProviderSchema = z
stt: OpenAiSpeechEndpointSchema.optional(),
tts: OpenAiSpeechEndpointSchema.optional(),
})
.strict();
.strip();
const LocalSpeechProviderSchema = z
.object({
modelsDir: z.string().min(1).optional(),
})
.strict();
.strip();
const ProvidersSchema = z
.object({
openai: OpenAiProviderSchema.optional(),
local: LocalSpeechProviderSchema.optional(),
})
.strict();
.strip();
const WorktreesConfigSchema = z
.object({
root: z.string().min(1).optional(),
})
.strict();
.strip();
const BcryptHashSchema = z.string().regex(/^\$2[aby]\$\d{2}\$[./A-Za-z0-9]{53}$/, {
message: "Expected a bcrypt hash",
@@ -88,7 +88,7 @@ const DaemonAuthSchema = z
.object({
password: BcryptHashSchema.optional(),
})
.strict();
.strip();
const SpeechProviderIdSchema = z
.string()
@@ -106,10 +106,10 @@ const FeatureDictationSchema = z
language: z.string().trim().min(1).optional(),
confidenceThreshold: z.number().optional(),
})
.strict()
.strip()
.optional(),
})
.strict();
.strip();
const FeatureVoiceModeSchema = z
.object({
@@ -119,7 +119,7 @@ const FeatureVoiceModeSchema = z
provider: z.string().optional(),
model: z.string().min(1).optional(),
})
.strict()
.strip()
.optional(),
stt: z
.object({
@@ -127,13 +127,13 @@ const FeatureVoiceModeSchema = z
model: z.string().min(1).optional(),
language: z.string().trim().min(1).optional(),
})
.strict()
.strip()
.optional(),
turnDetection: z
.object({
provider: SpeechProviderIdSchema.optional(),
})
.strict()
.strip()
.optional(),
tts: z
.object({
@@ -143,17 +143,17 @@ const FeatureVoiceModeSchema = z
speakerId: z.number().int().optional(),
speed: z.number().optional(),
})
.strict()
.strip()
.optional(),
})
.strict();
.strip();
const FeatureWebUiSchema = z
.object({
enabled: z.boolean().optional(),
distDir: z.string().min(1).optional(),
})
.strict();
.strip();
const StructuredGenerationProviderConfigSchema = z
.object({
@@ -161,13 +161,13 @@ const StructuredGenerationProviderConfigSchema = z
model: z.string().min(1).optional(),
thinkingOptionId: z.string().min(1).optional(),
})
.strict();
.strip();
const AgentMetadataGenerationSchema = z
.object({
providers: z.array(StructuredGenerationProviderConfigSchema).optional(),
})
.strict();
.strip();
const BUILTIN_PROVIDER_IDS = ["claude", "codex", "copilot", "opencode", "pi", "omp"] as const;
@@ -254,7 +254,7 @@ export const PersistedConfigSchema = z
.object({
allowedOrigins: z.array(z.string()).optional(),
})
.strict()
.strip()
.optional(),
relay: z
.object({
@@ -264,7 +264,7 @@ export const PersistedConfigSchema = z
useTls: z.boolean().optional(),
publicUseTls: z.boolean().optional(),
})
.strict()
.strip()
.optional(),
serviceProxy: z
.object({
@@ -275,11 +275,11 @@ export const PersistedConfigSchema = z
listen: z.string().optional(),
publicBaseUrl: z.url().optional(),
})
.strict()
.strip()
.optional(),
auth: DaemonAuthSchema.optional(),
})
.strict()
.strip()
.transform(({ allowedHosts, ...daemon }) => {
const hostnames = daemon.hostnames ?? allowedHosts;
return hostnames === undefined ? daemon : { ...daemon, hostnames };
@@ -290,7 +290,7 @@ export const PersistedConfigSchema = z
.object({
baseUrl: z.string().optional(),
})
.strict()
.strip()
.optional(),
providers: ProvidersSchema.optional(),
@@ -300,7 +300,7 @@ export const PersistedConfigSchema = z
providers: z.preprocess(normalizeAgentProviders, ProviderOverridesSchema).optional(),
metadataGeneration: AgentMetadataGenerationSchema.optional(),
})
.strict()
.strip()
.optional(),
features: z
.object({
@@ -308,12 +308,12 @@ export const PersistedConfigSchema = z
voiceMode: FeatureVoiceModeSchema.optional(),
webUi: FeatureWebUiSchema.optional(),
})
.strict()
.strip()
.optional(),
log: LogConfigSchema.optional(),
})
.strict();
.strip();
type PersistedConfigSchemaOutput = z.infer<typeof PersistedConfigSchema>;

Some files were not shown because too many files have changed in this diff Show More