mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-27 19:52:49 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Mohamed Boudra
|
9eec33773a | ||
|
Mohamed Boudra
|
5616c6af6a |
@@ -1,26 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## 0.1.104-beta.1 - 2026-07-03
|
||||
|
||||
### Added
|
||||
|
||||
- Agents can drive the in-app browser: open tabs, navigate, click, type, and take screenshots
|
||||
- Inspect, annotate, and send page elements from a browser tab to the agent ([#1708](https://github.com/getpaseo/paseo/pull/1708) by [@huiliaoning](https://github.com/huiliaoning))
|
||||
- Schedules screen to create and manage recurring agents ([#1246](https://github.com/getpaseo/paseo/pull/1246))
|
||||
- Open a project from anywhere with Cmd+O ([#1849](https://github.com/getpaseo/paseo/pull/1849))
|
||||
- ByteDance TRAE CLI available as an agent provider ([#1831](https://github.com/getpaseo/paseo/pull/1831) by [@park0er](https://github.com/park0er))
|
||||
|
||||
### Improved
|
||||
|
||||
- Clearer cards when an agent asks a question ([#1643](https://github.com/getpaseo/paseo/pull/1643) by [@cleiter](https://github.com/cleiter))
|
||||
|
||||
### Fixed
|
||||
|
||||
- New Workspace drafts survive archiving a workspace ([#1838](https://github.com/getpaseo/paseo/pull/1838))
|
||||
- Composer autocomplete stays open after switching screens ([#1851](https://github.com/getpaseo/paseo/pull/1851))
|
||||
- Claude usage appears when a quota window has no scheduled reset ([#1855](https://github.com/getpaseo/paseo/pull/1855))
|
||||
- New workspace action shows for non-git projects in the sidebar ([#1857](https://github.com/getpaseo/paseo/pull/1857) by [@cleiter](https://github.com/cleiter))
|
||||
|
||||
## 0.1.103 - 2026-07-01
|
||||
|
||||
### Added
|
||||
|
||||
@@ -21,35 +21,34 @@ This is an npm workspace monorepo:
|
||||
|
||||
At the start of non-trivial work, list `docs/` and skim anything relevant to the task. When you learn something meta worth preserving — a gotcha, a convention, a workflow, a piece of system context that will outlive the current task — update an existing doc or propose a new one. Code-level facts belong in inline comments next to the code; system, process, and gotcha-level facts belong in `docs/`.
|
||||
|
||||
| Doc | What's in it |
|
||||
| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
|
||||
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
|
||||
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
|
||||
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
|
||||
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
|
||||
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
|
||||
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
|
||||
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
|
||||
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
|
||||
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
|
||||
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
|
||||
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
|
||||
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
|
||||
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
|
||||
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
|
||||
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
|
||||
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
|
||||
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
|
||||
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
|
||||
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
|
||||
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
|
||||
| [docs/browser-capture-harness.md](docs/browser-capture-harness.md) | Real-Electron browser screenshot harness and compositor-surface gotcha |
|
||||
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
|
||||
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
|
||||
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
|
||||
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
|
||||
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
|
||||
| Doc | What's in it |
|
||||
| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
|
||||
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
|
||||
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
|
||||
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
|
||||
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
|
||||
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
|
||||
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
|
||||
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
|
||||
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
|
||||
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
|
||||
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
|
||||
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
|
||||
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
|
||||
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
|
||||
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
|
||||
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
|
||||
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
|
||||
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
|
||||
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
|
||||
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
|
||||
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
|
||||
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
|
||||
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
|
||||
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
|
||||
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
|
||||
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
|
||||
|
||||
## Quick start
|
||||
|
||||
|
||||
@@ -138,7 +138,7 @@ Electron wrapper for macOS, Linux, and Windows.
|
||||
|
||||
> **Window-state v1 limitation:** only the _first_ window of a session restores and persists saved geometry (size/position/maximized). Windows opened via ⌘⇧N / second-instance / "Open in new window" open at the default size, OS-cascaded, and do not persist — this avoids every window stacking on the same restored bounds and fighting over the single window-state store. Lifting this needs per-window state keys.
|
||||
>
|
||||
> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus still records the workspace-active browser for UI state and `list_tabs` reporting, but agent automation targets only explicit browser ids returned by `browser_new_tab` or `browser_list_tabs`. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up.
|
||||
> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus and agent automation targets are intentionally separate: the workspace-active browser follows the user's focused tab, while the agent-active browser is the default target for browser MCP commands. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up.
|
||||
|
||||
### `packages/website` — Marketing site
|
||||
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
# Browser Capture Harness
|
||||
|
||||
The desktop capture harness is the real-Electron verification path for browser screenshots.
|
||||
It validates the compositor behavior that unit tests cannot see:
|
||||
|
||||
- the resident automation `<webview>` starts in the production parking state;
|
||||
- the parked guest has no copyable viewport frame;
|
||||
- the resident webview guest is sized to 1280x800 logical pixels;
|
||||
- multiple resident webviews are parked as an overlapping stack, and the capture
|
||||
target is raised above its sibling webviews before capture;
|
||||
- a newly attached resident webview whose first useful frame is delayed can be captured
|
||||
by holding prep active and retrying until the frame appears;
|
||||
- the app-style prep sequence moves the host to a paintable 1x1 clipped state, waits two animation frames and a layout read, then both viewport `capturePage` and full-page CDP screenshots return real pixels;
|
||||
- restore returns the host to offscreen parking after every capture.
|
||||
|
||||
Run it with the repo Electron:
|
||||
|
||||
```bash
|
||||
npm run capture-harness --workspace=@getpaseo/desktop
|
||||
```
|
||||
|
||||
The harness writes PNG evidence and `results.json` to:
|
||||
|
||||
```text
|
||||
packages/desktop/capture-harness/out/
|
||||
```
|
||||
|
||||
A passing run prints `PASS` lines for both guest sizes, the expected parked-capture
|
||||
failure, the legacy stacked-below-the-clip failure for the second webview, five viewport
|
||||
prep captures and five full-page prep captures for each of the first two webviews,
|
||||
fresh-delayed-first-frame viewport and full-page captures for a newly attached webview,
|
||||
and final completion. The PNG sizes may be device-pixel scaled; on a Retina display the
|
||||
1280x800 logical viewport is usually saved as 2560x1600.
|
||||
|
||||
## Mechanism
|
||||
|
||||
Electron captures copy from the guest web contents' compositor surface. A resident
|
||||
webview parked at `left:-20000px` and `opacity:0` does not have a copyable surface, and
|
||||
`capturePage({ stayHidden:false })` or CDP `Page.captureScreenshot` cannot rescue it.
|
||||
|
||||
Before pixel capture, the app renderer temporarily makes the resident host paintable:
|
||||
`left:0`, `top:0`, `opacity:1`, `pointer-events:none`, host size `1x1`, and
|
||||
`overflow:hidden`, with the full-size 1280x800 webview inside. Resident webviews are
|
||||
parked absolutely at `0,0` inside that host because a second webview stacked below the
|
||||
1px clip still has no copyable surface. During capture, the renderer raises the target
|
||||
webview above the other resident webviews; an overlay or sibling above the target can make
|
||||
full-page CDP capture fail. Main captures only after the renderer acknowledges two
|
||||
animation frames plus a `getBoundingClientRect()` read, and the renderer restores parking
|
||||
in a `finally`.
|
||||
@@ -195,6 +195,9 @@ Single file, validated with `PersistedConfigSchema`.
|
||||
|
||||
All fields are optional with sensible defaults.
|
||||
|
||||
Config parsing strips unrecognized object keys so a config written by a newer daemon does not brick
|
||||
older read paths such as `paseo daemon status`. Malformed known fields still fail validation.
|
||||
|
||||
`agents.metadataGeneration.providers` controls the preferred structured-generation fallback order for daemon-side metadata tasks such as commit messages, PR text, branch names, and generated agent titles. Entries are tried first in the configured order, then Paseo falls through to dynamically discovered defaults and finally the current selection when available.
|
||||
|
||||
Local speech model ids are intentionally narrow: STT uses `parakeet-tdt-0.6b-v2-int8`, TTS uses `kokoro-en-v0_19`, and turn detection uses the bundled Silero VAD model.
|
||||
|
||||
Generated
+21
-21
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "paseo",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "paseo",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"hasInstallScript": true,
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"workspaces": [
|
||||
@@ -35132,7 +35132,7 @@
|
||||
},
|
||||
"packages/app": {
|
||||
"name": "@getpaseo/app",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"dependencies": {
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
"@dnd-kit/sortable": "^10.0.0",
|
||||
@@ -36150,12 +36150,12 @@
|
||||
},
|
||||
"packages/cli": {
|
||||
"name": "@getpaseo/cli",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^1.0.0",
|
||||
"@getpaseo/client": "0.1.104-beta.1",
|
||||
"@getpaseo/protocol": "0.1.104-beta.1",
|
||||
"@getpaseo/server": "0.1.104-beta.1",
|
||||
"@getpaseo/client": "0.1.103",
|
||||
"@getpaseo/protocol": "0.1.103",
|
||||
"@getpaseo/server": "0.1.103",
|
||||
"chalk": "^5.3.0",
|
||||
"commander": "^12.0.0",
|
||||
"mime-types": "^2.1.35",
|
||||
@@ -36401,10 +36401,10 @@
|
||||
},
|
||||
"packages/client": {
|
||||
"name": "@getpaseo/client",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"dependencies": {
|
||||
"@getpaseo/protocol": "0.1.104-beta.1",
|
||||
"@getpaseo/relay": "0.1.104-beta.1",
|
||||
"@getpaseo/protocol": "0.1.103",
|
||||
"@getpaseo/relay": "0.1.103",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -36415,7 +36415,7 @@
|
||||
},
|
||||
"packages/desktop": {
|
||||
"name": "@getpaseo/desktop",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"dependencies": {
|
||||
"@getpaseo/cli": "*",
|
||||
@@ -36658,7 +36658,7 @@
|
||||
},
|
||||
"packages/expo-two-way-audio": {
|
||||
"name": "@getpaseo/expo-two-way-audio",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.14",
|
||||
@@ -37554,7 +37554,7 @@
|
||||
},
|
||||
"packages/highlight": {
|
||||
"name": "@getpaseo/highlight",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"dependencies": {
|
||||
"@codemirror/language": "^6.12.3",
|
||||
"@codemirror/legacy-modes": "^6.5.3",
|
||||
@@ -37786,7 +37786,7 @@
|
||||
},
|
||||
"packages/protocol": {
|
||||
"name": "@getpaseo/protocol",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"dependencies": {
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
@@ -37798,7 +37798,7 @@
|
||||
},
|
||||
"packages/relay": {
|
||||
"name": "@getpaseo/relay",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"dependencies": {
|
||||
"base64-js": "^1.5.1",
|
||||
"tweetnacl": "^1.0.3",
|
||||
@@ -38016,15 +38016,15 @@
|
||||
},
|
||||
"packages/server": {
|
||||
"name": "@getpaseo/server",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"dependencies": {
|
||||
"@agentclientprotocol/sdk": "^0.17.1",
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.3.195",
|
||||
"@anthropic-ai/sdk": "^0.104.2",
|
||||
"@getpaseo/client": "0.1.104-beta.1",
|
||||
"@getpaseo/highlight": "0.1.104-beta.1",
|
||||
"@getpaseo/protocol": "0.1.104-beta.1",
|
||||
"@getpaseo/relay": "0.1.104-beta.1",
|
||||
"@getpaseo/client": "0.1.103",
|
||||
"@getpaseo/highlight": "0.1.103",
|
||||
"@getpaseo/protocol": "0.1.103",
|
||||
"@getpaseo/relay": "0.1.103",
|
||||
"@isaacs/ttlcache": "^2.1.4",
|
||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||
"@opencode-ai/sdk": "1.14.46",
|
||||
@@ -38561,7 +38561,7 @@
|
||||
},
|
||||
"packages/website": {
|
||||
"name": "@getpaseo/website",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"dependencies": {
|
||||
"@cloudflare/vite-plugin": "^1.29.1",
|
||||
"@cloudflare/workers-types": "^4.20260317.1",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "paseo",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"private": true,
|
||||
"description": "Paseo: voice-controlled development environment for local AI coding agents",
|
||||
"keywords": [
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/app",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"private": true,
|
||||
"main": "index.ts",
|
||||
"scripts": {
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
} from "lucide-react-native";
|
||||
import { withUnistyles } from "react-native-unistyles";
|
||||
import type { AgentAttachment } from "@getpaseo/protocol/messages";
|
||||
import type { WorkspaceComposerAttachment } from "@/attachments/types";
|
||||
import type { BrowserAnnotationIntent, WorkspaceComposerAttachment } from "@/attachments/types";
|
||||
import { getFileTypeLabel } from "@/attachments/file-types";
|
||||
import { isPullRequestContextAttachment } from "@/attachments/workspace-attachment-utils";
|
||||
import { ICON_SIZE, type Theme } from "@/styles/theme";
|
||||
@@ -36,6 +36,13 @@ function getPullRequestContextSubtitle(attachment: WorkspaceComposerAttachment):
|
||||
return "Review";
|
||||
}
|
||||
|
||||
const BROWSER_INTENT_LABEL_KEYS: Record<BrowserAnnotationIntent, string> = {
|
||||
fix: "workspace.browser.annotate.intents.fix",
|
||||
change: "workspace.browser.annotate.intents.change",
|
||||
question: "workspace.browser.annotate.intents.question",
|
||||
approve: "workspace.browser.annotate.intents.approve",
|
||||
};
|
||||
|
||||
function getTextAttachmentSubtitle(
|
||||
attachment: Extract<AgentAttachment, { type: "text" }>,
|
||||
t: TFunction,
|
||||
@@ -89,10 +96,11 @@ export function getWorkspaceAttachmentPillContent(
|
||||
t: TFunction,
|
||||
): AttachmentPillContent {
|
||||
if (attachment.kind === "browser_element") {
|
||||
const intent = attachment.attachment.intent;
|
||||
return {
|
||||
icon: attachmentBrowserIcon,
|
||||
title: attachment.attachment.tag,
|
||||
subtitle: t("composer.attachments.element"),
|
||||
subtitle: intent ? t(BROWSER_INTENT_LABEL_KEYS[intent]) : t("composer.attachments.element"),
|
||||
};
|
||||
}
|
||||
if (isPullRequestContextAttachment(attachment)) {
|
||||
|
||||
@@ -21,6 +21,12 @@ export interface AttachmentMetadata {
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The kind of review feedback the user is attaching to a selected browser
|
||||
* element, sent to the agent alongside the element context.
|
||||
*/
|
||||
export type BrowserAnnotationIntent = "fix" | "change" | "question" | "approve";
|
||||
|
||||
export interface BrowserElementAttachment {
|
||||
url: string;
|
||||
selector: string;
|
||||
@@ -44,6 +50,8 @@ export interface BrowserElementAttachment {
|
||||
children: string[];
|
||||
/** Free-text review note the user wrote about this element, if any. */
|
||||
comment?: string;
|
||||
/** What the user wants the agent to do with this element, if annotated. */
|
||||
intent?: BrowserAnnotationIntent;
|
||||
/**
|
||||
* Cropped screenshot of the selected element, sent to the agent as an image
|
||||
* alongside the textual element context. Persisted via the attachment store;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { beforeEach, describe, expect, test } from "vitest";
|
||||
import { beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import { createJSONStorage, type StateStorage } from "zustand/middleware";
|
||||
import { mountBrowserAutomationHandler } from "./handler";
|
||||
import type { DesktopHostBridge } from "@/desktop/host";
|
||||
import { useBrowserStore } from "@/stores/browser-store";
|
||||
@@ -9,6 +8,20 @@ import {
|
||||
useWorkspaceLayoutStore,
|
||||
} from "@/stores/workspace-layout-store";
|
||||
|
||||
vi.mock("expo-router", () => ({
|
||||
router: {
|
||||
navigate: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("@react-native-async-storage/async-storage", () => ({
|
||||
default: {
|
||||
getItem: vi.fn(async () => null),
|
||||
setItem: vi.fn(async () => undefined),
|
||||
removeItem: vi.fn(async () => undefined),
|
||||
},
|
||||
}));
|
||||
|
||||
type BrowserAutomationExecuteRequest = Extract<
|
||||
SessionOutboundMessage,
|
||||
{ type: "browser.automation.execute.request" }
|
||||
@@ -17,28 +30,9 @@ type BrowserAutomationExecuteResponse = Extract<
|
||||
SessionInboundMessage,
|
||||
{ type: "browser.automation.execute.response" }
|
||||
>;
|
||||
type BrowserAutomationResponsePayload = BrowserAutomationExecuteResponse["payload"];
|
||||
|
||||
class FakeStateStorage implements StateStorage {
|
||||
private readonly values = new Map<string, string>();
|
||||
|
||||
public getItem = (key: string): string | null => this.values.get(key) ?? null;
|
||||
|
||||
public setItem = (key: string, value: string): void => {
|
||||
this.values.set(key, value);
|
||||
};
|
||||
|
||||
public removeItem = (key: string): void => {
|
||||
this.values.delete(key);
|
||||
};
|
||||
|
||||
public clear(): void {
|
||||
this.values.clear();
|
||||
}
|
||||
}
|
||||
|
||||
class FakeDaemonClient {
|
||||
public readonly sentResponses: BrowserAutomationExecuteResponse[] = [];
|
||||
public sentResponses: BrowserAutomationExecuteResponse[] = [];
|
||||
private handler: ((request: BrowserAutomationExecuteRequest) => void) | null = null;
|
||||
|
||||
public on(
|
||||
@@ -61,97 +55,6 @@ class FakeDaemonClient {
|
||||
public receive(nextRequest: BrowserAutomationExecuteRequest): void {
|
||||
this.handler?.(nextRequest);
|
||||
}
|
||||
|
||||
public payloadAt(index: number): BrowserAutomationResponsePayload {
|
||||
const response = this.sentResponses[index];
|
||||
if (!response) {
|
||||
throw new Error(`Missing browser automation response at index ${index}`);
|
||||
}
|
||||
return response.payload;
|
||||
}
|
||||
}
|
||||
|
||||
class FakeBrowserBridge {
|
||||
public readonly executedRequests: BrowserAutomationExecuteRequest[] = [];
|
||||
public readonly registeredWorkspaceBrowsers: Array<{ browserId: string; workspaceId: string }> =
|
||||
[];
|
||||
public readonly activeWorkspaceBrowsers: Array<{
|
||||
browserId: string | null;
|
||||
workspaceId: string;
|
||||
}> = [];
|
||||
public response: BrowserAutomationResponsePayload | null = null;
|
||||
public thrownError: unknown = null;
|
||||
|
||||
public executeAutomationCommand = async (
|
||||
request: BrowserAutomationExecuteRequest,
|
||||
): Promise<BrowserAutomationResponsePayload> => {
|
||||
this.executedRequests.push(request);
|
||||
if (this.thrownError) {
|
||||
throw this.thrownError;
|
||||
}
|
||||
return this.response ?? currentListTabsPayload(request.requestId);
|
||||
};
|
||||
|
||||
public registerWorkspaceBrowser = async (input: {
|
||||
browserId: string;
|
||||
workspaceId: string;
|
||||
}): Promise<void> => {
|
||||
this.registeredWorkspaceBrowsers.push(input);
|
||||
};
|
||||
|
||||
public setWorkspaceActiveBrowser = async (input: {
|
||||
browserId: string | null;
|
||||
workspaceId: string;
|
||||
}): Promise<void> => {
|
||||
this.activeWorkspaceBrowsers.push(input);
|
||||
};
|
||||
}
|
||||
|
||||
class FakeResidentBrowser {
|
||||
public readonly ensuredWebviews: Array<{ browserId: string; url: string }> = [];
|
||||
|
||||
public ensure = (input: { browserId: string; url: string }): HTMLElement | null => {
|
||||
this.ensuredWebviews.push(input);
|
||||
return null;
|
||||
};
|
||||
}
|
||||
|
||||
class BrowserAutomationHandlerHarness {
|
||||
public readonly client = new FakeDaemonClient();
|
||||
public readonly browser = new FakeBrowserBridge();
|
||||
public readonly resident = new FakeResidentBrowser();
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
public mount(
|
||||
input: {
|
||||
serverId?: string;
|
||||
host?: DesktopHostBridge | null;
|
||||
registrationWaitTimeoutMs?: number;
|
||||
registrationPollIntervalMs?: number;
|
||||
} = {},
|
||||
): void {
|
||||
this.unsubscribe = mountBrowserAutomationHandler({
|
||||
client: this.client,
|
||||
...(input.serverId ? { serverId: input.serverId } : {}),
|
||||
getHost: () => (input.host === undefined ? { browser: this.browser } : input.host),
|
||||
ensureResidentBrowserWebview: this.resident.ensure,
|
||||
...(input.registrationWaitTimeoutMs !== undefined
|
||||
? { registrationWaitTimeoutMs: input.registrationWaitTimeoutMs }
|
||||
: {}),
|
||||
...(input.registrationPollIntervalMs !== undefined
|
||||
? { registrationPollIntervalMs: input.registrationPollIntervalMs }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
public unmount(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
}
|
||||
|
||||
public receive(request: BrowserAutomationExecuteRequest): void {
|
||||
this.client.receive(request);
|
||||
}
|
||||
}
|
||||
|
||||
function browserAutomationRequest(): BrowserAutomationExecuteRequest {
|
||||
@@ -167,40 +70,45 @@ function browserNewTabRequest(): BrowserAutomationExecuteRequest {
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-new",
|
||||
agentId: "agent-1",
|
||||
workspaceId: "wks_workspace_a",
|
||||
command: {
|
||||
command: "new_tab",
|
||||
args: { url: "https://example.com" },
|
||||
},
|
||||
workspaceId: "/repo",
|
||||
command: { command: "new_tab", args: { workspaceId: "/repo", url: "https://example.com" } },
|
||||
};
|
||||
}
|
||||
|
||||
function emptyListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload {
|
||||
function emptyListTabsPayload(requestId = "req-new:list_tabs") {
|
||||
return {
|
||||
requestId,
|
||||
ok: true,
|
||||
ok: true as const,
|
||||
result: {
|
||||
command: "list_tabs",
|
||||
command: "list_tabs" as const,
|
||||
tabs: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function currentListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload {
|
||||
function currentListTabsPayload(requestId = "req-new:list_tabs") {
|
||||
return {
|
||||
requestId,
|
||||
ok: true,
|
||||
ok: true as const,
|
||||
result: {
|
||||
command: "list_tabs",
|
||||
command: "list_tabs" as const,
|
||||
tabs: currentBrowserTabs(),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function flushPromises(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function waitForAsyncWork(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
|
||||
function currentBrowserTabs() {
|
||||
return Object.values(useBrowserStore.getState().browsersById).map((browser) => ({
|
||||
browserId: browser.browserId,
|
||||
workspaceId: "wks_workspace_a",
|
||||
workspaceId: "/repo",
|
||||
url: browser.url,
|
||||
title: browser.title,
|
||||
isActive: true,
|
||||
@@ -208,170 +116,151 @@ function currentBrowserTabs() {
|
||||
}));
|
||||
}
|
||||
|
||||
function newTabResultFrom(payload: BrowserAutomationResponsePayload) {
|
||||
expect(payload).toMatchObject({
|
||||
requestId: "req-new",
|
||||
ok: true,
|
||||
result: { command: "new_tab", workspaceId: "wks_workspace_a", url: "https://example.com" },
|
||||
});
|
||||
if (!payload.ok || payload.result.command !== "new_tab") {
|
||||
throw new Error("Expected browser_new_tab success payload");
|
||||
}
|
||||
return payload.result;
|
||||
}
|
||||
|
||||
function workspaceBrowserTabs(workspaceKey: string, browserId: string) {
|
||||
return useWorkspaceLayoutStore
|
||||
.getState()
|
||||
.getWorkspaceTabs(workspaceKey)
|
||||
.filter((tab) => tab.target.kind === "browser" && tab.target.browserId === browserId);
|
||||
}
|
||||
|
||||
function flushAsyncWork(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function waitForRegistrationTimeout(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 20));
|
||||
}
|
||||
|
||||
const browserAutomationStorage = new FakeStateStorage();
|
||||
useBrowserStore.persist.setOptions({
|
||||
storage: createJSONStorage(() => browserAutomationStorage),
|
||||
});
|
||||
useWorkspaceLayoutStore.persist.setOptions({
|
||||
storage: createJSONStorage(() => browserAutomationStorage),
|
||||
});
|
||||
|
||||
describe("mountBrowserAutomationHandler", () => {
|
||||
beforeEach(() => {
|
||||
browserAutomationStorage.clear();
|
||||
useBrowserStore.setState({ browsersById: {} });
|
||||
useWorkspaceLayoutStore.setState({ layoutByWorkspace: {} });
|
||||
});
|
||||
|
||||
test("browser_new_tab creates and focuses a workspace browser tab", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
test("creates an unfocused workspace browser tab for browser_new_tab", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
const setWorkspaceActiveBrowser = vi.fn(async () => undefined);
|
||||
const setAgentActiveBrowser = vi.fn(async () => undefined);
|
||||
const registerWorkspaceBrowser = vi.fn(async () => undefined);
|
||||
const ensureResidentBrowserWebview = vi.fn();
|
||||
const executeAutomationCommand = vi.fn(async () => currentListTabsPayload());
|
||||
const workspaceKey = buildWorkspaceTabPersistenceKey({
|
||||
serverId: "server-1",
|
||||
workspaceId: "wks_workspace_a",
|
||||
workspaceId: "/repo",
|
||||
});
|
||||
if (!workspaceKey) {
|
||||
throw new Error("Expected workspace key");
|
||||
}
|
||||
const previousFocusedTabId = useWorkspaceLayoutStore
|
||||
if (!workspaceKey) throw new Error("expected workspace key");
|
||||
const focusedTabId = useWorkspaceLayoutStore
|
||||
.getState()
|
||||
.openTabFocused(workspaceKey, { kind: "draft", draftId: "human-draft" });
|
||||
browser.mount({ serverId: "server-1" });
|
||||
if (!focusedTabId) throw new Error("expected focused tab");
|
||||
mountBrowserAutomationHandler({
|
||||
client,
|
||||
serverId: "server-1",
|
||||
getHost: () =>
|
||||
({
|
||||
browser: {
|
||||
executeAutomationCommand,
|
||||
registerWorkspaceBrowser,
|
||||
setWorkspaceActiveBrowser,
|
||||
setAgentActiveBrowser,
|
||||
},
|
||||
}) satisfies DesktopHostBridge,
|
||||
ensureResidentBrowserWebview,
|
||||
});
|
||||
|
||||
browser.receive(browserNewTabRequest());
|
||||
await flushAsyncWork();
|
||||
client.receive(browserNewTabRequest());
|
||||
await flushPromises();
|
||||
|
||||
const result = newTabResultFrom(browser.client.payloadAt(0));
|
||||
const openedTabs = workspaceBrowserTabs(workspaceKey, result.browserId);
|
||||
const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
|
||||
expect(openedTabs).toEqual([
|
||||
expect.objectContaining({
|
||||
target: { kind: "browser", browserId: result.browserId },
|
||||
}),
|
||||
]);
|
||||
expect(layout?.root).toEqual(
|
||||
expect.objectContaining({
|
||||
kind: "pane",
|
||||
pane: expect.objectContaining({ focusedTabId: openedTabs[0]?.tabId }),
|
||||
}),
|
||||
const payload = client.sentResponses[0]?.payload;
|
||||
expect(payload?.ok).toBe(true);
|
||||
if (!payload?.ok) throw new Error("expected success");
|
||||
expect(payload.result.command).toBe("new_tab");
|
||||
if (payload.result.command !== "new_tab") throw new Error("expected new_tab result");
|
||||
expect(payload.result.url).toBe("https://example.com");
|
||||
expect(useWorkspaceLayoutStore.getState().getWorkspaceTabs(workspaceKey)).toContainEqual(
|
||||
expect.objectContaining({ target: { kind: "browser", browserId: payload.result.browserId } }),
|
||||
);
|
||||
expect(openedTabs[0]?.tabId).not.toBe(previousFocusedTabId);
|
||||
expect(browser.browser.registeredWorkspaceBrowsers).toEqual([
|
||||
{ browserId: result.browserId, workspaceId: "wks_workspace_a" },
|
||||
]);
|
||||
expect(browser.browser.activeWorkspaceBrowsers).toEqual([
|
||||
{ browserId: result.browserId, workspaceId: "wks_workspace_a" },
|
||||
]);
|
||||
expect(browser.resident.ensuredWebviews).toEqual([
|
||||
{ browserId: result.browserId, url: "https://example.com" },
|
||||
]);
|
||||
expect(browser.browser.executedRequests).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-new:list_tabs",
|
||||
agentId: "agent-1",
|
||||
workspaceId: "wks_workspace_a",
|
||||
command: { command: "list_tabs", args: {} },
|
||||
},
|
||||
]);
|
||||
const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
|
||||
expect(layout?.root.kind).toBe("pane");
|
||||
if (layout?.root.kind !== "pane") throw new Error("expected root pane");
|
||||
expect(layout.root.pane.focusedTabId).toBe(focusedTabId);
|
||||
expect(registerWorkspaceBrowser).toHaveBeenCalledWith({
|
||||
browserId: payload.result.browserId,
|
||||
workspaceId: "/repo",
|
||||
});
|
||||
expect(setAgentActiveBrowser).toHaveBeenCalledWith({
|
||||
agentId: "agent-1",
|
||||
browserId: payload.result.browserId,
|
||||
});
|
||||
expect(setWorkspaceActiveBrowser).not.toHaveBeenCalled();
|
||||
expect(ensureResidentBrowserWebview).toHaveBeenCalledWith({
|
||||
browserId: payload.result.browserId,
|
||||
url: "https://example.com",
|
||||
});
|
||||
expect(executeAutomationCommand).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
test("browser_new_tab returns a retryable timeout when the resident webview does not register", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.browser.response = emptyListTabsPayload();
|
||||
browser.mount({
|
||||
test("returns browser_timeout when the resident webview does not register", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
const ensureResidentBrowserWebview = vi.fn();
|
||||
const executeAutomationCommand = vi.fn(async () => emptyListTabsPayload());
|
||||
mountBrowserAutomationHandler({
|
||||
client,
|
||||
serverId: "server-1",
|
||||
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
|
||||
ensureResidentBrowserWebview,
|
||||
registrationWaitTimeoutMs: 1,
|
||||
registrationPollIntervalMs: 1,
|
||||
});
|
||||
|
||||
browser.receive(browserNewTabRequest());
|
||||
await waitForRegistrationTimeout();
|
||||
client.receive(browserNewTabRequest());
|
||||
await waitForAsyncWork();
|
||||
|
||||
expect(browser.client.sentResponses).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-new",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_timeout",
|
||||
message: expect.stringContaining("Timed out waiting for browser tab"),
|
||||
retryable: true,
|
||||
},
|
||||
},
|
||||
expect(client.sentResponses[0]?.payload).toMatchObject({
|
||||
requestId: "req-new",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_timeout",
|
||||
retryable: true,
|
||||
},
|
||||
]);
|
||||
expect(browser.resident.ensuredWebviews).toEqual([
|
||||
});
|
||||
expect(ensureResidentBrowserWebview).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ url: "https://example.com" }),
|
||||
]);
|
||||
});
|
||||
|
||||
test("browser_new_tab wraps registration bridge errors in a response", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.browser.thrownError = new Error("IPC registration check failed");
|
||||
browser.mount({ serverId: "server-1" });
|
||||
|
||||
browser.receive(browserNewTabRequest());
|
||||
await flushAsyncWork();
|
||||
|
||||
expect(browser.client.sentResponses).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-new",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unknown_error",
|
||||
message: "IPC registration check failed",
|
||||
retryable: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("non-new-tab requests send the desktop bridge response", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.browser.response = {
|
||||
requestId: "desktop-req",
|
||||
);
|
||||
expect(client.sentResponses[0]?.payload).not.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
};
|
||||
browser.mount();
|
||||
const request = browserAutomationRequest();
|
||||
result: { command: "new_tab" },
|
||||
});
|
||||
});
|
||||
|
||||
browser.receive(request);
|
||||
await flushAsyncWork();
|
||||
test("wraps browser_new_tab registration bridge errors in a response", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
const executeAutomationCommand = vi.fn(async () => {
|
||||
throw new Error("IPC registration check failed");
|
||||
});
|
||||
mountBrowserAutomationHandler({
|
||||
client,
|
||||
serverId: "server-1",
|
||||
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
|
||||
});
|
||||
|
||||
expect(browser.browser.executedRequests).toEqual([request]);
|
||||
expect(browser.client.sentResponses).toEqual([
|
||||
client.receive(browserNewTabRequest());
|
||||
await flushPromises();
|
||||
|
||||
expect(client.sentResponses[0]?.payload).toEqual({
|
||||
requestId: "req-new",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unknown_error",
|
||||
message: "IPC registration check failed",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("sends a success response from the desktop bridge", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
const executeAutomationCommand = vi.fn(async () => ({
|
||||
requestId: "req-1",
|
||||
ok: true as const,
|
||||
result: { command: "list_tabs" as const, tabs: [] },
|
||||
}));
|
||||
|
||||
mountBrowserAutomationHandler({
|
||||
client,
|
||||
getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge,
|
||||
});
|
||||
|
||||
client.receive(browserAutomationRequest());
|
||||
await flushPromises();
|
||||
|
||||
expect(executeAutomationCommand).toHaveBeenCalledWith(browserAutomationRequest());
|
||||
expect(client.sentResponses).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
@@ -383,14 +272,14 @@ describe("mountBrowserAutomationHandler", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("missing desktop bridge sends browser_unsupported", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.mount({ host: null });
|
||||
test("missing bridge sends browser_unsupported", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
mountBrowserAutomationHandler({ client, getHost: () => null });
|
||||
|
||||
browser.receive(browserAutomationRequest());
|
||||
await flushAsyncWork();
|
||||
client.receive(browserAutomationRequest());
|
||||
await flushPromises();
|
||||
|
||||
expect(browser.client.sentResponses).toEqual([
|
||||
expect(client.sentResponses).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
@@ -407,68 +296,80 @@ describe("mountBrowserAutomationHandler", () => {
|
||||
});
|
||||
|
||||
test("typed bridge errors become failure responses", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.browser.thrownError = {
|
||||
code: "browser_tab_not_found",
|
||||
message: "Browser tab browser-1 was not found.",
|
||||
retryable: false,
|
||||
};
|
||||
browser.mount();
|
||||
|
||||
browser.receive(browserAutomationRequest());
|
||||
await flushAsyncWork();
|
||||
|
||||
expect(browser.client.sentResponses).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: "Browser tab browser-1 was not found.",
|
||||
retryable: false,
|
||||
const client = new FakeDaemonClient();
|
||||
mountBrowserAutomationHandler({
|
||||
client,
|
||||
getHost: () => ({
|
||||
browser: {
|
||||
executeAutomationCommand: async () => {
|
||||
throw {
|
||||
code: "browser_tab_not_found",
|
||||
message: "Browser tab browser-1 was not found.",
|
||||
retryable: false,
|
||||
};
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
client.receive(browserAutomationRequest());
|
||||
await flushPromises();
|
||||
|
||||
expect(client.sentResponses[0]?.payload).toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: "Browser tab browser-1 was not found.",
|
||||
retryable: false,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test("unimplemented preload IPC reports browser_unsupported", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.browser.thrownError = new Error(
|
||||
'No handler registered for "paseo:browser:execute-automation-command"',
|
||||
);
|
||||
browser.mount();
|
||||
|
||||
browser.receive(browserAutomationRequest());
|
||||
await flushAsyncWork();
|
||||
|
||||
expect(browser.client.sentResponses).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: "Desktop browser automation is not implemented by this desktop build yet.",
|
||||
retryable: false,
|
||||
const client = new FakeDaemonClient();
|
||||
mountBrowserAutomationHandler({
|
||||
client,
|
||||
getHost: () => ({
|
||||
browser: {
|
||||
executeAutomationCommand: async () => {
|
||||
throw new Error('No handler registered for "paseo:browser:execute-automation-command"');
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
client.receive(browserAutomationRequest());
|
||||
await flushPromises();
|
||||
|
||||
expect(client.sentResponses[0]?.payload).toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: "Desktop browser automation is not implemented by this desktop build yet.",
|
||||
retryable: false,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test("unsubscribe stops handling browser automation requests", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.mount();
|
||||
test("unsubscribe stops handling requests", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
const executeAutomationCommand = vi.fn(async () => ({
|
||||
requestId: "req-1",
|
||||
ok: true as const,
|
||||
result: { command: "list_tabs" as const, tabs: [] },
|
||||
}));
|
||||
const unsubscribe = mountBrowserAutomationHandler({
|
||||
client,
|
||||
getHost: () => ({ browser: { executeAutomationCommand } }),
|
||||
});
|
||||
|
||||
browser.unmount();
|
||||
browser.receive(browserAutomationRequest());
|
||||
await flushAsyncWork();
|
||||
unsubscribe();
|
||||
client.receive(browserAutomationRequest());
|
||||
await flushPromises();
|
||||
|
||||
expect(browser.browser.executedRequests).toEqual([]);
|
||||
expect(browser.client.sentResponses).toEqual([]);
|
||||
expect(executeAutomationCommand).not.toHaveBeenCalled();
|
||||
expect(client.sentResponses).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
|
||||
import {
|
||||
ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault,
|
||||
installResidentBrowserCaptureBridge,
|
||||
} from "@/components/browser-webview-resident";
|
||||
import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
|
||||
import { createWorkspaceBrowser } from "@/stores/browser-store";
|
||||
import {
|
||||
buildWorkspaceTabPersistenceKey,
|
||||
@@ -43,8 +40,7 @@ export function mountBrowserAutomationHandler(
|
||||
options: BrowserAutomationHandlerOptions,
|
||||
): () => void {
|
||||
const getHost = options.getHost ?? getDesktopHost;
|
||||
const uninstallCaptureBridge = installResidentBrowserCaptureBridge();
|
||||
const unsubscribe = options.client.on("browser.automation.execute.request", (request) => {
|
||||
return options.client.on("browser.automation.execute.request", (request) => {
|
||||
void handleBrowserAutomationRequest({
|
||||
client: options.client,
|
||||
getHost,
|
||||
@@ -60,10 +56,6 @@ export function mountBrowserAutomationHandler(
|
||||
: {}),
|
||||
});
|
||||
});
|
||||
return () => {
|
||||
unsubscribe();
|
||||
uninstallCaptureBridge();
|
||||
};
|
||||
}
|
||||
|
||||
export function mountBrowserAutomationDaemonClientHandler(
|
||||
@@ -119,6 +111,8 @@ async function handleBrowserAutomationRequest(params: {
|
||||
return;
|
||||
}
|
||||
|
||||
await rememberAgentBrowserTarget({ request, browserHost });
|
||||
|
||||
if (!executeAutomationCommand) {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
@@ -165,11 +159,11 @@ async function openBrowserTabForRequest(params: {
|
||||
BrowserAutomationExecuteRequest["command"],
|
||||
{ command: "new_tab" }
|
||||
>;
|
||||
const workspaceId = request.workspaceId;
|
||||
const workspaceId = request.workspaceId ?? command.args.workspaceId;
|
||||
if (!serverId || !workspaceId) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_unsupported",
|
||||
code: "browser_no_tab",
|
||||
message: "Cannot create a browser tab without a workspace context.",
|
||||
});
|
||||
}
|
||||
@@ -180,17 +174,19 @@ async function openBrowserTabForRequest(params: {
|
||||
if (!workspaceKey) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_unsupported",
|
||||
code: "browser_no_tab",
|
||||
message: "Cannot create a browser tab without a workspace context.",
|
||||
});
|
||||
}
|
||||
useWorkspaceLayoutStore.getState().openTabFocused(workspaceKey, {
|
||||
useWorkspaceLayoutStore.getState().openTabInBackground(workspaceKey, {
|
||||
kind: "browser",
|
||||
browserId,
|
||||
});
|
||||
|
||||
await browserHost?.registerWorkspaceBrowser?.({ browserId, workspaceId });
|
||||
await browserHost?.setWorkspaceActiveBrowser?.({ browserId, workspaceId });
|
||||
if (request.agentId) {
|
||||
await browserHost?.setAgentActiveBrowser?.({ agentId: request.agentId, browserId });
|
||||
}
|
||||
|
||||
if (browserHost?.executeAutomationCommand) {
|
||||
ensureResidentBrowserWebview({ browserId, url: normalizedUrl });
|
||||
@@ -239,7 +235,7 @@ async function waitForBrowserRegistration(params: {
|
||||
agentId: params.request.agentId,
|
||||
cwd: params.request.cwd,
|
||||
workspaceId: params.workspaceId,
|
||||
command: { command: "list_tabs", args: {} },
|
||||
command: { command: "list_tabs", args: { workspaceId: params.workspaceId } },
|
||||
});
|
||||
if (payload.ok && payload.result.command === "list_tabs") {
|
||||
if (payload.result.tabs.some((tab) => tab.browserId === params.browserId)) {
|
||||
@@ -255,6 +251,28 @@ function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function rememberAgentBrowserTarget(input: {
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
browserHost: DesktopHostBridge["browser"] | undefined;
|
||||
}): Promise<void> {
|
||||
if (!input.request.agentId) {
|
||||
return;
|
||||
}
|
||||
const browserId = readRequestBrowserId(input.request);
|
||||
if (!browserId) {
|
||||
return;
|
||||
}
|
||||
await input.browserHost?.setAgentActiveBrowser?.({ agentId: input.request.agentId, browserId });
|
||||
}
|
||||
|
||||
function readRequestBrowserId(request: BrowserAutomationExecuteRequest): string | null {
|
||||
if (request.browserId) {
|
||||
return request.browserId;
|
||||
}
|
||||
const args = request.command.args as { browserId?: unknown };
|
||||
return typeof args.browserId === "string" && args.browserId.length > 0 ? args.browserId : null;
|
||||
}
|
||||
|
||||
function normalizeBridgePayload(
|
||||
requestId: string,
|
||||
payload: BrowserAutomationResponsePayload,
|
||||
|
||||
@@ -12,22 +12,21 @@ import { Pressable, Text, TextInput, View, type StyleProp, type ViewStyle } from
|
||||
import {
|
||||
ArrowLeft,
|
||||
ArrowRight,
|
||||
Camera,
|
||||
ChevronDown,
|
||||
Copy,
|
||||
Maximize,
|
||||
Monitor,
|
||||
MousePointer2,
|
||||
PencilRuler,
|
||||
RotateCw,
|
||||
Smartphone,
|
||||
Tablet,
|
||||
Wrench,
|
||||
X,
|
||||
type LucideIcon,
|
||||
} from "lucide-react-native";
|
||||
import { StyleSheet, useUnistyles, withUnistyles } from "react-native-unistyles";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import * as Clipboard from "expo-clipboard";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
|
||||
import { useToast } from "@/contexts/toast-context";
|
||||
import {
|
||||
@@ -41,7 +40,11 @@ import {
|
||||
useWorkspaceAttachments,
|
||||
useWorkspaceAttachmentsStore,
|
||||
} from "@/attachments/workspace-attachments-store";
|
||||
import type { AttachmentMetadata, BrowserElementAttachment } from "@/attachments/types";
|
||||
import type {
|
||||
AttachmentMetadata,
|
||||
BrowserAnnotationIntent,
|
||||
BrowserElementAttachment,
|
||||
} from "@/attachments/types";
|
||||
import { persistAttachmentFromDataUrl } from "@/attachments/service";
|
||||
import { WORKSPACE_SECONDARY_HEADER_HEIGHT } from "@/constants/layout";
|
||||
import {
|
||||
@@ -49,6 +52,7 @@ import {
|
||||
isElectronRuntime,
|
||||
type DesktopBrowserShortcutEvent,
|
||||
} from "@/desktop/host";
|
||||
import { isDev } from "@/constants/platform";
|
||||
import { useBrowserStore, normalizeWorkspaceBrowserUrl } from "@/stores/browser-store";
|
||||
import {
|
||||
prepareBrowserWebview,
|
||||
@@ -75,14 +79,25 @@ type WebTextInput = TextInput & {
|
||||
getNativeRef?: () => unknown;
|
||||
};
|
||||
|
||||
type BrowserElementSelection = Omit<BrowserElementAttachment, "formatted" | "comment"> & {
|
||||
type BrowserElementSelection = Omit<
|
||||
BrowserElementAttachment,
|
||||
"formatted" | "comment" | "intent"
|
||||
> & {
|
||||
attributes?: Record<string, string>;
|
||||
};
|
||||
|
||||
interface BrowserElementAnnotation {
|
||||
comment: string;
|
||||
intent: BrowserAnnotationIntent;
|
||||
}
|
||||
|
||||
const BROWSER_ANNOTATION_INTENTS: readonly BrowserAnnotationIntent[] = [
|
||||
"fix",
|
||||
"change",
|
||||
"question",
|
||||
"approve",
|
||||
];
|
||||
|
||||
type DeviceSizeId =
|
||||
| "responsive"
|
||||
| "iphone-se"
|
||||
@@ -208,6 +223,10 @@ function formatElementAttachment(
|
||||
const html = truncateText(selection.outerHTML.trim(), 800);
|
||||
const parts: string[] = [];
|
||||
|
||||
if (annotation) {
|
||||
parts.push(`intent: ${annotation.intent}`);
|
||||
}
|
||||
|
||||
if (selection.reactSource?.fileName) {
|
||||
const loc = [
|
||||
selection.reactSource.fileName,
|
||||
@@ -270,6 +289,7 @@ function buildBrowserElementAttachment(
|
||||
parentChain: selection.parentChain,
|
||||
children: selection.children,
|
||||
...(comment ? { comment } : {}),
|
||||
...(annotation ? { intent: annotation.intent } : {}),
|
||||
...(screenshot ? { screenshot } : {}),
|
||||
formatted: formatElementAttachment(selection, annotation),
|
||||
};
|
||||
@@ -621,11 +641,10 @@ export function BrowserPane({
|
||||
const pendingNavigationUrlRef = useRef<string | null>(null);
|
||||
const domReadyRef = useRef(false);
|
||||
const annotationMarkersRef = useRef<BrowserAnnotationMarker[]>([]);
|
||||
const [selectorMode, setSelectorMode] = useState<"annotate" | "screenshot" | null>(null);
|
||||
const selectorActive = selectorMode !== null;
|
||||
const [selectorActive, setSelectorActive] = useState(false);
|
||||
// Which action the active selector performs on click: open the annotation card
|
||||
// ("annotate") or copy a screenshot of the element to the clipboard ("screenshot").
|
||||
const selectorModeRef = useRef<"annotate" | "screenshot">("annotate");
|
||||
// ("annotate") or copy the element to the clipboard ("grab").
|
||||
const selectorModeRef = useRef<"annotate" | "grab">("annotate");
|
||||
const toast = useToast();
|
||||
const toastRef = useRef(toast);
|
||||
toastRef.current = toast;
|
||||
@@ -1055,7 +1074,7 @@ export function BrowserPane({
|
||||
[],
|
||||
);
|
||||
|
||||
const screenshotElementToClipboard = useCallback(
|
||||
const grabElementToClipboard = useCallback(
|
||||
async (selection: BrowserElementSelection) => {
|
||||
const text = formatElementAttachment(selection);
|
||||
const copyElement = getDesktopHost()?.browser?.copyElement;
|
||||
@@ -1068,23 +1087,19 @@ export function BrowserPane({
|
||||
const dataUrl = await captureElement(browserIdRef.current, { x, y, width, height });
|
||||
imageDataUrl = dataUrl ?? undefined;
|
||||
} catch (error) {
|
||||
console.warn("[browser-pane] capture element for screenshot failed", error);
|
||||
console.warn("[browser-pane] capture element for grab failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
const copiedMessage = imageDataUrl
|
||||
? t("workspace.browser.controls.screenshotCopied")
|
||||
: t("workspace.browser.controls.elementCopied");
|
||||
|
||||
// Copy via the main process; the renderer's navigator.clipboard rejects
|
||||
// with NotAllowedError because focus is inside the guest <webview>.
|
||||
if (typeof copyElement === "function") {
|
||||
try {
|
||||
const ok = await copyElement({ text, imageDataUrl });
|
||||
if (ok) {
|
||||
toastRef.current?.show(copiedMessage, { variant: "success" });
|
||||
toastRef.current?.copied(t("workspace.browser.controls.grabElementLabel"));
|
||||
} else {
|
||||
toastRef.current?.error(t("workspace.browser.controls.screenshotFailed"));
|
||||
toastRef.current?.error(t("workspace.browser.controls.grabFailed"));
|
||||
}
|
||||
return;
|
||||
} catch (error) {
|
||||
@@ -1095,12 +1110,10 @@ export function BrowserPane({
|
||||
// Fallback to expo-clipboard (text only) when the bridge is unavailable.
|
||||
try {
|
||||
await Clipboard.setStringAsync(text);
|
||||
toastRef.current?.show(t("workspace.browser.controls.elementCopied"), {
|
||||
variant: "success",
|
||||
});
|
||||
toastRef.current?.copied(t("workspace.browser.controls.grabElementLabel"));
|
||||
} catch (error) {
|
||||
console.warn("[browser-pane] clipboard fallback failed", error);
|
||||
toastRef.current?.error(t("workspace.browser.controls.screenshotFailed"));
|
||||
toastRef.current?.error(t("workspace.browser.controls.grabFailed"));
|
||||
}
|
||||
},
|
||||
[t],
|
||||
@@ -1108,8 +1121,8 @@ export function BrowserPane({
|
||||
|
||||
const handleSelectorResult = useCallback(
|
||||
(selection: BrowserElementSelection) => {
|
||||
if (selectorModeRef.current === "screenshot") {
|
||||
void screenshotElementToClipboard(selection);
|
||||
if (selectorModeRef.current === "grab") {
|
||||
void grabElementToClipboard(selection);
|
||||
return;
|
||||
}
|
||||
pendingScreenshotRef.current = undefined;
|
||||
@@ -1119,7 +1132,7 @@ export function BrowserPane({
|
||||
return undefined;
|
||||
});
|
||||
},
|
||||
[captureElementScreenshot, screenshotElementToClipboard],
|
||||
[captureElementScreenshot, grabElementToClipboard],
|
||||
);
|
||||
|
||||
const submitAnnotation = useCallback(
|
||||
@@ -1142,15 +1155,15 @@ export function BrowserPane({
|
||||
}, []);
|
||||
|
||||
const startElementSelector = useCallback(
|
||||
(mode: "annotate" | "screenshot") => {
|
||||
(mode: "annotate" | "grab") => {
|
||||
const webview = webviewRef.current;
|
||||
if (!webview || !domReadyRef.current) return;
|
||||
// Annotate needs a workspace scope to attach to; screenshot only copies.
|
||||
// Annotate needs a workspace scope to attach to; grab only copies.
|
||||
if (mode === "annotate" && !workspaceAttachmentScopeKey) return;
|
||||
selectorModeRef.current = mode;
|
||||
pendingScreenshotRef.current = undefined;
|
||||
setPendingSelection(null);
|
||||
setSelectorMode(mode);
|
||||
setSelectorActive(true);
|
||||
|
||||
const js = `
|
||||
(function() {
|
||||
@@ -1376,11 +1389,11 @@ export function BrowserPane({
|
||||
const poll = startSelectorResultPolling({
|
||||
webview,
|
||||
onSelection: handleSelectorResult,
|
||||
onDone: () => setSelectorMode(null),
|
||||
onDone: () => setSelectorActive(false),
|
||||
});
|
||||
window.setTimeout(() => {
|
||||
window.clearInterval(poll);
|
||||
setSelectorMode(null);
|
||||
setSelectorActive(false);
|
||||
if (webviewRef.current !== webview || !domReadyRef.current) {
|
||||
return;
|
||||
}
|
||||
@@ -1389,10 +1402,10 @@ export function BrowserPane({
|
||||
return undefined;
|
||||
})
|
||||
.catch(() => {
|
||||
setSelectorMode(null);
|
||||
setSelectorActive(false);
|
||||
});
|
||||
} catch {
|
||||
setSelectorMode(null);
|
||||
setSelectorActive(false);
|
||||
}
|
||||
},
|
||||
[handleSelectorResult, workspaceAttachmentScopeKey],
|
||||
@@ -1400,7 +1413,7 @@ export function BrowserPane({
|
||||
|
||||
const cancelElementSelector = useCallback(() => {
|
||||
const webview = webviewRef.current;
|
||||
setSelectorMode(null);
|
||||
setSelectorActive(false);
|
||||
if (webview && domReadyRef.current) {
|
||||
try {
|
||||
clearWebviewSelector(webview);
|
||||
@@ -1457,12 +1470,12 @@ export function BrowserPane({
|
||||
startElementSelector("annotate");
|
||||
}, [cancelElementSelector, selectorActive, startElementSelector]);
|
||||
|
||||
const handleToggleScreenshot = useCallback(() => {
|
||||
const handleToggleGrab = useCallback(() => {
|
||||
if (selectorActive) {
|
||||
cancelElementSelector();
|
||||
return;
|
||||
}
|
||||
startElementSelector("screenshot");
|
||||
startElementSelector("grab");
|
||||
}, [cancelElementSelector, selectorActive, startElementSelector]);
|
||||
|
||||
const handleOpenDevTools = useCallback(() => {
|
||||
@@ -1508,21 +1521,13 @@ export function BrowserPane({
|
||||
],
|
||||
[browser?.canGoForward],
|
||||
);
|
||||
const annotateIconButtonStyle = useCallback(
|
||||
const selectorIconButtonStyle = useCallback(
|
||||
({ hovered, pressed }: { hovered?: boolean; pressed?: boolean }) => [
|
||||
styles.iconButton,
|
||||
selectorMode === "annotate" && styles.selectorActiveButton,
|
||||
selectorActive && styles.selectorActiveButton,
|
||||
(hovered || pressed) && styles.iconButtonHovered,
|
||||
],
|
||||
[selectorMode],
|
||||
);
|
||||
const screenshotIconButtonStyle = useCallback(
|
||||
({ hovered, pressed }: { hovered?: boolean; pressed?: boolean }) => [
|
||||
styles.iconButton,
|
||||
selectorMode === "screenshot" && styles.selectorActiveButton,
|
||||
(hovered || pressed) && styles.iconButtonHovered,
|
||||
],
|
||||
[selectorMode],
|
||||
[selectorActive],
|
||||
);
|
||||
|
||||
const devicePreset = useMemo(
|
||||
@@ -1628,46 +1633,36 @@ export function BrowserPane({
|
||||
onSelect={setDeviceSizeId}
|
||||
triggerStyle={baseIconButtonStyle}
|
||||
/>
|
||||
<ToolbarButton
|
||||
label={t("workspace.browser.controls.openDevTools")}
|
||||
onPress={handleOpenDevTools}
|
||||
style={baseIconButtonStyle}
|
||||
>
|
||||
<Wrench size={16} color={theme.colors.foregroundMuted} />
|
||||
</ToolbarButton>
|
||||
{isDev ? (
|
||||
<ToolbarButton
|
||||
label={t("workspace.browser.controls.openDevTools")}
|
||||
onPress={handleOpenDevTools}
|
||||
style={baseIconButtonStyle}
|
||||
>
|
||||
<PencilRuler size={16} color={theme.colors.foregroundMuted} />
|
||||
</ToolbarButton>
|
||||
) : null}
|
||||
<ToolbarButton
|
||||
label={
|
||||
selectorMode === "annotate"
|
||||
selectorActive
|
||||
? t("workspace.browser.controls.cancelSelector")
|
||||
: t("workspace.browser.controls.annotateElement")
|
||||
: t("workspace.browser.controls.selectElement")
|
||||
}
|
||||
active={selectorMode === "annotate"}
|
||||
active={selectorActive}
|
||||
onPress={handleToggleElementSelector}
|
||||
style={annotateIconButtonStyle}
|
||||
style={selectorIconButtonStyle}
|
||||
>
|
||||
<MousePointer2
|
||||
size={16}
|
||||
color={
|
||||
selectorMode === "annotate" ? theme.colors.accent : theme.colors.foregroundMuted
|
||||
}
|
||||
color={selectorActive ? theme.colors.accent : theme.colors.foregroundMuted}
|
||||
/>
|
||||
</ToolbarButton>
|
||||
<ToolbarButton
|
||||
label={
|
||||
selectorMode === "screenshot"
|
||||
? t("workspace.browser.controls.cancelSelector")
|
||||
: t("workspace.browser.controls.screenshotElement")
|
||||
}
|
||||
active={selectorMode === "screenshot"}
|
||||
onPress={handleToggleScreenshot}
|
||||
style={screenshotIconButtonStyle}
|
||||
label={t("workspace.browser.controls.grabElement")}
|
||||
onPress={handleToggleGrab}
|
||||
style={baseIconButtonStyle}
|
||||
>
|
||||
<Camera
|
||||
size={16}
|
||||
color={
|
||||
selectorMode === "screenshot" ? theme.colors.accent : theme.colors.foregroundMuted
|
||||
}
|
||||
/>
|
||||
<Copy size={16} color={theme.colors.foregroundMuted} />
|
||||
</ToolbarButton>
|
||||
</View>
|
||||
</View>
|
||||
@@ -1695,6 +1690,45 @@ export function BrowserPane({
|
||||
);
|
||||
}
|
||||
|
||||
const INTENT_LABEL_KEYS: Record<BrowserAnnotationIntent, string> = {
|
||||
fix: "workspace.browser.annotate.intents.fix",
|
||||
change: "workspace.browser.annotate.intents.change",
|
||||
question: "workspace.browser.annotate.intents.question",
|
||||
approve: "workspace.browser.annotate.intents.approve",
|
||||
};
|
||||
|
||||
function IntentChip({
|
||||
active,
|
||||
intent,
|
||||
label,
|
||||
onSelect,
|
||||
}: {
|
||||
active: boolean;
|
||||
intent: BrowserAnnotationIntent;
|
||||
label: string;
|
||||
onSelect: (intent: BrowserAnnotationIntent) => void;
|
||||
}) {
|
||||
const handlePress = useCallback(() => {
|
||||
onSelect(intent);
|
||||
}, [intent, onSelect]);
|
||||
const chipStyle = useMemo(() => [styles.intentChip, active && styles.intentChipActive], [active]);
|
||||
const textStyle = useMemo(
|
||||
() => [styles.intentChipText, active && styles.intentChipTextActive],
|
||||
[active],
|
||||
);
|
||||
const accessibilityState = useMemo(() => ({ selected: active }), [active]);
|
||||
return (
|
||||
<Pressable
|
||||
accessibilityRole="button"
|
||||
accessibilityState={accessibilityState}
|
||||
onPress={handlePress}
|
||||
style={chipStyle}
|
||||
>
|
||||
<Text style={textStyle}>{label}</Text>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
function BrowserElementAnnotationCard({
|
||||
selection,
|
||||
onSubmit,
|
||||
@@ -1706,11 +1740,14 @@ function BrowserElementAnnotationCard({
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const [comment, setComment] = useState("");
|
||||
const [intent, setIntent] = useState<BrowserAnnotationIntent>("fix");
|
||||
const commentRef = useRef(comment);
|
||||
commentRef.current = comment;
|
||||
const intentRef = useRef(intent);
|
||||
intentRef.current = intent;
|
||||
|
||||
const handleSubmit = useCallback(() => {
|
||||
onSubmit({ comment: commentRef.current });
|
||||
onSubmit({ comment: commentRef.current, intent: intentRef.current });
|
||||
}, [onSubmit]);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -1755,6 +1792,17 @@ function BrowserElementAnnotationCard({
|
||||
<Text numberOfLines={1} style={styles.annotationElement}>
|
||||
{elementLabel}
|
||||
</Text>
|
||||
<View style={styles.annotationIntents}>
|
||||
{BROWSER_ANNOTATION_INTENTS.map((option) => (
|
||||
<IntentChip
|
||||
key={option}
|
||||
active={option === intent}
|
||||
intent={option}
|
||||
label={t(INTENT_LABEL_KEYS[option])}
|
||||
onSelect={setIntent}
|
||||
/>
|
||||
))}
|
||||
</View>
|
||||
<ThemedAnnotationInput
|
||||
accessibilityLabel={t("workspace.browser.annotate.placeholder")}
|
||||
autoFocus
|
||||
@@ -1766,12 +1814,24 @@ function BrowserElementAnnotationCard({
|
||||
value={comment}
|
||||
/>
|
||||
<View style={styles.annotationActions}>
|
||||
<Button variant="ghost" size="sm" onPress={onCancel}>
|
||||
{t("workspace.browser.annotate.cancel")}
|
||||
</Button>
|
||||
<Button variant="default" size="sm" onPress={handleSubmit}>
|
||||
{t("workspace.browser.annotate.submit")}
|
||||
</Button>
|
||||
<Pressable
|
||||
accessibilityRole="button"
|
||||
onPress={onCancel}
|
||||
style={styles.annotationSecondaryButton}
|
||||
>
|
||||
<Text style={styles.annotationSecondaryText}>
|
||||
{t("workspace.browser.annotate.cancel")}
|
||||
</Text>
|
||||
</Pressable>
|
||||
<Pressable
|
||||
accessibilityRole="button"
|
||||
onPress={handleSubmit}
|
||||
style={styles.annotationPrimaryButton}
|
||||
>
|
||||
<Text style={styles.annotationPrimaryText}>
|
||||
{t("workspace.browser.annotate.submit")}
|
||||
</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
@@ -1926,7 +1986,31 @@ const styles = StyleSheet.create((theme) => ({
|
||||
annotationElement: {
|
||||
fontSize: theme.fontSize.xs,
|
||||
color: theme.colors.foregroundMuted,
|
||||
marginBottom: theme.spacing[2],
|
||||
},
|
||||
annotationIntents: {
|
||||
flexDirection: "row",
|
||||
flexWrap: "wrap",
|
||||
gap: theme.spacing[1],
|
||||
},
|
||||
intentChip: {
|
||||
paddingHorizontal: theme.spacing[2],
|
||||
paddingVertical: theme.spacing[1],
|
||||
borderRadius: theme.borderRadius.md,
|
||||
borderWidth: 1,
|
||||
borderColor: theme.colors.border,
|
||||
backgroundColor: theme.colors.surface1,
|
||||
},
|
||||
intentChipActive: {
|
||||
borderColor: theme.colors.accent,
|
||||
backgroundColor: `${String(theme.colors.accent)}20`,
|
||||
},
|
||||
intentChipText: {
|
||||
fontSize: theme.fontSize.xs,
|
||||
fontWeight: "500",
|
||||
color: theme.colors.foregroundMuted,
|
||||
},
|
||||
intentChipTextActive: {
|
||||
color: theme.colors.accent,
|
||||
},
|
||||
annotationInput: {
|
||||
minHeight: 64,
|
||||
@@ -1945,6 +2029,27 @@ const styles = StyleSheet.create((theme) => ({
|
||||
justifyContent: "flex-end",
|
||||
gap: theme.spacing[2],
|
||||
},
|
||||
annotationSecondaryButton: {
|
||||
paddingHorizontal: theme.spacing[3],
|
||||
paddingVertical: theme.spacing[2],
|
||||
borderRadius: theme.borderRadius.md,
|
||||
},
|
||||
annotationSecondaryText: {
|
||||
fontSize: theme.fontSize.sm,
|
||||
fontWeight: "500",
|
||||
color: theme.colors.foregroundMuted,
|
||||
},
|
||||
annotationPrimaryButton: {
|
||||
paddingHorizontal: theme.spacing[3],
|
||||
paddingVertical: theme.spacing[2],
|
||||
borderRadius: theme.borderRadius.md,
|
||||
backgroundColor: theme.colors.accent,
|
||||
},
|
||||
annotationPrimaryText: {
|
||||
fontSize: theme.fontSize.sm,
|
||||
fontWeight: "600",
|
||||
color: theme.colors.accentForeground ?? "#ffffff",
|
||||
},
|
||||
unavailableState: {
|
||||
flex: 1,
|
||||
alignItems: "center",
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
cancelResidentBrowserWebviewPixelCapture,
|
||||
clearResidentBrowserWebviewsForTests,
|
||||
ensureResidentBrowserWebview,
|
||||
prepareBrowserWebview,
|
||||
prepareResidentBrowserWebviewForPixelCapture,
|
||||
releaseResidentBrowserWebview,
|
||||
removeResidentBrowserWebview,
|
||||
restoreResidentBrowserWebviewAfterPixelCapture,
|
||||
takeResidentBrowserWebview,
|
||||
} from "./browser-webview-resident";
|
||||
|
||||
@@ -26,21 +22,12 @@ describe("resident browser webviews", () => {
|
||||
|
||||
expect(host.children).toHaveLength(0);
|
||||
expect(webview.isConnected).toBe(true);
|
||||
expect(webview.style.display).toBe("inline-flex");
|
||||
expect(webview.style.width).toBe("1280px");
|
||||
expect(webview.style.height).toBe("800px");
|
||||
expect(webview.style.position).toBe("absolute");
|
||||
expect(webview.style.left).toBe("0px");
|
||||
expect(webview.style.top).toBe("0px");
|
||||
expect(webview.style.zIndex).toBe("0");
|
||||
|
||||
const reused = takeResidentBrowserWebview("browser-a");
|
||||
|
||||
expect(reused).toBe(webview);
|
||||
expect(webview.style.position).toBe("");
|
||||
expect(webview.style.left).toBe("");
|
||||
expect(webview.style.top).toBe("");
|
||||
expect(webview.style.zIndex).toBe("");
|
||||
expect(takeResidentBrowserWebview("browser-a")).toBeNull();
|
||||
});
|
||||
|
||||
@@ -70,231 +57,4 @@ describe("resident browser webviews", () => {
|
||||
expect(webview?.isConnected).toBe(false);
|
||||
expect(takeResidentBrowserWebview("browser-closed")).toBeNull();
|
||||
});
|
||||
|
||||
it("temporarily makes resident webviews paintable for pixel capture", async () => {
|
||||
const webview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-capture",
|
||||
url: "https://example.com",
|
||||
});
|
||||
if (!webview) {
|
||||
throw new Error("Expected resident browser webview");
|
||||
}
|
||||
|
||||
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-capture",
|
||||
});
|
||||
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||
|
||||
expect(preparation.token).toBe("capture-1");
|
||||
expect(host?.style.left).toBe("0px");
|
||||
expect(host?.style.top).toBe("0px");
|
||||
expect(host?.style.width).toBe("1px");
|
||||
expect(host?.style.height).toBe("1px");
|
||||
expect(host?.style.overflow).toBe("hidden");
|
||||
expect(host?.style.opacity).toBe("1");
|
||||
expect(host?.style.pointerEvents).toBe("none");
|
||||
expect(webview.style.display).toBe("inline-flex");
|
||||
expect(webview.style.width).toBe("1280px");
|
||||
expect(webview.style.height).toBe("800px");
|
||||
expect(webview.style.position).toBe("absolute");
|
||||
expect(webview.style.left).toBe("0px");
|
||||
expect(webview.style.top).toBe("0px");
|
||||
expect(webview.style.zIndex).toBe("2");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
|
||||
|
||||
expect(host?.style.left).toBe("-20000px");
|
||||
expect(host?.style.width).toBe("1280px");
|
||||
expect(host?.style.height).toBe("800px");
|
||||
expect(host?.style.opacity).toBe("0");
|
||||
expect(webview.style.zIndex).toBe("0");
|
||||
});
|
||||
|
||||
it("parks resident webviews as an overlapping stack and raises the capture target", async () => {
|
||||
const firstWebview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-first",
|
||||
url: "https://example.com/first",
|
||||
});
|
||||
const secondWebview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-second",
|
||||
url: "https://example.com/second",
|
||||
});
|
||||
if (!firstWebview || !secondWebview) {
|
||||
throw new Error("Expected resident browser webviews");
|
||||
}
|
||||
|
||||
expect(firstWebview.style.position).toBe("absolute");
|
||||
expect(firstWebview.style.left).toBe("0px");
|
||||
expect(firstWebview.style.top).toBe("0px");
|
||||
expect(firstWebview.style.zIndex).toBe("0");
|
||||
expect(secondWebview.style.position).toBe("absolute");
|
||||
expect(secondWebview.style.left).toBe("0px");
|
||||
expect(secondWebview.style.top).toBe("0px");
|
||||
expect(secondWebview.style.zIndex).toBe("0");
|
||||
|
||||
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-second",
|
||||
});
|
||||
|
||||
expect(firstWebview.style.zIndex).toBe("0");
|
||||
expect(secondWebview.style.zIndex).toBe("2");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
|
||||
|
||||
expect(firstWebview.style.zIndex).toBe("0");
|
||||
expect(secondWebview.style.zIndex).toBe("0");
|
||||
});
|
||||
|
||||
it("prepares the physically resident webview when a focused duplicate exists outside the resident host", async () => {
|
||||
const residentWebview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-focused-undisplayed",
|
||||
url: "https://example.com/resident",
|
||||
});
|
||||
if (!residentWebview) {
|
||||
throw new Error("Expected resident browser webview");
|
||||
}
|
||||
const visibleHost = document.createElement("div");
|
||||
const duplicateWebview = document.createElement("webview");
|
||||
prepareBrowserWebview(duplicateWebview, {
|
||||
browserId: "browser-focused-undisplayed",
|
||||
initialUrl: "https://example.com/duplicate",
|
||||
});
|
||||
visibleHost.appendChild(duplicateWebview);
|
||||
document.body.prepend(visibleHost);
|
||||
|
||||
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-focused-undisplayed",
|
||||
});
|
||||
const residentHost = document.getElementById("paseo-browser-resident-webviews");
|
||||
|
||||
expect(residentHost?.style.left).toBe("0px");
|
||||
expect(residentHost?.style.width).toBe("1px");
|
||||
expect(residentWebview.style.zIndex).toBe("2");
|
||||
expect(duplicateWebview.style.zIndex).toBe("");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
|
||||
});
|
||||
|
||||
it("does not apply resident prep styles for a genuinely visible pane webview", async () => {
|
||||
const visibleHost = document.createElement("div");
|
||||
const visibleWebview = document.createElement("webview");
|
||||
prepareBrowserWebview(visibleWebview, {
|
||||
browserId: "browser-visible-pane",
|
||||
initialUrl: "https://example.com",
|
||||
});
|
||||
visibleWebview.style.display = "flex";
|
||||
visibleWebview.style.width = "100%";
|
||||
visibleWebview.style.height = "100%";
|
||||
visibleHost.appendChild(visibleWebview);
|
||||
document.body.appendChild(visibleHost);
|
||||
|
||||
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-visible-pane",
|
||||
});
|
||||
const residentHost = document.getElementById("paseo-browser-resident-webviews");
|
||||
|
||||
expect(residentHost?.style.left).toBe("-20000px");
|
||||
expect(residentHost?.style.width).toBe("1280px");
|
||||
expect(visibleWebview.style.position).toBe("");
|
||||
expect(visibleWebview.style.zIndex).toBe("");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
|
||||
});
|
||||
|
||||
it("clears capture preparation when a resident webview is taken visible", async () => {
|
||||
const webview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-visible",
|
||||
url: "https://example.com",
|
||||
});
|
||||
if (!webview) {
|
||||
throw new Error("Expected resident browser webview");
|
||||
}
|
||||
|
||||
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-visible",
|
||||
});
|
||||
const visibleWebview = takeResidentBrowserWebview("browser-visible");
|
||||
|
||||
expect(visibleWebview).toBe(webview);
|
||||
expect(webview.style.position).toBe("");
|
||||
expect(webview.style.left).toBe("");
|
||||
expect(webview.style.top).toBe("");
|
||||
expect(webview.style.zIndex).toBe("");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
|
||||
|
||||
expect(webview.style.position).toBe("");
|
||||
expect(webview.style.left).toBe("");
|
||||
expect(webview.style.top).toBe("");
|
||||
expect(webview.style.zIndex).toBe("");
|
||||
});
|
||||
|
||||
it("keeps the resident host paintable until every capture token is restored", async () => {
|
||||
ensureResidentBrowserWebview({
|
||||
browserId: "browser-overlap",
|
||||
url: "https://example.com",
|
||||
});
|
||||
|
||||
const first = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-overlap",
|
||||
});
|
||||
const second = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-overlap",
|
||||
});
|
||||
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(first);
|
||||
|
||||
expect(host?.style.left).toBe("0px");
|
||||
expect(host?.style.width).toBe("1px");
|
||||
expect(host?.style.opacity).toBe("1");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(second);
|
||||
|
||||
expect(host?.style.left).toBe("-20000px");
|
||||
expect(host?.style.width).toBe("1280px");
|
||||
expect(host?.style.opacity).toBe("0");
|
||||
});
|
||||
|
||||
it("cancels an in-flight pixel capture preparation by request id", async () => {
|
||||
ensureResidentBrowserWebview({
|
||||
browserId: "browser-cancel",
|
||||
url: "https://example.com",
|
||||
});
|
||||
|
||||
const preparation = prepareResidentBrowserWebviewForPixelCapture({
|
||||
requestId: "prepare-1",
|
||||
browserId: "browser-cancel",
|
||||
});
|
||||
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||
expect(host?.style.left).toBe("0px");
|
||||
expect(host?.style.opacity).toBe("1");
|
||||
|
||||
await cancelResidentBrowserWebviewPixelCapture({ requestId: "prepare-1" });
|
||||
|
||||
await expect(preparation).rejects.toThrow("Browser pixel capture preparation was canceled.");
|
||||
expect(host?.style.left).toBe("-20000px");
|
||||
expect(host?.style.width).toBe("1280px");
|
||||
expect(host?.style.opacity).toBe("0");
|
||||
});
|
||||
|
||||
it("parks the resident host when a prepared browser tab is removed", async () => {
|
||||
const webview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-detached",
|
||||
url: "https://example.com",
|
||||
});
|
||||
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-detached",
|
||||
});
|
||||
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||
|
||||
removeResidentBrowserWebview("browser-detached");
|
||||
|
||||
expect(webview?.isConnected).toBe(false);
|
||||
expect(host?.style.left).toBe("-20000px");
|
||||
expect(host?.style.width).toBe("1280px");
|
||||
expect(host?.style.opacity).toBe("0");
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
|
||||
expect(host?.style.left).toBe("-20000px");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,28 +1,14 @@
|
||||
import { getDesktopHost } from "@/desktop/host";
|
||||
|
||||
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
|
||||
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
|
||||
const RESIDENT_VIEWPORT_WIDTH = 1280;
|
||||
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
||||
|
||||
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
|
||||
const activeCapturePreparations = new Map<string, ActiveCapturePreparation>();
|
||||
|
||||
let captureBridgeInstallCount = 0;
|
||||
let captureBridgeDisposer: (() => void) | null = null;
|
||||
let nextCapturePreparationId = 0;
|
||||
|
||||
interface BrowserWebviewElement extends HTMLElement {
|
||||
src: string;
|
||||
}
|
||||
|
||||
interface ActiveCapturePreparation {
|
||||
browserId: string;
|
||||
requestId?: string;
|
||||
preparesResidentHost: boolean;
|
||||
webview: HTMLElement;
|
||||
}
|
||||
|
||||
function trimNonEmpty(value: string | null | undefined): string | null {
|
||||
if (typeof value !== "string") {
|
||||
return null;
|
||||
@@ -35,7 +21,14 @@ function readDocument(): Document | null {
|
||||
return typeof document === "undefined" ? null : document;
|
||||
}
|
||||
|
||||
function applyResidentHostParkingStyle(host: HTMLElement): void {
|
||||
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
||||
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const host = ownerDocument.createElement("div");
|
||||
host.id = RESIDENT_BROWSER_HOST_ID;
|
||||
host.setAttribute("aria-hidden", "true");
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "-20000px";
|
||||
@@ -45,194 +38,25 @@ function applyResidentHostParkingStyle(host: HTMLElement): void {
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "0";
|
||||
host.style.pointerEvents = "none";
|
||||
host.style.zIndex = "";
|
||||
host.style.clipPath = "";
|
||||
host.style.visibility = "";
|
||||
host.style.transform = "";
|
||||
}
|
||||
|
||||
function applyResidentHostCaptureStyle(host: HTMLElement): void {
|
||||
host.setAttribute("aria-hidden", "true");
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "0";
|
||||
host.style.top = "0";
|
||||
host.style.width = "1px";
|
||||
host.style.height = "1px";
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "1";
|
||||
host.style.pointerEvents = "none";
|
||||
host.style.zIndex = "1";
|
||||
host.style.clipPath = "";
|
||||
host.style.visibility = "";
|
||||
host.style.transform = "";
|
||||
}
|
||||
|
||||
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
||||
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const host = ownerDocument.createElement("div");
|
||||
host.id = RESIDENT_BROWSER_HOST_ID;
|
||||
applyResidentHostParkingStyle(host);
|
||||
ownerDocument.body.appendChild(host);
|
||||
return host;
|
||||
}
|
||||
|
||||
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
|
||||
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
|
||||
if (!(element instanceof HTMLElement)) {
|
||||
continue;
|
||||
}
|
||||
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
|
||||
return element;
|
||||
return element as HTMLElement;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function findBrowserWebviewForPixelCapture(
|
||||
browserId: string,
|
||||
ownerDocument: Document,
|
||||
): HTMLElement | null {
|
||||
const resident = residentWebviewsByBrowserId.get(browserId) ?? null;
|
||||
if (resident?.isConnected) {
|
||||
return resident;
|
||||
}
|
||||
return findBrowserWebview(browserId, ownerDocument);
|
||||
}
|
||||
|
||||
function applyResidentWebviewStyle(webview: HTMLElement): void {
|
||||
webview.style.display = "inline-flex";
|
||||
webview.style.flex = "0 0 auto";
|
||||
webview.style.display = "block";
|
||||
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
webview.style.border = "0";
|
||||
webview.style.background = "transparent";
|
||||
webview.style.position = "absolute";
|
||||
webview.style.left = "0";
|
||||
webview.style.top = "0";
|
||||
webview.style.marginTop = "0";
|
||||
webview.style.zIndex = "0";
|
||||
}
|
||||
|
||||
function clearResidentWebviewParkingStyle(webview: HTMLElement): void {
|
||||
webview.style.position = "";
|
||||
webview.style.left = "";
|
||||
webview.style.top = "";
|
||||
webview.style.marginTop = "";
|
||||
webview.style.zIndex = "";
|
||||
}
|
||||
|
||||
function residentWebviewChildren(host: HTMLElement): HTMLElement[] {
|
||||
const webviews: HTMLElement[] = [];
|
||||
for (const element of host.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
|
||||
if (element instanceof HTMLElement) {
|
||||
webviews.push(element);
|
||||
}
|
||||
}
|
||||
return webviews;
|
||||
}
|
||||
|
||||
function raiseResidentWebviewForCapture(host: HTMLElement, target: HTMLElement): void {
|
||||
for (const webview of residentWebviewChildren(host)) {
|
||||
applyResidentWebviewStyle(webview);
|
||||
}
|
||||
applyResidentWebviewStyle(target);
|
||||
target.style.zIndex = "2";
|
||||
}
|
||||
|
||||
function nextAnimationFrame(): Promise<void> {
|
||||
if (typeof requestAnimationFrame !== "function") {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
requestAnimationFrame(() => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForCapturePaint(webview: HTMLElement): Promise<void> {
|
||||
await nextAnimationFrame();
|
||||
await nextAnimationFrame();
|
||||
webview.getBoundingClientRect();
|
||||
}
|
||||
|
||||
function activeCaptureTokenFor(input: { token?: string; requestId?: string }): string | null {
|
||||
const token = trimNonEmpty(input.token);
|
||||
if (token && activeCapturePreparations.has(token)) {
|
||||
return token;
|
||||
}
|
||||
const requestId = trimNonEmpty(input.requestId);
|
||||
if (!requestId) {
|
||||
return null;
|
||||
}
|
||||
for (const [candidateToken, preparation] of activeCapturePreparations.entries()) {
|
||||
if (preparation.requestId === requestId) {
|
||||
return candidateToken;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function latestActiveResidentHostPreparation(): ActiveCapturePreparation | null {
|
||||
let latest: ActiveCapturePreparation | null = null;
|
||||
for (const preparation of activeCapturePreparations.values()) {
|
||||
if (preparation.preparesResidentHost) {
|
||||
latest = preparation;
|
||||
}
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
|
||||
function syncResidentHostCaptureState(): void {
|
||||
const host = readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||
if (!(host instanceof HTMLElement)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const latest = latestActiveResidentHostPreparation();
|
||||
if (latest) {
|
||||
applyResidentHostCaptureStyle(host);
|
||||
raiseResidentWebviewForCapture(host, latest.webview);
|
||||
return;
|
||||
}
|
||||
|
||||
applyResidentHostParkingStyle(host);
|
||||
for (const webview of residentWebviewChildren(host)) {
|
||||
applyResidentWebviewStyle(webview);
|
||||
}
|
||||
}
|
||||
|
||||
function releaseCapturePreparationToken(token: string): void {
|
||||
const preparation = activeCapturePreparations.get(token);
|
||||
if (!preparation) {
|
||||
return;
|
||||
}
|
||||
activeCapturePreparations.delete(token);
|
||||
if (preparation.preparesResidentHost) {
|
||||
const host = readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||
if (host instanceof HTMLElement && preparation.webview.parentElement === host) {
|
||||
applyResidentWebviewStyle(preparation.webview);
|
||||
}
|
||||
syncResidentHostCaptureState();
|
||||
}
|
||||
}
|
||||
|
||||
function releaseCapturePreparationsForBrowser(browserId: string): void {
|
||||
for (const [token, preparation] of activeCapturePreparations.entries()) {
|
||||
if (preparation.browserId === browserId) {
|
||||
activeCapturePreparations.delete(token);
|
||||
}
|
||||
}
|
||||
syncResidentHostCaptureState();
|
||||
}
|
||||
|
||||
function releaseAllCapturePreparations(): void {
|
||||
activeCapturePreparations.clear();
|
||||
syncResidentHostCaptureState();
|
||||
}
|
||||
|
||||
export function prepareBrowserWebview(
|
||||
@@ -269,9 +93,6 @@ export function ensureResidentBrowserWebview(input: {
|
||||
|
||||
const existing = findBrowserWebview(browserId, ownerDocument);
|
||||
if (existing) {
|
||||
if (existing.parentElement?.id === RESIDENT_BROWSER_HOST_ID) {
|
||||
residentWebviewsByBrowserId.set(browserId, existing);
|
||||
}
|
||||
return existing;
|
||||
}
|
||||
|
||||
@@ -293,8 +114,6 @@ export function takeResidentBrowserWebview(browserId: string): HTMLElement | nul
|
||||
}
|
||||
|
||||
residentWebviewsByBrowserId.delete(normalizedBrowserId);
|
||||
releaseCapturePreparationsForBrowser(normalizedBrowserId);
|
||||
clearResidentWebviewParkingStyle(webview);
|
||||
return webview;
|
||||
}
|
||||
|
||||
@@ -314,98 +133,6 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl
|
||||
getResidentBrowserHost(ownerDocument).appendChild(webview);
|
||||
}
|
||||
|
||||
export async function prepareResidentBrowserWebviewForPixelCapture(input: {
|
||||
browserId: string;
|
||||
requestId?: string;
|
||||
}): Promise<{ token: string }> {
|
||||
const browserId = trimNonEmpty(input.browserId);
|
||||
if (!browserId) {
|
||||
throw new Error("Browser id is required for pixel capture preparation.");
|
||||
}
|
||||
const ownerDocument = readDocument();
|
||||
if (!ownerDocument) {
|
||||
throw new Error("Browser pixel capture preparation requires a document.");
|
||||
}
|
||||
|
||||
const host = getResidentBrowserHost(ownerDocument);
|
||||
const webview = findBrowserWebviewForPixelCapture(browserId, ownerDocument);
|
||||
if (!webview) {
|
||||
throw new Error(`Browser webview ${browserId} is not mounted.`);
|
||||
}
|
||||
|
||||
const token = `capture-${++nextCapturePreparationId}`;
|
||||
const preparesResidentHost = webview.parentElement === host;
|
||||
const requestId = trimNonEmpty(input.requestId);
|
||||
activeCapturePreparations.set(token, {
|
||||
browserId,
|
||||
...(requestId ? { requestId } : {}),
|
||||
preparesResidentHost,
|
||||
webview,
|
||||
});
|
||||
try {
|
||||
if (preparesResidentHost) {
|
||||
applyResidentHostCaptureStyle(host);
|
||||
raiseResidentWebviewForCapture(host, webview);
|
||||
}
|
||||
await waitForCapturePaint(webview);
|
||||
if (!activeCapturePreparations.has(token)) {
|
||||
throw new Error("Browser pixel capture preparation was canceled.");
|
||||
}
|
||||
return { token };
|
||||
} catch (error) {
|
||||
releaseCapturePreparationToken(token);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function restoreResidentBrowserWebviewAfterPixelCapture(input: {
|
||||
token: string;
|
||||
}): Promise<void> {
|
||||
releaseCapturePreparationToken(input.token);
|
||||
}
|
||||
|
||||
export async function cancelResidentBrowserWebviewPixelCapture(input: {
|
||||
requestId?: string;
|
||||
token?: string;
|
||||
}): Promise<void> {
|
||||
const token = activeCaptureTokenFor(input);
|
||||
if (!token) {
|
||||
return;
|
||||
}
|
||||
releaseCapturePreparationToken(token);
|
||||
}
|
||||
|
||||
export function installResidentBrowserCaptureBridge(): () => void {
|
||||
captureBridgeInstallCount += 1;
|
||||
if (!captureBridgeDisposer) {
|
||||
const browserBridge = getDesktopHost()?.browser;
|
||||
const disposePrepare = browserBridge?.onPrepareForPixelCapture?.(
|
||||
prepareResidentBrowserWebviewForPixelCapture,
|
||||
);
|
||||
const disposeRestore = browserBridge?.onRestorePixelCapture?.(
|
||||
restoreResidentBrowserWebviewAfterPixelCapture,
|
||||
);
|
||||
const disposeCancel = browserBridge?.onCancelPixelCapture?.(
|
||||
cancelResidentBrowserWebviewPixelCapture,
|
||||
);
|
||||
captureBridgeDisposer = () => {
|
||||
disposePrepare?.();
|
||||
disposeRestore?.();
|
||||
disposeCancel?.();
|
||||
};
|
||||
}
|
||||
|
||||
return () => {
|
||||
captureBridgeInstallCount = Math.max(0, captureBridgeInstallCount - 1);
|
||||
if (captureBridgeInstallCount > 0) {
|
||||
return;
|
||||
}
|
||||
captureBridgeDisposer?.();
|
||||
captureBridgeDisposer = null;
|
||||
releaseAllCapturePreparations();
|
||||
};
|
||||
}
|
||||
|
||||
export function removeResidentBrowserWebview(browserId: string): void {
|
||||
const normalizedBrowserId = trimNonEmpty(browserId);
|
||||
if (!normalizedBrowserId) {
|
||||
@@ -414,7 +141,6 @@ export function removeResidentBrowserWebview(browserId: string): void {
|
||||
|
||||
const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
|
||||
residentWebviewsByBrowserId.delete(normalizedBrowserId);
|
||||
releaseCapturePreparationsForBrowser(normalizedBrowserId);
|
||||
resident?.remove();
|
||||
}
|
||||
|
||||
@@ -423,7 +149,5 @@ export function clearResidentBrowserWebviewsForTests(): void {
|
||||
webview.remove();
|
||||
}
|
||||
residentWebviewsByBrowserId.clear();
|
||||
releaseAllCapturePreparations();
|
||||
nextCapturePreparationId = 0;
|
||||
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ export function getAttachmentKey(attachment: WorkspaceComposerAttachment): strin
|
||||
tag: attachment.attachment.tag,
|
||||
text: attachment.attachment.text,
|
||||
html: attachment.attachment.outerHTML,
|
||||
intent: attachment.attachment.intent ?? null,
|
||||
comment: attachment.attachment.comment ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -37,10 +37,10 @@ const CATALOG_DATA = [
|
||||
title: "Auggie CLI",
|
||||
description:
|
||||
"Augment Code's powerful software agent, backed by industry-leading context engine",
|
||||
version: "0.32.0",
|
||||
version: "0.31.0",
|
||||
iconId: "auggie",
|
||||
installLink: "https://www.augmentcode.com/",
|
||||
command: ["npx", "-y", "@augmentcode/auggie@0.32.0", "--acp"],
|
||||
command: ["npx", "-y", "@augmentcode/auggie@0.31.0", "--acp"],
|
||||
env: {
|
||||
AUGMENT_DISABLE_AUTO_UPDATE: "1",
|
||||
},
|
||||
@@ -68,10 +68,10 @@ const CATALOG_DATA = [
|
||||
id: "codebuddy-code",
|
||||
title: "Codebuddy Code",
|
||||
description: "Tencent Cloud's official intelligent coding tool",
|
||||
version: "2.115.0",
|
||||
version: "2.114.0",
|
||||
iconId: "codebuddy-code",
|
||||
installLink: "https://www.codebuddy.cn/cli/",
|
||||
command: ["npx", "-y", "@tencent-ai/codebuddy-code@2.115.0", "--acp"],
|
||||
command: ["npx", "-y", "@tencent-ai/codebuddy-code@2.114.0", "--acp"],
|
||||
},
|
||||
{
|
||||
id: "codewhale",
|
||||
@@ -140,29 +140,29 @@ const CATALOG_DATA = [
|
||||
id: "dimcode",
|
||||
title: "DimCode",
|
||||
description: "A coding agent that puts leading models at your command.",
|
||||
version: "0.2.17",
|
||||
version: "0.2.12",
|
||||
iconId: "dimcode",
|
||||
installLink: "https://dimcode.dev/docs/acp.html",
|
||||
command: ["npx", "-y", "dimcode@0.2.17", "acp"],
|
||||
command: ["npx", "-y", "dimcode@0.2.12", "acp"],
|
||||
},
|
||||
{
|
||||
id: "dirac",
|
||||
title: "Dirac",
|
||||
description:
|
||||
"Reduces API costs by more than 50%, produces better and faster work. Uses Hash anchored parallel edits, AST manipulation and a whole lot of neat optimizations. Fully Open Source.",
|
||||
version: "0.4.13",
|
||||
version: "0.4.12",
|
||||
iconId: "dirac",
|
||||
installLink: "https://dirac.run",
|
||||
command: ["npx", "-y", "dirac-cli@0.4.13", "--acp"],
|
||||
command: ["npx", "-y", "dirac-cli@0.4.12", "--acp"],
|
||||
},
|
||||
{
|
||||
id: "factory-droid",
|
||||
title: "Factory Droid",
|
||||
description: "Factory Droid - AI coding agent powered by Factory AI",
|
||||
version: "0.164.0",
|
||||
version: "0.161.0",
|
||||
iconId: "factory-droid",
|
||||
installLink: "https://factory.ai/product/cli",
|
||||
command: ["npx", "-y", "droid@0.164.0", "exec", "--output-format", "acp-daemon"],
|
||||
command: ["npx", "-y", "droid@0.161.0", "exec", "--output-format", "acp-daemon"],
|
||||
env: {
|
||||
DROID_DISABLE_AUTO_UPDATE: "true",
|
||||
FACTORY_DROID_AUTO_UPDATE_ENABLED: "false",
|
||||
@@ -173,10 +173,10 @@ const CATALOG_DATA = [
|
||||
id: "fast-agent",
|
||||
title: "fast-agent",
|
||||
description: "Code and build agents with comprehensive multi-provider support",
|
||||
version: "0.8.3",
|
||||
version: "0.8.1",
|
||||
iconId: "fast-agent",
|
||||
installLink: "https://fast-agent.ai/acp/",
|
||||
command: ["uvx", "--from", "fast-agent-acp==0.8.3", "fast-agent-acp", "-x"],
|
||||
command: ["uvx", "--from", "fast-agent-acp==0.8.1", "fast-agent-acp", "-x"],
|
||||
},
|
||||
{
|
||||
id: "gemini",
|
||||
@@ -284,10 +284,10 @@ const CATALOG_DATA = [
|
||||
id: "nova",
|
||||
title: "Nova",
|
||||
description: "Nova by Compass AI - a fully-fledged software engineer at your command",
|
||||
version: "1.1.24",
|
||||
version: "1.1.22",
|
||||
iconId: "nova",
|
||||
installLink: "https://www.compassap.ai/portfolio/nova.html",
|
||||
command: ["npx", "-y", "@compass-ai/nova@1.1.24", "acp"],
|
||||
command: ["npx", "-y", "@compass-ai/nova@1.1.22", "acp"],
|
||||
},
|
||||
{
|
||||
id: "poolside",
|
||||
@@ -302,19 +302,19 @@ const CATALOG_DATA = [
|
||||
id: "qoder",
|
||||
title: "Qoder CLI",
|
||||
description: "AI coding assistant with agentic capabilities",
|
||||
version: "1.0.36",
|
||||
version: "1.0.33",
|
||||
iconId: "qoder",
|
||||
installLink: "https://qoder.com",
|
||||
command: ["npx", "-y", "@qoder-ai/qodercli@1.0.36", "--acp"],
|
||||
command: ["npx", "-y", "@qoder-ai/qodercli@1.0.33", "--acp"],
|
||||
},
|
||||
{
|
||||
id: "qwen-code",
|
||||
title: "Qwen Code",
|
||||
description: "Alibaba's Qwen coding assistant",
|
||||
version: "0.19.5",
|
||||
version: "0.19.3",
|
||||
iconId: "qwen-code",
|
||||
installLink: "https://qwenlm.github.io/qwen-code-docs/en/users/overview",
|
||||
command: ["npx", "-y", "@qwen-code/qwen-code@0.19.5", "--acp", "--experimental-skills"],
|
||||
command: ["npx", "-y", "@qwen-code/qwen-code@0.19.3", "--acp", "--experimental-skills"],
|
||||
},
|
||||
{
|
||||
id: "sigit",
|
||||
|
||||
@@ -121,10 +121,6 @@ export interface DesktopBrowserShortcutEvent {
|
||||
action: "focus-url";
|
||||
}
|
||||
|
||||
export interface DesktopBrowserPixelCapturePreparation {
|
||||
token: string;
|
||||
}
|
||||
|
||||
export interface DesktopBrowserNewTabRequestEvent {
|
||||
sourceBrowserId: string;
|
||||
url: string;
|
||||
@@ -136,6 +132,7 @@ export interface DesktopBrowserBridge {
|
||||
workspaceId: string;
|
||||
browserId: string | null;
|
||||
}) => Promise<void>;
|
||||
setAgentActiveBrowser?: (input: { agentId: string; browserId: string | null }) => Promise<void>;
|
||||
openDevTools?: (browserId: string) => Promise<unknown>;
|
||||
clearPartition?: (browserId: string) => Promise<void>;
|
||||
executeAutomationCommand?: (
|
||||
@@ -148,18 +145,6 @@ export interface DesktopBrowserBridge {
|
||||
) => Promise<string | null>;
|
||||
/** Copy element text and/or an image to the system clipboard from main. */
|
||||
copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
|
||||
onPrepareForPixelCapture?: (
|
||||
handler: (input: {
|
||||
requestId: string;
|
||||
browserId: string;
|
||||
}) => Promise<DesktopBrowserPixelCapturePreparation>,
|
||||
) => () => void;
|
||||
onRestorePixelCapture?: (
|
||||
handler: (input: DesktopBrowserPixelCapturePreparation) => Promise<void>,
|
||||
) => () => void;
|
||||
onCancelPixelCapture?: (
|
||||
handler: (input: { requestId?: string; token?: string }) => Promise<void>,
|
||||
) => () => void;
|
||||
}
|
||||
|
||||
export interface DesktopInvokeBridge {
|
||||
|
||||
@@ -422,17 +422,22 @@ export const ar: TranslationResources = {
|
||||
enterUrl: "أدخل URL",
|
||||
openDevTools: "افتح أدوات تطوير المتصفح",
|
||||
cancelSelector: "إلغاء محدد العنصر",
|
||||
annotateElement: "التعليق على العنصر",
|
||||
screenshotElement: "لقطة للعنصر",
|
||||
screenshotCopied: "تم نسخ لقطة الشاشة إلى الحافظة",
|
||||
elementCopied: "تم نسخ العنصر إلى الحافظة",
|
||||
screenshotFailed: "تعذّر نسخ لقطة الشاشة",
|
||||
selectElement: "حدد العنصر",
|
||||
grabElement: "نسخ العنصر إلى الحافظة",
|
||||
grabElementLabel: "العنصر",
|
||||
grabFailed: "تعذّر نسخ العنصر",
|
||||
},
|
||||
annotate: {
|
||||
title: "التعليق على العنصر",
|
||||
placeholder: "رسالة إلى الوكيل حول هذا العنصر…",
|
||||
title: "إرسال ملاحظات إلى الوكيل",
|
||||
placeholder: "صف ما الذي ينبغي تغييره…",
|
||||
submit: "إرفاق",
|
||||
cancel: "إلغاء",
|
||||
intents: {
|
||||
fix: "إصلاح",
|
||||
change: "تغيير",
|
||||
question: "سؤال",
|
||||
approve: "موافقة",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "حجم الجهاز",
|
||||
|
||||
@@ -422,17 +422,22 @@ export const en = {
|
||||
enterUrl: "Enter URL",
|
||||
openDevTools: "Open browser dev tools",
|
||||
cancelSelector: "Cancel element selector",
|
||||
annotateElement: "Annotate element",
|
||||
screenshotElement: "Screenshot element",
|
||||
screenshotCopied: "Copied screenshot to clipboard",
|
||||
elementCopied: "Copied element to clipboard",
|
||||
screenshotFailed: "Couldn't copy screenshot",
|
||||
selectElement: "Select element",
|
||||
grabElement: "Copy element to clipboard",
|
||||
grabElementLabel: "element",
|
||||
grabFailed: "Couldn't copy element",
|
||||
},
|
||||
annotate: {
|
||||
title: "Annotate element",
|
||||
placeholder: "Message to the agent about this element…",
|
||||
title: "Send feedback to agent",
|
||||
placeholder: "Describe what should change…",
|
||||
submit: "Attach",
|
||||
cancel: "Cancel",
|
||||
intents: {
|
||||
fix: "Fix",
|
||||
change: "Change",
|
||||
question: "Question",
|
||||
approve: "Approve",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Device size",
|
||||
|
||||
@@ -426,17 +426,22 @@ export const es: TranslationResources = {
|
||||
enterUrl: "IngreseURL",
|
||||
openDevTools: "Abrir herramientas de desarrollo del navegador",
|
||||
cancelSelector: "Cancelar selector de elementos",
|
||||
annotateElement: "Anotar elemento",
|
||||
screenshotElement: "Capturar elemento",
|
||||
screenshotCopied: "Captura copiada al portapapeles",
|
||||
elementCopied: "Elemento copiado al portapapeles",
|
||||
screenshotFailed: "No se pudo copiar la captura",
|
||||
selectElement: "Seleccionar elemento",
|
||||
grabElement: "Copiar elemento al portapapeles",
|
||||
grabElementLabel: "elemento",
|
||||
grabFailed: "No se pudo copiar el elemento",
|
||||
},
|
||||
annotate: {
|
||||
title: "Anotar elemento",
|
||||
placeholder: "Mensaje al agente sobre este elemento…",
|
||||
title: "Enviar comentarios al agente",
|
||||
placeholder: "Describe qué debería cambiar…",
|
||||
submit: "Adjuntar",
|
||||
cancel: "Cancelar",
|
||||
intents: {
|
||||
fix: "Corregir",
|
||||
change: "Cambiar",
|
||||
question: "Pregunta",
|
||||
approve: "Aprobar",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Tamaño del dispositivo",
|
||||
|
||||
@@ -426,17 +426,22 @@ export const fr: TranslationResources = {
|
||||
enterUrl: "EntrezURL",
|
||||
openDevTools: "Outils de développement du navigateur ouvert",
|
||||
cancelSelector: "Annuler le sélecteur d'élément",
|
||||
annotateElement: "Annoter l'élément",
|
||||
screenshotElement: "Capturer l'élément",
|
||||
screenshotCopied: "Capture d'écran copiée dans le presse-papiers",
|
||||
elementCopied: "Élément copié dans le presse-papiers",
|
||||
screenshotFailed: "Impossible de copier la capture",
|
||||
selectElement: "Sélectionner un élément",
|
||||
grabElement: "Copier l'élément dans le presse-papiers",
|
||||
grabElementLabel: "élément",
|
||||
grabFailed: "Impossible de copier l'élément",
|
||||
},
|
||||
annotate: {
|
||||
title: "Annoter l'élément",
|
||||
placeholder: "Message à l'agent concernant cet élément…",
|
||||
title: "Envoyer un retour à l'agent",
|
||||
placeholder: "Décrivez ce qui doit changer…",
|
||||
submit: "Joindre",
|
||||
cancel: "Annuler",
|
||||
intents: {
|
||||
fix: "Corriger",
|
||||
change: "Modifier",
|
||||
question: "Question",
|
||||
approve: "Approuver",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Taille de l'appareil",
|
||||
|
||||
@@ -426,17 +426,22 @@ export const ja: TranslationResources = {
|
||||
enterUrl: "URLを入力",
|
||||
openDevTools: "ブラウザ開発ツールを開く",
|
||||
cancelSelector: "要素セレクターをキャンセル",
|
||||
annotateElement: "要素に注釈を付ける",
|
||||
screenshotElement: "要素のスクリーンショット",
|
||||
screenshotCopied: "スクリーンショットをクリップボードにコピーしました",
|
||||
elementCopied: "要素をクリップボードにコピーしました",
|
||||
screenshotFailed: "スクリーンショットをコピーできませんでした",
|
||||
selectElement: "要素を選択",
|
||||
grabElement: "要素をクリップボードにコピー",
|
||||
grabElementLabel: "要素",
|
||||
grabFailed: "要素をコピーできませんでした",
|
||||
},
|
||||
annotate: {
|
||||
title: "要素に注釈を付ける",
|
||||
placeholder: "この要素についてエージェントへのメッセージ…",
|
||||
title: "エージェントにフィードバックを送信",
|
||||
placeholder: "変更すべき内容を記述…",
|
||||
submit: "添付",
|
||||
cancel: "キャンセル",
|
||||
intents: {
|
||||
fix: "修正",
|
||||
change: "変更",
|
||||
question: "質問",
|
||||
approve: "承認",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "デバイスサイズ",
|
||||
|
||||
@@ -426,17 +426,22 @@ export const ptBR: TranslationResources = {
|
||||
enterUrl: "Inserir URL",
|
||||
openDevTools: "Abrir ferramentas de desenvolvedor do navegador",
|
||||
cancelSelector: "Cancelar seletor de elemento",
|
||||
annotateElement: "Anotar elemento",
|
||||
screenshotElement: "Capturar elemento",
|
||||
screenshotCopied: "Captura copiada para a área de transferência",
|
||||
elementCopied: "Elemento copiado para a área de transferência",
|
||||
screenshotFailed: "Não foi possível copiar a captura",
|
||||
selectElement: "Selecionar elemento",
|
||||
grabElement: "Copiar elemento para a área de transferência",
|
||||
grabElementLabel: "elemento",
|
||||
grabFailed: "Não foi possível copiar o elemento",
|
||||
},
|
||||
annotate: {
|
||||
title: "Anotar elemento",
|
||||
placeholder: "Mensagem ao agente sobre este elemento…",
|
||||
title: "Enviar feedback ao agente",
|
||||
placeholder: "Descreva o que deve mudar…",
|
||||
submit: "Anexar",
|
||||
cancel: "Cancelar",
|
||||
intents: {
|
||||
fix: "Corrigir",
|
||||
change: "Alterar",
|
||||
question: "Pergunta",
|
||||
approve: "Aprovar",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Tamanho do dispositivo",
|
||||
|
||||
@@ -426,17 +426,22 @@ export const ru: TranslationResources = {
|
||||
enterUrl: "Введите URL",
|
||||
openDevTools: "Открыть инструменты разработки браузера",
|
||||
cancelSelector: "Отменить выбор элемента",
|
||||
annotateElement: "Аннотировать элемент",
|
||||
screenshotElement: "Снимок элемента",
|
||||
screenshotCopied: "Снимок скопирован в буфер обмена",
|
||||
elementCopied: "Элемент скопирован в буфер обмена",
|
||||
screenshotFailed: "Не удалось скопировать снимок",
|
||||
selectElement: "Выберите элемент",
|
||||
grabElement: "Скопировать элемент в буфер обмена",
|
||||
grabElementLabel: "элемент",
|
||||
grabFailed: "Не удалось скопировать элемент",
|
||||
},
|
||||
annotate: {
|
||||
title: "Аннотировать элемент",
|
||||
placeholder: "Сообщение агенту об этом элементе…",
|
||||
title: "Отправить отзыв агенту",
|
||||
placeholder: "Опишите, что нужно изменить…",
|
||||
submit: "Прикрепить",
|
||||
cancel: "Отмена",
|
||||
intents: {
|
||||
fix: "Исправить",
|
||||
change: "Изменить",
|
||||
question: "Вопрос",
|
||||
approve: "Одобрить",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "Размер устройства",
|
||||
|
||||
@@ -422,17 +422,22 @@ export const zhCN: TranslationResources = {
|
||||
enterUrl: "输入 URL",
|
||||
openDevTools: "打开浏览器开发者工具",
|
||||
cancelSelector: "取消元素选择器",
|
||||
annotateElement: "标注元素",
|
||||
screenshotElement: "截图元素",
|
||||
screenshotCopied: "已将截图复制到剪贴板",
|
||||
elementCopied: "已将元素复制到剪贴板",
|
||||
screenshotFailed: "无法复制截图",
|
||||
selectElement: "选择元素",
|
||||
grabElement: "复制元素到剪贴板",
|
||||
grabElementLabel: "元素",
|
||||
grabFailed: "复制元素失败",
|
||||
},
|
||||
annotate: {
|
||||
title: "标注元素",
|
||||
placeholder: "给智能体关于此元素的留言…",
|
||||
title: "发送反馈给智能体",
|
||||
placeholder: "描述需要修改的内容…",
|
||||
submit: "附加",
|
||||
cancel: "取消",
|
||||
intents: {
|
||||
fix: "修复",
|
||||
change: "修改",
|
||||
question: "提问",
|
||||
approve: "认可",
|
||||
},
|
||||
},
|
||||
devices: {
|
||||
label: "设备尺寸",
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import AsyncStorage from "@react-native-async-storage/async-storage";
|
||||
import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { create } from "zustand";
|
||||
import { createJSONStorage, persist } from "zustand/middleware";
|
||||
import {
|
||||
@@ -23,14 +22,10 @@ interface BrowserStoreState extends BrowserIndexState {
|
||||
}
|
||||
|
||||
function createBrowserId(): string {
|
||||
let browserId: string;
|
||||
if (typeof globalThis.crypto?.randomUUID === "function") {
|
||||
browserId = globalThis.crypto.randomUUID();
|
||||
} else {
|
||||
const randomSuffix = Math.random().toString(16).slice(2) || "0";
|
||||
browserId = `${Date.now()}-${randomSuffix}`;
|
||||
return globalThis.crypto.randomUUID();
|
||||
}
|
||||
return BrowserAutomationBrowserIdSchema.parse(browserId);
|
||||
return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
}
|
||||
|
||||
export const useBrowserStore = create<BrowserStoreState>()(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/cli",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"description": "Paseo CLI - control your AI coding agents from the command line",
|
||||
"bin": {
|
||||
"paseo": "bin/paseo"
|
||||
@@ -27,9 +27,9 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^1.0.0",
|
||||
"@getpaseo/client": "0.1.104-beta.1",
|
||||
"@getpaseo/protocol": "0.1.104-beta.1",
|
||||
"@getpaseo/server": "0.1.104-beta.1",
|
||||
"@getpaseo/client": "0.1.103",
|
||||
"@getpaseo/protocol": "0.1.103",
|
||||
"@getpaseo/server": "0.1.103",
|
||||
"chalk": "^5.3.0",
|
||||
"commander": "^12.0.0",
|
||||
"mime-types": "^2.1.35",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/client",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"description": "Paseo client SDK package",
|
||||
"files": [
|
||||
"dist",
|
||||
@@ -35,8 +35,8 @@
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@getpaseo/protocol": "0.1.104-beta.1",
|
||||
"@getpaseo/relay": "0.1.104-beta.1",
|
||||
"@getpaseo/protocol": "0.1.103",
|
||||
"@getpaseo/relay": "0.1.103",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
out/
|
||||
fatal-error.txt
|
||||
@@ -1,74 +0,0 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Paseo Capture Harness Target</title>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
width: 1280px;
|
||||
min-height: 1600px;
|
||||
margin: 0;
|
||||
background: #ff00ff;
|
||||
overflow: auto;
|
||||
font-family: Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
.label {
|
||||
position: absolute;
|
||||
left: 48px;
|
||||
top: 48px;
|
||||
color: #ffffff;
|
||||
font-size: 72px;
|
||||
font-weight: 800;
|
||||
line-height: 1;
|
||||
text-shadow:
|
||||
5px 5px 0 #000000,
|
||||
-2px -2px 0 #000000;
|
||||
}
|
||||
|
||||
.sub {
|
||||
position: absolute;
|
||||
left: 54px;
|
||||
top: 150px;
|
||||
color: #000000;
|
||||
background: #ffffff;
|
||||
font-size: 34px;
|
||||
font-weight: 700;
|
||||
padding: 10px 18px;
|
||||
}
|
||||
|
||||
.block {
|
||||
position: absolute;
|
||||
left: 56px;
|
||||
top: 220px;
|
||||
width: 360px;
|
||||
height: 92px;
|
||||
background: linear-gradient(
|
||||
90deg,
|
||||
#000000 0 25%,
|
||||
#ffffff 25% 50%,
|
||||
#000000 50% 75%,
|
||||
#ffffff 75% 100%
|
||||
);
|
||||
border: 8px solid #000000;
|
||||
}
|
||||
|
||||
.bottom {
|
||||
position: absolute;
|
||||
left: 48px;
|
||||
top: 1370px;
|
||||
color: #ffffff;
|
||||
font-size: 60px;
|
||||
font-weight: 800;
|
||||
text-shadow: 5px 5px 0 #000000;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="label">PASEO CAPTURE HARNESS</div>
|
||||
<div class="sub">LOCAL MAGENTA TARGET 1280x800 VIEWPORT</div>
|
||||
<div class="block"></div>
|
||||
<div class="bottom">FULL PAGE MARKER AT Y=1370</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,93 +0,0 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Paseo Delayed Capture Harness Target</title>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
width: 1280px;
|
||||
min-height: 1600px;
|
||||
margin: 0;
|
||||
background: #202020;
|
||||
overflow: auto;
|
||||
font-family: Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
.content {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
body.ready {
|
||||
background: #ff00ff;
|
||||
}
|
||||
|
||||
body.ready .content {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.label {
|
||||
position: absolute;
|
||||
left: 48px;
|
||||
top: 48px;
|
||||
color: #ffffff;
|
||||
font-size: 72px;
|
||||
font-weight: 800;
|
||||
line-height: 1;
|
||||
text-shadow:
|
||||
5px 5px 0 #000000,
|
||||
-2px -2px 0 #000000;
|
||||
}
|
||||
|
||||
.sub {
|
||||
position: absolute;
|
||||
left: 54px;
|
||||
top: 150px;
|
||||
color: #000000;
|
||||
background: #ffffff;
|
||||
font-size: 34px;
|
||||
font-weight: 700;
|
||||
padding: 10px 18px;
|
||||
}
|
||||
|
||||
.block {
|
||||
position: absolute;
|
||||
left: 56px;
|
||||
top: 220px;
|
||||
width: 360px;
|
||||
height: 92px;
|
||||
background: linear-gradient(
|
||||
90deg,
|
||||
#000000 0 25%,
|
||||
#ffffff 25% 50%,
|
||||
#000000 50% 75%,
|
||||
#ffffff 75% 100%
|
||||
);
|
||||
border: 8px solid #000000;
|
||||
}
|
||||
|
||||
.bottom {
|
||||
position: absolute;
|
||||
left: 48px;
|
||||
top: 1370px;
|
||||
color: #ffffff;
|
||||
font-size: 60px;
|
||||
font-weight: 800;
|
||||
text-shadow: 5px 5px 0 #000000;
|
||||
}
|
||||
</style>
|
||||
<script>
|
||||
setTimeout(() => {
|
||||
document.body.classList.add("ready");
|
||||
}, 1500);
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="content">
|
||||
<div class="label">PASEO CAPTURE HARNESS</div>
|
||||
<div class="sub">DELAYED FIRST PAINT TARGET</div>
|
||||
<div class="block"></div>
|
||||
<div class="bottom">FULL PAGE MARKER AT Y=1370</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,257 +0,0 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Paseo Browser Capture Harness</title>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
background: #202020;
|
||||
overflow: hidden;
|
||||
font-family: Arial, sans-serif;
|
||||
}
|
||||
|
||||
#paseo-browser-resident-webviews {
|
||||
position: fixed;
|
||||
left: -20000px;
|
||||
top: 0;
|
||||
width: 1280px;
|
||||
height: 800px;
|
||||
overflow: hidden;
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.capture-harness-webview {
|
||||
display: inline-flex;
|
||||
flex: 0 0 auto;
|
||||
width: 1280px;
|
||||
height: 800px;
|
||||
border: 0;
|
||||
background: transparent;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="paseo-browser-resident-webviews" aria-hidden="true"></div>
|
||||
|
||||
<script>
|
||||
const RESIDENT_VIEWPORT_WIDTH = 1280;
|
||||
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
||||
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
const webviewCount = Math.max(2, Number(params.get("webviewCount")) || 2);
|
||||
const webviews = [];
|
||||
const activeTokens = new Set();
|
||||
let nextTokenId = 0;
|
||||
|
||||
function appendHarnessWebview(sourceUrl) {
|
||||
const index = webviews.length;
|
||||
const webview = document.createElement("webview");
|
||||
webview.id = `target-webview-${index + 1}`;
|
||||
webview.className = "capture-harness-webview";
|
||||
webview.setAttribute("data-paseo-browser-id", `capture-harness-${index + 1}`);
|
||||
webview.setAttribute("partition", `persist:paseo-capture-harness-${index + 1}`);
|
||||
webview.setAttribute("allowpopups", "true");
|
||||
webview.setAttribute("spellcheck", "false");
|
||||
webview.setAttribute("autosize", "on");
|
||||
webview.src = sourceUrl;
|
||||
applyStackedWebviewStyle(webview);
|
||||
host.appendChild(webview);
|
||||
webviews.push(webview);
|
||||
return index;
|
||||
}
|
||||
|
||||
for (let index = 0; index < webviewCount; index += 1) {
|
||||
appendHarnessWebview(params.get("targetUrl") || "bright.html");
|
||||
}
|
||||
|
||||
function applyHostParking() {
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "-20000px";
|
||||
host.style.top = "0";
|
||||
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "0";
|
||||
host.style.pointerEvents = "none";
|
||||
host.style.zIndex = "";
|
||||
host.style.clipPath = "";
|
||||
host.style.visibility = "";
|
||||
host.style.transform = "";
|
||||
}
|
||||
|
||||
function applyWebviewBaseStyle(webview) {
|
||||
webview.style.display = "inline-flex";
|
||||
webview.style.flex = "0 0 auto";
|
||||
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
webview.style.border = "0";
|
||||
webview.style.background = "transparent";
|
||||
}
|
||||
|
||||
function applyLegacyVerticalWebviewStyle(webview, index) {
|
||||
applyWebviewBaseStyle(webview);
|
||||
webview.style.position = "";
|
||||
webview.style.left = "";
|
||||
webview.style.top = "";
|
||||
webview.style.marginTop = index === 0 ? "0" : "4px";
|
||||
webview.style.zIndex = "";
|
||||
}
|
||||
|
||||
function applyStackedWebviewStyle(webview, zIndex = "0") {
|
||||
applyWebviewBaseStyle(webview);
|
||||
webview.style.position = "absolute";
|
||||
webview.style.left = "0";
|
||||
webview.style.top = "0";
|
||||
webview.style.marginTop = "0";
|
||||
webview.style.zIndex = zIndex;
|
||||
}
|
||||
|
||||
function applyParking() {
|
||||
applyHostParking();
|
||||
webviews.forEach((webview) => {
|
||||
applyStackedWebviewStyle(webview);
|
||||
});
|
||||
}
|
||||
|
||||
function applyLegacyVerticalParking() {
|
||||
applyHostParking();
|
||||
webviews.forEach((webview, index) => {
|
||||
applyLegacyVerticalWebviewStyle(webview, index);
|
||||
});
|
||||
}
|
||||
|
||||
function applyCapturePrep() {
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "0";
|
||||
host.style.top = "0";
|
||||
host.style.width = "1px";
|
||||
host.style.height = "1px";
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "1";
|
||||
host.style.pointerEvents = "none";
|
||||
host.style.zIndex = "1";
|
||||
host.style.clipPath = "";
|
||||
host.style.visibility = "";
|
||||
host.style.transform = "";
|
||||
}
|
||||
|
||||
function applyTargetStacking(targetIndex) {
|
||||
webviews.forEach((webview, index) => {
|
||||
applyStackedWebviewStyle(webview, index === targetIndex ? "2" : "0");
|
||||
});
|
||||
}
|
||||
|
||||
function waitFrames(count) {
|
||||
return new Promise((resolve) => {
|
||||
function step(remaining) {
|
||||
if (remaining <= 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
requestAnimationFrame(() => step(remaining - 1));
|
||||
}
|
||||
step(count);
|
||||
});
|
||||
}
|
||||
|
||||
function state(targetIndex = 0) {
|
||||
const hostRect = host.getBoundingClientRect();
|
||||
const targetWebview = webviews[targetIndex] || webviews[0];
|
||||
const webviewRect = targetWebview.getBoundingClientRect();
|
||||
return {
|
||||
hostStyle: {
|
||||
left: host.style.left,
|
||||
top: host.style.top,
|
||||
width: host.style.width,
|
||||
height: host.style.height,
|
||||
overflow: host.style.overflow,
|
||||
opacity: host.style.opacity,
|
||||
pointerEvents: host.style.pointerEvents,
|
||||
},
|
||||
hostRect: {
|
||||
x: hostRect.x,
|
||||
y: hostRect.y,
|
||||
width: hostRect.width,
|
||||
height: hostRect.height,
|
||||
},
|
||||
webviewRect: {
|
||||
x: webviewRect.x,
|
||||
y: webviewRect.y,
|
||||
width: webviewRect.width,
|
||||
height: webviewRect.height,
|
||||
},
|
||||
webviews: webviews.map((webview) => {
|
||||
const rect = webview.getBoundingClientRect();
|
||||
return {
|
||||
id: webview.id,
|
||||
zIndex: webview.style.zIndex,
|
||||
position: webview.style.position,
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
window.captureHarness = {
|
||||
async waitForFrames(count = 2) {
|
||||
await waitFrames(count);
|
||||
return state();
|
||||
},
|
||||
webContentsIds() {
|
||||
return webviews.map((webview) =>
|
||||
typeof webview.getWebContentsId === "function" ? webview.getWebContentsId() : null,
|
||||
);
|
||||
},
|
||||
addWebview(sourceUrl) {
|
||||
return appendHarnessWebview(sourceUrl || params.get("targetUrl") || "bright.html");
|
||||
},
|
||||
async prepareForPixelCapture(targetIndex = 0) {
|
||||
const token = `capture-${++nextTokenId}`;
|
||||
activeTokens.add(token);
|
||||
applyCapturePrep();
|
||||
applyTargetStacking(targetIndex);
|
||||
await waitFrames(2);
|
||||
webviews[targetIndex].getBoundingClientRect();
|
||||
return { token, state: state(targetIndex) };
|
||||
},
|
||||
async prepareLegacyVerticalPixelCapture(targetIndex = 0) {
|
||||
const token = `legacy-capture-${++nextTokenId}`;
|
||||
activeTokens.add(token);
|
||||
applyLegacyVerticalParking();
|
||||
applyCapturePrep();
|
||||
await waitFrames(2);
|
||||
webviews[targetIndex].getBoundingClientRect();
|
||||
return { token, state: state(targetIndex) };
|
||||
},
|
||||
async restorePixelCapture(token) {
|
||||
activeTokens.delete(token);
|
||||
if (activeTokens.size === 0) {
|
||||
applyParking();
|
||||
}
|
||||
await waitFrames(2);
|
||||
return state();
|
||||
},
|
||||
async restoreLegacyVerticalParking(token) {
|
||||
activeTokens.delete(token);
|
||||
applyLegacyVerticalParking();
|
||||
await waitFrames(2);
|
||||
return state();
|
||||
},
|
||||
async restoreParking() {
|
||||
activeTokens.clear();
|
||||
applyParking();
|
||||
await waitFrames(2);
|
||||
return state();
|
||||
},
|
||||
};
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,533 +0,0 @@
|
||||
const fs = require("node:fs");
|
||||
const fsp = require("node:fs/promises");
|
||||
const path = require("node:path");
|
||||
const { app, BrowserWindow, nativeImage } = require("electron");
|
||||
|
||||
const ROOT = __dirname;
|
||||
const OUT_DIR = process.env.PASEO_CAPTURE_HARNESS_OUT_DIR || path.join(ROOT, "out");
|
||||
const VIEWPORT_WIDTH = 1280;
|
||||
const VIEWPORT_HEIGHT = 800;
|
||||
const FULL_PAGE_HEIGHT = 1600;
|
||||
const CAPTURE_TIMEOUT_MS = 5000;
|
||||
const CAPTURE_RETRY_INTERVAL_MS = 200;
|
||||
const REPEAT_COUNT = 5;
|
||||
|
||||
function fileUrl(filePath) {
|
||||
return new URL(`file://${filePath}`).toString();
|
||||
}
|
||||
|
||||
function ensureDirSync(dir) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
function withTimeout(promise, label) {
|
||||
let timeoutId;
|
||||
const timeout = new Promise((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
reject(new Error(`${label} timed out after ${CAPTURE_TIMEOUT_MS}ms`));
|
||||
}, CAPTURE_TIMEOUT_MS);
|
||||
});
|
||||
return Promise.race([promise, timeout]).finally(() => {
|
||||
clearTimeout(timeoutId);
|
||||
});
|
||||
}
|
||||
|
||||
function pixelOffset(width, x, y) {
|
||||
return (y * width + x) * 4;
|
||||
}
|
||||
|
||||
function isBrightMagenta(bitmap, offset) {
|
||||
const c0 = bitmap[offset];
|
||||
const c1 = bitmap[offset + 1];
|
||||
const c2 = bitmap[offset + 2];
|
||||
return c0 > 200 && c1 < 90 && c2 > 200;
|
||||
}
|
||||
|
||||
function analyzeImage(image, expected, guestMetrics) {
|
||||
if (!image || image.isEmpty()) {
|
||||
return {
|
||||
width: 0,
|
||||
height: 0,
|
||||
brightRatio: 0,
|
||||
textNonUniform: false,
|
||||
pass: false,
|
||||
};
|
||||
}
|
||||
|
||||
const size = image.getSize();
|
||||
const width = size.width;
|
||||
const height = size.height;
|
||||
const bitmap = image.toBitmap();
|
||||
const totalPixels = width * height;
|
||||
let brightPixels = 0;
|
||||
for (let offset = 0; offset < bitmap.length; offset += 4) {
|
||||
if (isBrightMagenta(bitmap, offset)) {
|
||||
brightPixels += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const crop = {
|
||||
left: Math.min(40, Math.max(0, width - 1)),
|
||||
top: Math.min(40, Math.max(0, height - 1)),
|
||||
right: Math.min(width, 940),
|
||||
bottom: Math.min(height, 260),
|
||||
};
|
||||
let cropPixels = 0;
|
||||
let cropNonBright = 0;
|
||||
let luminanceSum = 0;
|
||||
let luminanceSqSum = 0;
|
||||
const quantized = new Set();
|
||||
for (let y = crop.top; y < crop.bottom; y += 1) {
|
||||
for (let x = crop.left; x < crop.right; x += 1) {
|
||||
const offset = pixelOffset(width, x, y);
|
||||
cropPixels += 1;
|
||||
if (!isBrightMagenta(bitmap, offset)) {
|
||||
cropNonBright += 1;
|
||||
}
|
||||
const r = bitmap[offset + 2];
|
||||
const g = bitmap[offset + 1];
|
||||
const b = bitmap[offset];
|
||||
const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
||||
luminanceSum += luma;
|
||||
luminanceSqSum += luma * luma;
|
||||
quantized.add(`${r >> 5},${g >> 5},${b >> 5},${bitmap[offset + 3] >> 6}`);
|
||||
}
|
||||
}
|
||||
|
||||
const devicePixelRatio =
|
||||
typeof guestMetrics.devicePixelRatio === "number" && guestMetrics.devicePixelRatio > 0
|
||||
? guestMetrics.devicePixelRatio
|
||||
: 1;
|
||||
const sizeTargets = [
|
||||
{ width: expected.width, height: expected.height },
|
||||
{
|
||||
width: Math.round(expected.width * devicePixelRatio),
|
||||
height: Math.round(expected.height * devicePixelRatio),
|
||||
},
|
||||
];
|
||||
const matchedSize = sizeTargets.some(
|
||||
(target) => Math.abs(width - target.width) <= 2 && Math.abs(height - target.height) <= 2,
|
||||
);
|
||||
const luminanceMean = cropPixels ? luminanceSum / cropPixels : 0;
|
||||
const luminanceVariance = cropPixels
|
||||
? luminanceSqSum / cropPixels - luminanceMean * luminanceMean
|
||||
: 0;
|
||||
const brightRatio = totalPixels ? brightPixels / totalPixels : 0;
|
||||
const textNonUniform =
|
||||
cropPixels > 0 &&
|
||||
cropNonBright / cropPixels > 0.02 &&
|
||||
quantized.size >= 4 &&
|
||||
luminanceVariance > 100;
|
||||
|
||||
return {
|
||||
width,
|
||||
height,
|
||||
logicalWidthAtDpr: width / devicePixelRatio,
|
||||
logicalHeightAtDpr: height / devicePixelRatio,
|
||||
brightRatio,
|
||||
textNonUniform,
|
||||
matchedSize,
|
||||
pass: matchedSize && brightRatio >= expected.minBrightRatio && textNonUniform,
|
||||
};
|
||||
}
|
||||
|
||||
function pass(message) {
|
||||
console.log(`PASS ${message}`);
|
||||
}
|
||||
|
||||
function fail(message) {
|
||||
console.log(`FAIL ${message}`);
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async function saveImage(image, outputPath) {
|
||||
await fsp.writeFile(outputPath, image.toPNG());
|
||||
}
|
||||
|
||||
function delay(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function waitForGuestLoad(contents, input = {}) {
|
||||
await new Promise((resolve) => {
|
||||
if (!contents.isLoading()) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
contents.once("did-finish-load", resolve);
|
||||
contents.once("did-fail-load", resolve);
|
||||
});
|
||||
const settleMs = input.settleMs ?? 500;
|
||||
if (settleMs > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, settleMs));
|
||||
}
|
||||
}
|
||||
|
||||
async function renderer(win, expression) {
|
||||
return await win.webContents.executeJavaScript(expression, true);
|
||||
}
|
||||
|
||||
async function readGuestMetrics(contents) {
|
||||
return await contents.executeJavaScript(
|
||||
`({
|
||||
innerWidth: window.innerWidth,
|
||||
innerHeight: window.innerHeight,
|
||||
devicePixelRatio: window.devicePixelRatio,
|
||||
documentClientWidth: document.documentElement.clientWidth,
|
||||
documentClientHeight: document.documentElement.clientHeight,
|
||||
scrollWidth: document.documentElement.scrollWidth,
|
||||
scrollHeight: document.documentElement.scrollHeight,
|
||||
visualViewport: window.visualViewport ? {
|
||||
width: window.visualViewport.width,
|
||||
height: window.visualViewport.height,
|
||||
scale: window.visualViewport.scale
|
||||
} : null
|
||||
})`,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
async function capturePageSequence(contents) {
|
||||
const previousBackgroundThrottling = contents.getBackgroundThrottling();
|
||||
contents.setBackgroundThrottling(false);
|
||||
try {
|
||||
contents.invalidate();
|
||||
return await withTimeout(contents.capturePage(undefined, { stayHidden: false }), "capturePage");
|
||||
} finally {
|
||||
contents.setBackgroundThrottling(previousBackgroundThrottling);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureFullPage(contents) {
|
||||
let attachedHere = false;
|
||||
if (!contents.debugger.isAttached()) {
|
||||
contents.debugger.attach("1.3");
|
||||
attachedHere = true;
|
||||
}
|
||||
try {
|
||||
const metrics = await contents.debugger.sendCommand("Page.getLayoutMetrics");
|
||||
const contentSize = metrics.cssContentSize ||
|
||||
metrics.contentSize || {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: VIEWPORT_WIDTH,
|
||||
height: FULL_PAGE_HEIGHT,
|
||||
};
|
||||
const clip = {
|
||||
x: Math.floor(contentSize.x || 0),
|
||||
y: Math.floor(contentSize.y || 0),
|
||||
width: Math.ceil(contentSize.width || VIEWPORT_WIDTH),
|
||||
height: Math.ceil(contentSize.height || FULL_PAGE_HEIGHT),
|
||||
scale: 1,
|
||||
};
|
||||
const result = await withTimeout(
|
||||
contents.debugger.sendCommand("Page.captureScreenshot", {
|
||||
format: "png",
|
||||
captureBeyondViewport: true,
|
||||
clip,
|
||||
}),
|
||||
"CDP Page.captureScreenshot",
|
||||
);
|
||||
return nativeImage.createFromBuffer(Buffer.from(result.data, "base64"));
|
||||
} finally {
|
||||
if (attachedHere && contents.debugger.isAttached()) {
|
||||
contents.debugger.detach();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureWithPrep({
|
||||
win,
|
||||
contents,
|
||||
mode,
|
||||
repeatIndex,
|
||||
targetIndex,
|
||||
guestMetrics,
|
||||
repeatTotal = REPEAT_COUNT,
|
||||
label = "prep",
|
||||
retryUntilPass = false,
|
||||
}) {
|
||||
const preparation = await renderer(
|
||||
win,
|
||||
`window.captureHarness.prepareForPixelCapture(${JSON.stringify(targetIndex)})`,
|
||||
);
|
||||
const outputPath = path.join(
|
||||
OUT_DIR,
|
||||
`${mode}-webview-${targetIndex + 1}-${label}-${repeatIndex}.png`,
|
||||
);
|
||||
try {
|
||||
const expected =
|
||||
mode === "viewport"
|
||||
? { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 }
|
||||
: { width: VIEWPORT_WIDTH, height: FULL_PAGE_HEIGHT, minBrightRatio: 0.55 };
|
||||
const deadline = Date.now() + CAPTURE_TIMEOUT_MS;
|
||||
let attempt = 0;
|
||||
let lastFailure = "capture did not run";
|
||||
while (Date.now() < deadline) {
|
||||
attempt += 1;
|
||||
try {
|
||||
const image =
|
||||
mode === "viewport"
|
||||
? await capturePageSequence(contents)
|
||||
: await captureFullPage(contents);
|
||||
const analysis = analyzeImage(image, expected, guestMetrics);
|
||||
const size = `${analysis.width}x${analysis.height}`;
|
||||
const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`;
|
||||
const bright = analysis.brightRatio.toFixed(4);
|
||||
if (analysis.pass) {
|
||||
await saveImage(image, outputPath);
|
||||
pass(
|
||||
`${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} attempts=${attempt} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
|
||||
);
|
||||
return analysis;
|
||||
}
|
||||
lastFailure = `size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform}`;
|
||||
} catch (error) {
|
||||
lastFailure = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
if (!retryUntilPass) {
|
||||
break;
|
||||
}
|
||||
await delay(Math.min(CAPTURE_RETRY_INTERVAL_MS, Math.max(0, deadline - Date.now())));
|
||||
}
|
||||
fail(
|
||||
`${mode} webview ${targetIndex + 1} ${label} ${repeatIndex}/${repeatTotal} failed attempts=${attempt} last=${lastFailure} file=${outputPath}`,
|
||||
);
|
||||
} finally {
|
||||
const restoredState = await renderer(
|
||||
win,
|
||||
`window.captureHarness.restorePixelCapture(${JSON.stringify(preparation.token)})`,
|
||||
);
|
||||
const style = restoredState.hostStyle;
|
||||
if (style.left !== "-20000px" || style.opacity !== "0") {
|
||||
fail(`restore left=${style.left} opacity=${style.opacity}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function expectLegacySecondWebviewFailure({ win, contents, mode, guestMetrics }) {
|
||||
const targetIndex = 1;
|
||||
const preparation = await renderer(
|
||||
win,
|
||||
`window.captureHarness.prepareLegacyVerticalPixelCapture(${JSON.stringify(targetIndex)})`,
|
||||
);
|
||||
const outputPath = path.join(OUT_DIR, `${mode}-legacy-webview-${targetIndex + 1}.png`);
|
||||
try {
|
||||
const image =
|
||||
mode === "viewport" ? await capturePageSequence(contents) : await captureFullPage(contents);
|
||||
await saveImage(image, outputPath);
|
||||
const expected =
|
||||
mode === "viewport"
|
||||
? { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 }
|
||||
: { width: VIEWPORT_WIDTH, height: FULL_PAGE_HEIGHT, minBrightRatio: 0.55 };
|
||||
const analysis = analyzeImage(image, expected, guestMetrics);
|
||||
const size = `${analysis.width}x${analysis.height}`;
|
||||
const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`;
|
||||
const bright = analysis.brightRatio.toFixed(4);
|
||||
if (analysis.pass) {
|
||||
fail(
|
||||
`${mode} legacy webview ${targetIndex + 1} unexpectedly captured size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
|
||||
);
|
||||
}
|
||||
pass(
|
||||
`${mode} legacy webview ${targetIndex + 1} reproduces no-frame size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
|
||||
);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
pass(`${mode} legacy webview ${targetIndex + 1} reproduces no-frame error=${message}`);
|
||||
} finally {
|
||||
await renderer(
|
||||
win,
|
||||
`window.captureHarness.restoreLegacyVerticalParking(${JSON.stringify(preparation.token)})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode }) {
|
||||
const freshGuestPromise = waitForNextAttachedGuest();
|
||||
const targetIndex = await renderer(
|
||||
win,
|
||||
`window.captureHarness.addWebview(${JSON.stringify(fileUrl(path.join(ROOT, "delayed-bright.html")))})`,
|
||||
);
|
||||
const guest = await withTimeout(freshGuestPromise, "fresh did-attach-webview");
|
||||
const guestMetrics = await readGuestMetrics(guest);
|
||||
if (guestMetrics.innerWidth !== VIEWPORT_WIDTH || guestMetrics.innerHeight !== VIEWPORT_HEIGHT) {
|
||||
fail(
|
||||
`fresh guest viewport sizing webview ${targetIndex + 1} inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} expected=${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
|
||||
);
|
||||
}
|
||||
pass(
|
||||
`fresh guest viewport sizing webview ${targetIndex + 1} inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} dpr=${guestMetrics.devicePixelRatio}`,
|
||||
);
|
||||
await captureWithPrep({
|
||||
win,
|
||||
contents: guest,
|
||||
mode,
|
||||
repeatIndex: 1,
|
||||
targetIndex,
|
||||
guestMetrics,
|
||||
repeatTotal: 1,
|
||||
label: "fresh-delayed-first-frame",
|
||||
retryUntilPass: true,
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
ensureDirSync(OUT_DIR);
|
||||
|
||||
const attachedGuests = [];
|
||||
const freshGuestWaiters = [];
|
||||
let resolveGuests;
|
||||
const guestsPromise = new Promise((resolve) => {
|
||||
resolveGuests = resolve;
|
||||
});
|
||||
const waitForNextAttachedGuest = () =>
|
||||
new Promise((resolve) => {
|
||||
freshGuestWaiters.push(resolve);
|
||||
});
|
||||
const win = new BrowserWindow({
|
||||
width: 1000,
|
||||
height: 700,
|
||||
show: true,
|
||||
backgroundColor: "#202020",
|
||||
webPreferences: {
|
||||
webviewTag: true,
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
sandbox: false,
|
||||
},
|
||||
});
|
||||
|
||||
win.webContents.on("will-attach-webview", (_event, webPreferences) => {
|
||||
webPreferences.nodeIntegration = false;
|
||||
webPreferences.contextIsolation = true;
|
||||
});
|
||||
win.webContents.on("did-attach-webview", (_event, contents) => {
|
||||
attachedGuests.push(contents);
|
||||
const waiter = freshGuestWaiters.shift();
|
||||
if (waiter) {
|
||||
waiter(contents);
|
||||
}
|
||||
if (attachedGuests.length >= 2) {
|
||||
resolveGuests(attachedGuests);
|
||||
}
|
||||
});
|
||||
|
||||
await win.loadFile(path.join(ROOT, "index.html"), {
|
||||
query: { targetUrl: fileUrl(path.join(ROOT, "bright.html")), webviewCount: "2" },
|
||||
});
|
||||
await withTimeout(guestsPromise, "did-attach-webview");
|
||||
await Promise.all(attachedGuests.map((guest) => waitForGuestLoad(guest)));
|
||||
await renderer(win, "window.captureHarness.waitForFrames(2)");
|
||||
const webContentsIds = await renderer(win, "window.captureHarness.webContentsIds()");
|
||||
const guestsById = new Map(attachedGuests.map((guest) => [guest.id, guest]));
|
||||
const guests = webContentsIds.map((id) => guestsById.get(id));
|
||||
if (guests.some((guest) => !guest)) {
|
||||
fail(
|
||||
`could not map webviews to guest contents ids=${JSON.stringify(webContentsIds)} attached=${attachedGuests.map((guest) => guest.id).join(",")}`,
|
||||
);
|
||||
}
|
||||
const guestMetrics = await Promise.all(guests.map((guest) => readGuestMetrics(guest)));
|
||||
|
||||
guestMetrics.forEach((metrics, index) => {
|
||||
if (metrics.innerWidth !== VIEWPORT_WIDTH || metrics.innerHeight !== VIEWPORT_HEIGHT) {
|
||||
fail(
|
||||
`guest viewport sizing webview ${index + 1} inner=${metrics.innerWidth}x${metrics.innerHeight} expected=${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
|
||||
);
|
||||
}
|
||||
pass(
|
||||
`guest viewport sizing webview ${index + 1} inner=${metrics.innerWidth}x${metrics.innerHeight} dpr=${metrics.devicePixelRatio}`,
|
||||
);
|
||||
});
|
||||
|
||||
await renderer(win, "window.captureHarness.restoreParking()");
|
||||
try {
|
||||
const image = await capturePageSequence(guests[0]);
|
||||
const analysis = analyzeImage(
|
||||
image,
|
||||
{ width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 },
|
||||
guestMetrics[0],
|
||||
);
|
||||
fail(
|
||||
`parked webview unexpectedly captured size=${analysis.width}x${analysis.height} bright=${analysis.brightRatio.toFixed(4)}`,
|
||||
);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
pass(`parked webview has no copyable viewport frame error=${message}`);
|
||||
}
|
||||
|
||||
await expectLegacySecondWebviewFailure({
|
||||
win,
|
||||
contents: guests[1],
|
||||
mode: "viewport",
|
||||
guestMetrics: guestMetrics[1],
|
||||
});
|
||||
await expectLegacySecondWebviewFailure({
|
||||
win,
|
||||
contents: guests[1],
|
||||
mode: "full-page",
|
||||
guestMetrics: guestMetrics[1],
|
||||
});
|
||||
|
||||
await renderer(win, "window.captureHarness.restoreParking()");
|
||||
|
||||
await captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode: "viewport" });
|
||||
await renderer(win, "window.captureHarness.restoreParking()");
|
||||
await captureFreshDelayedWebview({ win, waitForNextAttachedGuest, mode: "full-page" });
|
||||
await renderer(win, "window.captureHarness.restoreParking()");
|
||||
|
||||
const results = [];
|
||||
for (const targetIndex of [0, 1]) {
|
||||
for (let index = 1; index <= REPEAT_COUNT; index += 1) {
|
||||
results.push(
|
||||
await captureWithPrep({
|
||||
win,
|
||||
contents: guests[targetIndex],
|
||||
mode: "viewport",
|
||||
repeatIndex: index,
|
||||
targetIndex,
|
||||
guestMetrics: guestMetrics[targetIndex],
|
||||
}),
|
||||
);
|
||||
}
|
||||
for (let index = 1; index <= REPEAT_COUNT; index += 1) {
|
||||
results.push(
|
||||
await captureWithPrep({
|
||||
win,
|
||||
contents: guests[targetIndex],
|
||||
mode: "full-page",
|
||||
repeatIndex: index,
|
||||
targetIndex,
|
||||
guestMetrics: guestMetrics[targetIndex],
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await fsp.writeFile(
|
||||
path.join(OUT_DIR, "results.json"),
|
||||
`${JSON.stringify({ generatedAt: new Date().toISOString(), guestMetrics, results }, null, 2)}\n`,
|
||||
);
|
||||
pass(`capture harness complete output=${OUT_DIR}`);
|
||||
|
||||
if (!win.isDestroyed()) {
|
||||
win.close();
|
||||
}
|
||||
}
|
||||
|
||||
app
|
||||
.whenReady()
|
||||
.then(main)
|
||||
.then(() => app.quit())
|
||||
.catch(async (error) => {
|
||||
console.error(error);
|
||||
try {
|
||||
await fsp.writeFile(
|
||||
path.join(OUT_DIR, "fatal-error.txt"),
|
||||
`${error && error.stack ? error.stack : String(error)}\n`,
|
||||
);
|
||||
} catch {
|
||||
// Ignore reporting failures during shutdown.
|
||||
}
|
||||
app.exit(1);
|
||||
});
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd)
|
||||
|
||||
exec "$REPO_ROOT/node_modules/.bin/electron" "$SCRIPT_DIR/main.js"
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/desktop",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"private": true,
|
||||
"description": "Paseo desktop app (Electron wrapper)",
|
||||
"homepage": "https://paseo.sh",
|
||||
@@ -17,7 +17,6 @@
|
||||
"scripts": {
|
||||
"build": "npm --prefix ../.. run build:server:clean && npm run build:main && electron-builder --config electron-builder.yml",
|
||||
"build:main": "tsc -p tsconfig.json",
|
||||
"capture-harness": "./capture-harness/run.sh",
|
||||
"dev": "./scripts/dev.sh",
|
||||
"dev:win": "powershell ./scripts/dev.ps1",
|
||||
"test": "vitest run",
|
||||
|
||||
@@ -1,486 +1,18 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import type { TabImage } from "./service.js";
|
||||
import { adaptWebContents } from "./ipc.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
class FakeImage implements TabImage {
|
||||
public toPNG(): Uint8Array {
|
||||
return new Uint8Array([137, 80, 78, 71]);
|
||||
}
|
||||
import { sanitizeDownloadFileName } from "./ipc.js";
|
||||
|
||||
public getSize(): { width: number; height: number } {
|
||||
return { width: 640, height: 480 };
|
||||
}
|
||||
}
|
||||
|
||||
class FakeDebugger {
|
||||
public isAttached(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public attach(): void {}
|
||||
|
||||
public async sendCommand(): Promise<unknown> {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
class FakeHostWebContents {
|
||||
public readonly sentMessages: Array<{ channel: string; payload: unknown }> = [];
|
||||
public destroyed = false;
|
||||
|
||||
public constructor(public readonly id: number) {}
|
||||
|
||||
public isDestroyed(): boolean {
|
||||
return this.destroyed;
|
||||
}
|
||||
|
||||
public send(channel: string, payload: unknown): void {
|
||||
this.sentMessages.push({ channel, payload });
|
||||
}
|
||||
}
|
||||
|
||||
interface FakeIpcEvent {
|
||||
sender: {
|
||||
id: number;
|
||||
};
|
||||
}
|
||||
|
||||
type IpcListener = (event: FakeIpcEvent, payload: unknown) => void;
|
||||
|
||||
class FakeIpcBridge {
|
||||
private readonly listeners = new Map<string, IpcListener[]>();
|
||||
|
||||
public on(channel: string, listener: IpcListener): void {
|
||||
const listeners = this.listeners.get(channel) ?? [];
|
||||
listeners.push(listener);
|
||||
this.listeners.set(channel, listeners);
|
||||
}
|
||||
|
||||
public removeListener(channel: string, listener: IpcListener): void {
|
||||
const listeners = this.listeners.get(channel) ?? [];
|
||||
this.listeners.set(
|
||||
channel,
|
||||
listeners.filter((candidate) => candidate !== listener),
|
||||
);
|
||||
}
|
||||
|
||||
public emit(channel: string, payload: unknown, input: { senderId?: number } = {}): void {
|
||||
const event = { sender: { id: input.senderId ?? 10 } };
|
||||
for (const listener of this.listeners.get(channel) ?? []) {
|
||||
listener(event, payload);
|
||||
}
|
||||
}
|
||||
|
||||
public listenerCount(channel: string): number {
|
||||
return this.listeners.get(channel)?.length ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
class FakeWebContents {
|
||||
public readonly debugger = new FakeDebugger();
|
||||
public readonly consoleMessages: unknown[] = [];
|
||||
public readonly destroyedListeners: Array<() => void> = [];
|
||||
public destroyed = false;
|
||||
|
||||
public constructor(
|
||||
public readonly id: number,
|
||||
public hostWebContents: FakeHostWebContents | null,
|
||||
) {}
|
||||
|
||||
public getURL(): string {
|
||||
return "https://example.com";
|
||||
}
|
||||
|
||||
public getTitle(): string {
|
||||
return "Example";
|
||||
}
|
||||
|
||||
public canGoBack(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public canGoForward(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public isLoading(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public isDestroyed(): boolean {
|
||||
return this.destroyed;
|
||||
}
|
||||
|
||||
public async executeJavaScript(): Promise<unknown> {
|
||||
return null;
|
||||
}
|
||||
|
||||
public async loadURL(): Promise<void> {}
|
||||
|
||||
public goBack(): void {}
|
||||
|
||||
public goForward(): void {}
|
||||
|
||||
public reload(): void {}
|
||||
|
||||
public async capturePage(
|
||||
_rect?: Rectangle,
|
||||
_options?: { stayHidden?: boolean },
|
||||
): Promise<TabImage> {
|
||||
return new FakeImage();
|
||||
}
|
||||
|
||||
public invalidate(): void {}
|
||||
|
||||
public getBackgroundThrottling(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
public setBackgroundThrottling(): void {}
|
||||
|
||||
public on(
|
||||
event: "console-message",
|
||||
listener: (
|
||||
event: unknown,
|
||||
level: unknown,
|
||||
message: unknown,
|
||||
line: unknown,
|
||||
sourceId: unknown,
|
||||
) => void,
|
||||
): void {
|
||||
this.consoleMessages.push({ event, listener });
|
||||
}
|
||||
|
||||
public once(event: "destroyed", listener: () => void): void {
|
||||
expect(event).toBe("destroyed");
|
||||
this.destroyedListeners.push(listener);
|
||||
}
|
||||
}
|
||||
|
||||
describe("browser automation IPC adapter", () => {
|
||||
test("prepareForPixelCapture asks the embedder renderer and resolves the ack token", async () => {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => "prepare-1",
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
]);
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "other",
|
||||
ok: true,
|
||||
token: "wrong-token",
|
||||
});
|
||||
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1);
|
||||
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: true,
|
||||
token: "token-a",
|
||||
});
|
||||
|
||||
await expect(preparation).resolves.toEqual({ token: "token-a" });
|
||||
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
|
||||
describe("browser automation IPC", () => {
|
||||
it("strips directories from agent-supplied download filenames", () => {
|
||||
expect(
|
||||
sanitizeDownloadFileName({
|
||||
url: "https://example.com/fallback.txt",
|
||||
fileName: "../../.ssh/authorized_keys",
|
||||
}),
|
||||
).toBe("authorized_keys");
|
||||
});
|
||||
|
||||
test("restorePixelCapture sends the capture token back to the embedder renderer", async () => {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const requestIds = ["prepare-1", "restore-1"];
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => {
|
||||
const requestId = requestIds.shift();
|
||||
if (!requestId) {
|
||||
throw new Error("Missing request id");
|
||||
}
|
||||
return requestId;
|
||||
},
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: true,
|
||||
token: "token-a",
|
||||
});
|
||||
await expect(preparation).resolves.toEqual({ token: "token-a" });
|
||||
|
||||
const restored = tab.restorePixelCapture({ token: "token-a" });
|
||||
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
{
|
||||
channel: "paseo:browser:capture-restore",
|
||||
payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" },
|
||||
},
|
||||
]);
|
||||
ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true });
|
||||
|
||||
await expect(restored).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test("restorePixelCapture uses the host captured during preparation when the guest detaches", async () => {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const requestIds = ["prepare-1", "restore-1"];
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => {
|
||||
const requestId = requestIds.shift();
|
||||
if (!requestId) {
|
||||
throw new Error("Missing request id");
|
||||
}
|
||||
return requestId;
|
||||
},
|
||||
});
|
||||
|
||||
const preparationPromise = tab.prepareForPixelCapture();
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: true,
|
||||
token: "token-a",
|
||||
});
|
||||
const preparation = await preparationPromise;
|
||||
contents.hostWebContents = null;
|
||||
|
||||
const restored = tab.restorePixelCapture(preparation);
|
||||
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
{
|
||||
channel: "paseo:browser:capture-restore",
|
||||
payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" },
|
||||
},
|
||||
]);
|
||||
ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true });
|
||||
|
||||
await expect(restored).resolves.toBeUndefined();
|
||||
await expect(tab.restorePixelCapture(preparation)).rejects.toThrow(
|
||||
"Browser pixel capture preparation is no longer active.",
|
||||
);
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture rejects when the renderer reports preparation failure", async () => {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => "prepare-1",
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: false,
|
||||
message: "renderer could not prep",
|
||||
});
|
||||
|
||||
await expect(preparation).rejects.toThrow("renderer could not prep");
|
||||
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture waits for the guest host renderer before asking for prep", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, null);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => "prepare-1",
|
||||
timeoutMs: 250,
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
await vi.advanceTimersByTimeAsync(49);
|
||||
|
||||
expect(host.sentMessages).toEqual([]);
|
||||
|
||||
contents.hostWebContents = host;
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
]);
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: true,
|
||||
token: "token-a",
|
||||
});
|
||||
|
||||
await expect(preparation).resolves.toEqual({ token: "token-a" });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture retries when the renderer prep handler is not registered yet", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const requestIds = ["prepare-1", "prepare-2"];
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => {
|
||||
const requestId = requestIds.shift();
|
||||
if (!requestId) {
|
||||
throw new Error("Missing request id");
|
||||
}
|
||||
return requestId;
|
||||
},
|
||||
timeoutMs: 250,
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
]);
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: false,
|
||||
message: "Browser pixel capture preparation is unavailable.",
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-2", browserId: "browser-a" },
|
||||
},
|
||||
]);
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-2",
|
||||
ok: true,
|
||||
token: "token-a",
|
||||
});
|
||||
|
||||
await expect(preparation).resolves.toEqual({ token: "token-a" });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture ignores matching responses from the wrong sender", async () => {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => "prepare-1",
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
ipc.emit(
|
||||
"paseo:browser:capture-prepared",
|
||||
{
|
||||
requestId: "prepare-1",
|
||||
ok: true,
|
||||
token: "spoofed-token",
|
||||
},
|
||||
{ senderId: 99 },
|
||||
);
|
||||
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1);
|
||||
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: true,
|
||||
token: "token-a",
|
||||
});
|
||||
|
||||
await expect(preparation).resolves.toEqual({ token: "token-a" });
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture cancels the renderer request when the ack times out", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => "prepare-1",
|
||||
timeoutMs: 25,
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
const rejection = expect(preparation).rejects.toThrow(
|
||||
"Browser pixel capture prepare timed out.",
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(25);
|
||||
|
||||
await rejection;
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
{
|
||||
channel: "paseo:browser:capture-cancel",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
]);
|
||||
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture reports prep_unavailable when the guest host renderer never appears", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const contents = new FakeWebContents(20, null);
|
||||
const tab = adaptWebContents(contents, "browser-a", { timeoutMs: 25 });
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
const rejection = expect(preparation).rejects.toThrow(
|
||||
"Browser screenshot prep_unavailable: Browser host renderer is not available.",
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(25);
|
||||
|
||||
await rejection;
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture rejects when the guest has no embedder renderer", async () => {
|
||||
const contents = new FakeWebContents(20, null);
|
||||
const tab = adaptWebContents(contents, "browser-a", { timeoutMs: 1 });
|
||||
|
||||
await expect(tab.prepareForPixelCapture()).rejects.toThrow("prep_unavailable");
|
||||
it("falls back to a safe filename when the URL has no basename", () => {
|
||||
expect(sanitizeDownloadFileName({ url: "https://example.com/" })).toBe("download");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,113 +1,35 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, join } from "node:path";
|
||||
import type { WebContents } from "electron";
|
||||
import { ipcMain } from "electron";
|
||||
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
|
||||
import type {
|
||||
BrowserAutomationConsoleLogEntry,
|
||||
BrowserAutomationCookieEntry,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { TabContents, BrowserRegistry } from "./service.js";
|
||||
import { executeAutomationCommand } from "./service.js";
|
||||
import {
|
||||
listRegisteredPaseoBrowserIds,
|
||||
listRegisteredPaseoBrowserIdsForWorkspace,
|
||||
getPaseoBrowserWebContents,
|
||||
getWorkspaceActivePaseoBrowserWebContents,
|
||||
getWorkspaceActivePaseoBrowserId,
|
||||
getAgentActivePaseoBrowserId,
|
||||
getPaseoBrowserWorkspaceId,
|
||||
} from "../browser-webviews/index.js";
|
||||
|
||||
const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
|
||||
const PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS = 5_000;
|
||||
const PIXEL_CAPTURE_PREP_RETRY_INTERVAL_MS = 50;
|
||||
const PIXEL_CAPTURE_PREP_UNAVAILABLE_PREFIX = "Browser screenshot prep_unavailable:";
|
||||
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
|
||||
const observedContentsIds = new Set<number>();
|
||||
let nextPixelCaptureBridgeRequest = 0;
|
||||
|
||||
interface IpcHandlerRegistry {
|
||||
handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
|
||||
}
|
||||
|
||||
interface IpcBridgeEvent {
|
||||
sender?: {
|
||||
id?: number;
|
||||
};
|
||||
}
|
||||
|
||||
type IpcListener = (event: IpcBridgeEvent, payload: unknown) => void;
|
||||
|
||||
interface IpcCaptureBridge {
|
||||
on(channel: string, listener: IpcListener): void;
|
||||
removeListener(channel: string, listener: IpcListener): void;
|
||||
}
|
||||
|
||||
interface HostWebContents {
|
||||
readonly id: number;
|
||||
isDestroyed(): boolean;
|
||||
send(channel: string, payload: unknown): void;
|
||||
}
|
||||
|
||||
interface WebContentsDebugger {
|
||||
isAttached(): boolean;
|
||||
attach(protocolVersion?: string): void;
|
||||
sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface ConsoleMessageEmitter {
|
||||
on(
|
||||
event: "console-message",
|
||||
listener: (
|
||||
event: unknown,
|
||||
level: unknown,
|
||||
message: unknown,
|
||||
line: unknown,
|
||||
sourceId: unknown,
|
||||
) => void,
|
||||
): void;
|
||||
once(event: "destroyed", listener: () => void): void;
|
||||
}
|
||||
|
||||
interface BrowserAutomationWebContents extends ConsoleMessageEmitter {
|
||||
readonly id: number;
|
||||
readonly hostWebContents: HostWebContents | null;
|
||||
readonly debugger: WebContentsDebugger;
|
||||
getURL(): string;
|
||||
getTitle(): string;
|
||||
canGoBack(): boolean;
|
||||
canGoForward(): boolean;
|
||||
isLoading(): boolean;
|
||||
isDestroyed(): boolean;
|
||||
executeJavaScript(code: string): Promise<unknown>;
|
||||
loadURL(url: string): Promise<void>;
|
||||
goBack(): void;
|
||||
goForward(): void;
|
||||
reload(): void;
|
||||
capturePage(rect?: Rectangle, options?: { stayHidden?: boolean }): Promise<TabImage>;
|
||||
invalidate(): void;
|
||||
getBackgroundThrottling(): boolean;
|
||||
setBackgroundThrottling(allowed: boolean): void;
|
||||
}
|
||||
|
||||
type PixelCaptureBridgeKind = "prepare" | "restore";
|
||||
|
||||
interface PixelCaptureBridgeSuccess {
|
||||
token?: string;
|
||||
}
|
||||
|
||||
interface PixelCaptureBridgeOptions {
|
||||
ipc?: IpcCaptureBridge;
|
||||
createRequestId?: () => string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
interface PreparedPixelCapture {
|
||||
browserId: string;
|
||||
host: HostWebContents;
|
||||
}
|
||||
|
||||
export function adaptWebContents(
|
||||
contents: BrowserAutomationWebContents,
|
||||
browserId: string,
|
||||
options?: PixelCaptureBridgeOptions,
|
||||
): TabContents {
|
||||
function adaptWebContents(contents: WebContents): TabContents {
|
||||
observeConsoleMessages(contents);
|
||||
const preparedPixelCapturesByToken = new Map<string, PreparedPixelCapture>();
|
||||
return {
|
||||
id: contents.id,
|
||||
getURL: () => contents.getURL(),
|
||||
@@ -121,244 +43,67 @@ export function adaptWebContents(
|
||||
goBack: () => contents.goBack(),
|
||||
goForward: () => contents.goForward(),
|
||||
reload: () => contents.reload(),
|
||||
capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
|
||||
prepareForPixelCapture: async () => {
|
||||
const result = await preparePixelCaptureBridgeWithRetry({ contents, browserId, options });
|
||||
if (!result.token) {
|
||||
throw new Error("Browser pixel capture preparation did not return a token.");
|
||||
}
|
||||
preparedPixelCapturesByToken.set(result.token, { browserId, host: result.host });
|
||||
return { token: result.token };
|
||||
},
|
||||
restorePixelCapture: async (preparation) => {
|
||||
const prepared = preparedPixelCapturesByToken.get(preparation.token);
|
||||
if (!prepared) {
|
||||
throw new Error("Browser pixel capture preparation is no longer active.");
|
||||
}
|
||||
try {
|
||||
await requestPixelCaptureBridge({
|
||||
host: prepared.host,
|
||||
browserId: prepared.browserId,
|
||||
kind: "restore",
|
||||
options,
|
||||
extraPayload: { token: preparation.token },
|
||||
});
|
||||
} finally {
|
||||
preparedPixelCapturesByToken.delete(preparation.token);
|
||||
}
|
||||
},
|
||||
invalidate: () => contents.invalidate(),
|
||||
isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(),
|
||||
setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed),
|
||||
capturePage: () => contents.capturePage(),
|
||||
getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
|
||||
getCookies: async (url: string) =>
|
||||
(await contents.session.cookies.get({ url })).map(normalizeCookie),
|
||||
sendDebugCommand: async (command: string, params?: Record<string, unknown>) => {
|
||||
if (!contents.debugger.isAttached()) {
|
||||
contents.debugger.attach("1.3");
|
||||
}
|
||||
return contents.debugger.sendCommand(command, params ?? {});
|
||||
},
|
||||
printToPDF: async (options?: Record<string, unknown>) => contents.printToPDF(options ?? {}),
|
||||
downloadURL: (input) => downloadWithContents(contents, input),
|
||||
};
|
||||
}
|
||||
|
||||
function getPixelCaptureHost(contents: BrowserAutomationWebContents): HostWebContents | null {
|
||||
const host = contents.hostWebContents;
|
||||
if (!host || host.isDestroyed()) {
|
||||
return null;
|
||||
}
|
||||
return host;
|
||||
}
|
||||
|
||||
async function preparePixelCaptureBridgeWithRetry(input: {
|
||||
contents: BrowserAutomationWebContents;
|
||||
browserId: string;
|
||||
options: PixelCaptureBridgeOptions | undefined;
|
||||
}): Promise<PixelCaptureBridgeSuccess & { host: HostWebContents }> {
|
||||
const timeoutMs = input.options?.timeoutMs ?? PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS;
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastUnavailableReason = "Browser host renderer is not available.";
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const host = getPixelCaptureHost(input.contents);
|
||||
if (!host) {
|
||||
await delayUntilRetry(deadline);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await requestPixelCaptureBridge({
|
||||
host,
|
||||
browserId: input.browserId,
|
||||
kind: "prepare",
|
||||
options: withBridgeTimeout(input.options, Math.max(1, deadline - Date.now())),
|
||||
});
|
||||
return { ...result, host };
|
||||
} catch (error) {
|
||||
if (!isRetryablePrepareUnavailableError(error)) {
|
||||
throw error;
|
||||
}
|
||||
lastUnavailableReason = errorMessage(error);
|
||||
await delayUntilRetry(deadline);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`${PIXEL_CAPTURE_PREP_UNAVAILABLE_PREFIX} ${lastUnavailableReason}`);
|
||||
}
|
||||
|
||||
function withBridgeTimeout(
|
||||
options: PixelCaptureBridgeOptions | undefined,
|
||||
timeoutMs: number,
|
||||
): PixelCaptureBridgeOptions {
|
||||
return {
|
||||
...options,
|
||||
timeoutMs,
|
||||
};
|
||||
}
|
||||
|
||||
function isRetryablePrepareUnavailableError(error: unknown): boolean {
|
||||
const message = errorMessage(error);
|
||||
return (
|
||||
message.includes("Browser pixel capture preparation is unavailable.") ||
|
||||
message.includes("Browser host renderer is not available.") ||
|
||||
message.includes("is not mounted.")
|
||||
);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
async function delayUntilRetry(deadline: number): Promise<void> {
|
||||
const remainingMs = deadline - Date.now();
|
||||
if (remainingMs <= 0) {
|
||||
return;
|
||||
}
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, Math.min(PIXEL_CAPTURE_PREP_RETRY_INTERVAL_MS, remainingMs));
|
||||
});
|
||||
}
|
||||
|
||||
function requestPixelCaptureBridge(input: {
|
||||
host: HostWebContents;
|
||||
browserId: string;
|
||||
kind: PixelCaptureBridgeKind;
|
||||
options: PixelCaptureBridgeOptions | undefined;
|
||||
extraPayload?: { token: string };
|
||||
}): Promise<PixelCaptureBridgeSuccess> {
|
||||
const { host, browserId, kind, options, extraPayload } = input;
|
||||
if (host.isDestroyed()) {
|
||||
return Promise.reject(new Error("Browser host renderer is not available."));
|
||||
}
|
||||
|
||||
const ipc = options?.ipc ?? ipcMain;
|
||||
const requestId =
|
||||
options?.createRequestId?.() ?? `browser-pixel-capture-${++nextPixelCaptureBridgeRequest}`;
|
||||
const timeoutMs = options?.timeoutMs ?? PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS;
|
||||
const requestChannel =
|
||||
kind === "prepare" ? "paseo:browser:capture-prepare" : "paseo:browser:capture-restore";
|
||||
const responseChannel =
|
||||
kind === "prepare" ? "paseo:browser:capture-prepared" : "paseo:browser:capture-restored";
|
||||
|
||||
function downloadWithContents(
|
||||
contents: WebContents,
|
||||
input: { url: string; fileName?: string },
|
||||
): Promise<{ filePath: string; totalBytes?: number; state: string }> {
|
||||
const downloadDir = join(tmpdir(), "paseo-browser-downloads");
|
||||
mkdirSync(downloadDir, { recursive: true });
|
||||
const filePath = join(downloadDir, sanitizeDownloadFileName(input));
|
||||
return new Promise((resolve, reject) => {
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
const cleanup = () => {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
ipc.removeListener(responseChannel, listener);
|
||||
};
|
||||
const listener: IpcListener = (event, payload) => {
|
||||
if (readSenderId(event) !== host.id) {
|
||||
const timeout = setTimeout(() => {
|
||||
contents.session.off("will-download", onDownload);
|
||||
reject(new Error(`Timed out waiting for browser download: ${input.url}`));
|
||||
}, 30_000);
|
||||
function onDownload(_event: Electron.Event, item: Electron.DownloadItem): void {
|
||||
if (item.getURL() !== input.url) {
|
||||
return;
|
||||
}
|
||||
const response = readPixelCaptureBridgeResponse(payload, requestId);
|
||||
if (!response) {
|
||||
return;
|
||||
}
|
||||
cleanup();
|
||||
if (response.ok) {
|
||||
resolve(response);
|
||||
} else {
|
||||
reject(new Error(response.message));
|
||||
}
|
||||
};
|
||||
|
||||
ipc.on(responseChannel, listener);
|
||||
timeoutId = setTimeout(() => {
|
||||
cleanup();
|
||||
sendPixelCaptureCancel(host, {
|
||||
requestId,
|
||||
browserId,
|
||||
...(extraPayload ? { token: extraPayload.token } : {}),
|
||||
clearTimeout(timeout);
|
||||
contents.session.off("will-download", onDownload);
|
||||
item.setSavePath(filePath);
|
||||
item.once("done", (_doneEvent, state) => {
|
||||
resolve({ filePath, totalBytes: item.getTotalBytes(), state });
|
||||
});
|
||||
reject(new Error(`Browser pixel capture ${kind} timed out.`));
|
||||
}, timeoutMs);
|
||||
|
||||
try {
|
||||
host.send(requestChannel, {
|
||||
requestId,
|
||||
browserId,
|
||||
...(extraPayload ? { token: extraPayload.token } : {}),
|
||||
});
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
reject(error);
|
||||
}
|
||||
contents.session.on("will-download", onDownload);
|
||||
contents.downloadURL(input.url);
|
||||
});
|
||||
}
|
||||
|
||||
function sendPixelCaptureCancel(
|
||||
host: HostWebContents,
|
||||
payload: { browserId: string; requestId?: string; token?: string },
|
||||
): void {
|
||||
if (host.isDestroyed()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
host.send("paseo:browser:capture-cancel", payload);
|
||||
} catch {
|
||||
// The original prepare/restore request owns the user-visible error.
|
||||
}
|
||||
export function sanitizeDownloadFileName(input: { url: string; fileName?: string }): string {
|
||||
const requestedName = input.fileName ?? basename(new URL(input.url).pathname);
|
||||
return basename(requestedName) || "download";
|
||||
}
|
||||
|
||||
function readSenderId(event: IpcBridgeEvent): number | null {
|
||||
const senderId = event.sender?.id;
|
||||
return typeof senderId === "number" ? senderId : null;
|
||||
function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry {
|
||||
return {
|
||||
name: cookie.name,
|
||||
value: cookie.value,
|
||||
...(cookie.domain ? { domain: cookie.domain } : {}),
|
||||
...(cookie.path ? { path: cookie.path } : {}),
|
||||
secure: cookie.secure,
|
||||
httpOnly: cookie.httpOnly,
|
||||
...(typeof cookie.expirationDate === "number" ? { expirationDate: cookie.expirationDate } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function readPixelCaptureBridgeResponse(
|
||||
payload: unknown,
|
||||
requestId: string,
|
||||
): ({ ok: true } & PixelCaptureBridgeSuccess) | { ok: false; message: string } | null {
|
||||
if (!isRecord(payload)) {
|
||||
return null;
|
||||
}
|
||||
const record = payload;
|
||||
if (record.requestId !== requestId) {
|
||||
return null;
|
||||
}
|
||||
if (record.ok === true) {
|
||||
return {
|
||||
ok: true,
|
||||
...(typeof record.token === "string" && record.token.length > 0
|
||||
? { token: record.token }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
if (record.ok === false) {
|
||||
const message =
|
||||
typeof record.message === "string" && record.message.length > 0
|
||||
? record.message
|
||||
: "Browser pixel capture bridge failed.";
|
||||
return { ok: false, message };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
|
||||
function observeConsoleMessages(contents: WebContents): void {
|
||||
if (observedContentsIds.has(contents.id)) {
|
||||
return;
|
||||
}
|
||||
@@ -398,10 +143,15 @@ function createRegistry(): BrowserRegistry {
|
||||
listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace,
|
||||
getTabContents(browserId: string): TabContents | null {
|
||||
const contents = getPaseoBrowserWebContents(browserId);
|
||||
return contents ? adaptWebContents(contents, browserId) : null;
|
||||
return contents ? adaptWebContents(contents) : null;
|
||||
},
|
||||
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
|
||||
getWorkspaceActiveTabContents(workspaceId: string): TabContents | null {
|
||||
const contents = getWorkspaceActivePaseoBrowserWebContents(workspaceId);
|
||||
return contents ? adaptWebContents(contents) : null;
|
||||
},
|
||||
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,
|
||||
getAgentActiveBrowserId: getAgentActivePaseoBrowserId,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -44,9 +44,7 @@ describe("BrowserSnapshotEngine", () => {
|
||||
ok: false,
|
||||
reason: "stale_ref",
|
||||
});
|
||||
await expect(
|
||||
engine.select({ browserId: "browser-1", page, ref: "@e1", value: "us" }),
|
||||
).resolves.toEqual({
|
||||
await expect(engine.focus({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: "stale_ref",
|
||||
});
|
||||
|
||||
@@ -108,6 +108,31 @@ export class BrowserSnapshotEngine {
|
||||
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
}
|
||||
|
||||
async focus(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildFocusScript(selector));
|
||||
}
|
||||
|
||||
async clear(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildClearScript(selector));
|
||||
}
|
||||
|
||||
async check(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
checked: boolean;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildCheckScript(selector, input.checked));
|
||||
}
|
||||
|
||||
async select(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
@@ -279,6 +304,57 @@ function buildKeypressScript(selector: string | undefined, key: string): string
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildFocusScript(selector: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
return document.activeElement === element;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildClearScript(selector: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
if ('value' in element) {
|
||||
element.value = '';
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
}
|
||||
element.textContent = '';
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildCheckScript(selector: string, checked: boolean): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
const nextChecked = ${JSON.stringify(checked)};
|
||||
if ('checked' in element) {
|
||||
element.checked = nextChecked;
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
}
|
||||
if (element.getAttribute('role') === 'checkbox' || element.getAttribute('role') === 'radio') {
|
||||
element.setAttribute('aria-checked', String(nextChecked));
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildSelectScript(selector: string, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
|
||||
@@ -75,6 +75,17 @@ export function getWorkspaceActivePaseoBrowserId(workspaceId: string): string |
|
||||
return browserRegistry.getWorkspaceActiveBrowserId(workspaceId);
|
||||
}
|
||||
|
||||
export function setAgentActivePaseoBrowserId(input: {
|
||||
agentId: string;
|
||||
browserId: string | null;
|
||||
}): void {
|
||||
browserRegistry.setAgentActiveBrowser(input);
|
||||
}
|
||||
|
||||
export function getAgentActivePaseoBrowserId(agentId: string): string | null {
|
||||
return browserRegistry.getAgentActiveBrowserId(agentId);
|
||||
}
|
||||
|
||||
export function getPaseoBrowserWebContents(browserId: string): WebContents | null {
|
||||
const contentsId = browserRegistry.getWebContentsIdForBrowser(browserId);
|
||||
if (contentsId === null) {
|
||||
@@ -88,6 +99,16 @@ export function getPaseoBrowserWebContents(browserId: string): WebContents | nul
|
||||
return null;
|
||||
}
|
||||
|
||||
export function getWorkspaceActivePaseoBrowserWebContents(workspaceId: string): WebContents | null {
|
||||
const activeBrowserId = getWorkspaceActivePaseoBrowserId(workspaceId);
|
||||
return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null;
|
||||
}
|
||||
|
||||
export function getAgentActivePaseoBrowserWebContents(agentId: string): WebContents | null {
|
||||
const activeBrowserId = getAgentActivePaseoBrowserId(agentId);
|
||||
return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null;
|
||||
}
|
||||
|
||||
export function getMostRecentWorkspaceActivePaseoBrowserWebContents(): WebContents | null {
|
||||
const browserId = browserRegistry.getMostRecentWorkspaceActiveBrowserId();
|
||||
return browserId ? getPaseoBrowserWebContents(browserId) : null;
|
||||
|
||||
@@ -8,6 +8,7 @@ describe("PaseoBrowserWebviewRegistry", () => {
|
||||
registry.registerWebContents({ webContentsId: 1, browserId: "browser-a" });
|
||||
registry.registerWorkspace({ browserId: "browser-a", workspaceId: "workspace-a" });
|
||||
registry.setWorkspaceActiveBrowser({ workspaceId: "workspace-a", browserId: "browser-a" });
|
||||
registry.setAgentActiveBrowser({ agentId: "agent-a", browserId: "browser-a" });
|
||||
registry.registerWebContents({ webContentsId: 2, browserId: "browser-a" });
|
||||
|
||||
expect(registry.getBrowserIdForWebContents(1)).toBeNull();
|
||||
@@ -15,6 +16,7 @@ describe("PaseoBrowserWebviewRegistry", () => {
|
||||
expect(registry.getWebContentsIdForBrowser("browser-a")).toBe(2);
|
||||
expect(registry.getWorkspaceId("browser-a")).toBe("workspace-a");
|
||||
expect(registry.getWorkspaceActiveBrowserId("workspace-a")).toBe("browser-a");
|
||||
expect(registry.getAgentActiveBrowserId("agent-a")).toBe("browser-a");
|
||||
});
|
||||
|
||||
it("ignores stale destroy events after a duplicate browserId moved", () => {
|
||||
|
||||
@@ -8,6 +8,7 @@ export class PaseoBrowserWebviewRegistry {
|
||||
private readonly webContentsIdsByBrowserId = new Map<string, number>();
|
||||
private readonly workspaceIdsByBrowserId = new Map<string, string>();
|
||||
private readonly activeBrowserIdsByWorkspaceId = new Map<string, string>();
|
||||
private readonly activeBrowserIdsByAgentId = new Map<string, string>();
|
||||
|
||||
public registerWebContents(input: { webContentsId: number; browserId: string }): void {
|
||||
const previousWebContentsId = this.webContentsIdsByBrowserId.get(input.browserId) ?? null;
|
||||
@@ -79,11 +80,29 @@ export class PaseoBrowserWebviewRegistry {
|
||||
return Array.from(this.activeBrowserIdsByWorkspaceId.values()).at(-1) ?? null;
|
||||
}
|
||||
|
||||
public setAgentActiveBrowser(input: { agentId: string; browserId: string | null }): void {
|
||||
if (input.browserId) {
|
||||
this.activeBrowserIdsByAgentId.delete(input.agentId);
|
||||
this.activeBrowserIdsByAgentId.set(input.agentId, input.browserId);
|
||||
return;
|
||||
}
|
||||
this.activeBrowserIdsByAgentId.delete(input.agentId);
|
||||
}
|
||||
|
||||
public getAgentActiveBrowserId(agentId: string): string | null {
|
||||
return this.activeBrowserIdsByAgentId.get(agentId) ?? null;
|
||||
}
|
||||
|
||||
private deleteActiveBrowserReferences(browserId: string): void {
|
||||
for (const [workspaceId, activeBrowserId] of this.activeBrowserIdsByWorkspaceId) {
|
||||
if (activeBrowserId === browserId) {
|
||||
this.activeBrowserIdsByWorkspaceId.delete(workspaceId);
|
||||
}
|
||||
}
|
||||
for (const [agentId, activeBrowserId] of this.activeBrowserIdsByAgentId) {
|
||||
if (activeBrowserId === browserId) {
|
||||
this.activeBrowserIdsByAgentId.delete(agentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@ import {
|
||||
registerBrowserWebviewNavigationGuards,
|
||||
registerPaseoBrowserWorkspace,
|
||||
registerPaseoBrowserWebContents,
|
||||
setAgentActivePaseoBrowserId,
|
||||
setWorkspaceActivePaseoBrowserId,
|
||||
} from "./features/browser-webviews/index.js";
|
||||
import { parseOpenProjectPathFromArgv } from "./open-project-routing.js";
|
||||
@@ -143,6 +144,20 @@ function readActiveBrowserInput(
|
||||
return { workspaceId: record.workspaceId.trim(), browserId: browserId || null };
|
||||
}
|
||||
|
||||
function readAgentActiveBrowserInput(
|
||||
input: unknown,
|
||||
): { agentId: string; browserId: string | null } | null {
|
||||
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
||||
return null;
|
||||
}
|
||||
const record = input as Record<string, unknown>;
|
||||
if (typeof record.agentId !== "string" || record.agentId.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
const browserId = typeof record.browserId === "string" ? record.browserId.trim() : null;
|
||||
return { agentId: record.agentId.trim(), browserId: browserId || null };
|
||||
}
|
||||
|
||||
const pendingBrowserWebviewIds: string[] = [];
|
||||
|
||||
function isBrowserRefreshInput(input: Electron.Input): boolean {
|
||||
@@ -335,6 +350,13 @@ ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput:
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle("paseo:browser:set-agent-active-browser", (_event, rawInput: unknown) => {
|
||||
const input = readAgentActiveBrowserInput(rawInput);
|
||||
if (input) {
|
||||
setAgentActivePaseoBrowserId(input);
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle("paseo:browser:open-devtools", (_event, browserId: unknown) => {
|
||||
if (typeof browserId !== "string" || browserId.trim().length === 0) {
|
||||
const result = {
|
||||
|
||||
@@ -1,117 +1,6 @@
|
||||
import { contextBridge, ipcRenderer, webUtils } from "electron";
|
||||
|
||||
type EventHandler = (payload: unknown) => void;
|
||||
type BrowserPixelCapturePrepareHandler = (input: {
|
||||
requestId: string;
|
||||
browserId: string;
|
||||
}) => Promise<{ token: string }>;
|
||||
type BrowserPixelCaptureRestoreHandler = (input: { token: string }) => Promise<void>;
|
||||
type BrowserPixelCaptureCancelHandler = (input: {
|
||||
requestId?: string;
|
||||
token?: string;
|
||||
}) => Promise<void>;
|
||||
|
||||
let prepareForPixelCaptureHandler: BrowserPixelCapturePrepareHandler | null = null;
|
||||
let restorePixelCaptureHandler: BrowserPixelCaptureRestoreHandler | null = null;
|
||||
let cancelPixelCaptureHandler: BrowserPixelCaptureCancelHandler | null = null;
|
||||
const canceledPixelCaptureRequestIds = new Set<string>();
|
||||
|
||||
function readStringField(payload: unknown, key: string): string | null {
|
||||
if (!isRecord(payload)) {
|
||||
return null;
|
||||
}
|
||||
const value = payload[key];
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
ipcRenderer.on("paseo:browser:capture-prepare", async (_event, payload: unknown) => {
|
||||
const requestId = readStringField(payload, "requestId");
|
||||
const browserId = readStringField(payload, "browserId");
|
||||
if (!requestId || !browserId || !prepareForPixelCaptureHandler) {
|
||||
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||
requestId: requestId ?? "unknown",
|
||||
ok: false,
|
||||
message: "Browser pixel capture preparation is unavailable.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const preparation = await prepareForPixelCaptureHandler({ requestId, browserId });
|
||||
if (canceledPixelCaptureRequestIds.delete(requestId)) {
|
||||
await restorePixelCaptureHandler?.({ token: preparation.token });
|
||||
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||
requestId,
|
||||
ok: false,
|
||||
message: "Browser pixel capture preparation was canceled.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||
requestId,
|
||||
ok: true,
|
||||
token: preparation.token,
|
||||
});
|
||||
} catch (error) {
|
||||
canceledPixelCaptureRequestIds.delete(requestId);
|
||||
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||
requestId,
|
||||
ok: false,
|
||||
message: errorMessage(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
ipcRenderer.on("paseo:browser:capture-cancel", async (_event, payload: unknown) => {
|
||||
const requestId = readStringField(payload, "requestId");
|
||||
const token = readStringField(payload, "token");
|
||||
if (requestId) {
|
||||
canceledPixelCaptureRequestIds.add(requestId);
|
||||
}
|
||||
if (!requestId && !token) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await cancelPixelCaptureHandler?.({
|
||||
...(requestId ? { requestId } : {}),
|
||||
...(token ? { token } : {}),
|
||||
});
|
||||
} catch {
|
||||
// The original prepare/restore request owns the user-visible error.
|
||||
}
|
||||
});
|
||||
|
||||
ipcRenderer.on("paseo:browser:capture-restore", async (_event, payload: unknown) => {
|
||||
const requestId = readStringField(payload, "requestId");
|
||||
const token = readStringField(payload, "token");
|
||||
if (!requestId || !token || !restorePixelCaptureHandler) {
|
||||
ipcRenderer.send("paseo:browser:capture-restored", {
|
||||
requestId: requestId ?? "unknown",
|
||||
ok: false,
|
||||
message: "Browser pixel capture restore is unavailable.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await restorePixelCaptureHandler({ token });
|
||||
ipcRenderer.send("paseo:browser:capture-restored", { requestId, ok: true });
|
||||
} catch (error) {
|
||||
ipcRenderer.send("paseo:browser:capture-restored", {
|
||||
requestId,
|
||||
ok: false,
|
||||
message: errorMessage(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
platform: process.platform,
|
||||
@@ -189,6 +78,8 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
ipcRenderer.invoke("paseo:browser:register-workspace-browser", input),
|
||||
setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) =>
|
||||
ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input),
|
||||
setAgentActiveBrowser: (input: { agentId: string; browserId: string | null }) =>
|
||||
ipcRenderer.invoke("paseo:browser:set-agent-active-browser", input),
|
||||
openDevTools: (browserId: string) =>
|
||||
ipcRenderer.invoke("paseo:browser:open-devtools", browserId),
|
||||
clearPartition: (browserId: string) =>
|
||||
@@ -201,29 +92,5 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect),
|
||||
copyElement: (payload: { text?: string; imageDataUrl?: string }) =>
|
||||
ipcRenderer.invoke("paseo:browser:copy-element", payload),
|
||||
onPrepareForPixelCapture: (handler: BrowserPixelCapturePrepareHandler): (() => void) => {
|
||||
prepareForPixelCaptureHandler = handler;
|
||||
return () => {
|
||||
if (prepareForPixelCaptureHandler === handler) {
|
||||
prepareForPixelCaptureHandler = null;
|
||||
}
|
||||
};
|
||||
},
|
||||
onRestorePixelCapture: (handler: BrowserPixelCaptureRestoreHandler): (() => void) => {
|
||||
restorePixelCaptureHandler = handler;
|
||||
return () => {
|
||||
if (restorePixelCaptureHandler === handler) {
|
||||
restorePixelCaptureHandler = null;
|
||||
}
|
||||
};
|
||||
},
|
||||
onCancelPixelCapture: (handler: BrowserPixelCaptureCancelHandler): (() => void) => {
|
||||
cancelPixelCaptureHandler = handler;
|
||||
return () => {
|
||||
if (cancelPixelCaptureHandler === handler) {
|
||||
cancelPixelCaptureHandler = null;
|
||||
}
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/expo-two-way-audio",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"description": "Native module for two way audio streaming",
|
||||
"keywords": [
|
||||
"ExpoTwoWayAudio",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/highlight",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"files": [
|
||||
"dist",
|
||||
"!dist/**/*.map"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/protocol",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"description": "Paseo shared protocol schemas and wire types",
|
||||
"files": [
|
||||
"dist",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,33 +3,20 @@ import { z } from "zod";
|
||||
export const BrowserAutomationErrorCodeSchema = z.enum([
|
||||
"browser_disabled",
|
||||
"browser_no_desktop",
|
||||
"browser_no_tab",
|
||||
"browser_tab_not_found",
|
||||
"browser_tab_closed",
|
||||
"browser_timeout",
|
||||
"screenshot_no_frame",
|
||||
"browser_denied",
|
||||
"browser_unsupported",
|
||||
"browser_stale_ref",
|
||||
"browser_unknown_error",
|
||||
]);
|
||||
|
||||
const BROWSER_AUTOMATION_BROWSER_ID_PATTERN =
|
||||
/^(?:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|\d{13,}-[0-9a-f]+)$/i;
|
||||
const BROWSER_AUTOMATION_BROWSER_ID_MESSAGE =
|
||||
"browserId must be a real id returned by browser_new_tab or browser_list_tabs";
|
||||
const BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE =
|
||||
"browser_wait requires exactly one of text or url";
|
||||
|
||||
export const BrowserAutomationBrowserIdSchema = z
|
||||
.string({ error: () => BROWSER_AUTOMATION_BROWSER_ID_MESSAGE })
|
||||
.min(1, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE)
|
||||
.regex(BROWSER_AUTOMATION_BROWSER_ID_PATTERN, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE);
|
||||
|
||||
const BrowserAutomationTabTargetSchema = z
|
||||
.object({
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
})
|
||||
.strict();
|
||||
const BrowserAutomationTabTargetSchema = z.object({
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
browserId: z.string().min(1).optional(),
|
||||
});
|
||||
|
||||
const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/);
|
||||
const BrowserAutomationHttpUrlSchema = z
|
||||
@@ -42,22 +29,31 @@ const BrowserAutomationHttpUrlSchema = z
|
||||
|
||||
export const BrowserAutomationListTabsCommandSchema = z.object({
|
||||
command: z.literal("list_tabs"),
|
||||
args: z.object({}).strict().default({}),
|
||||
args: z
|
||||
.object({
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
})
|
||||
.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationNewTabCommandSchema = z.object({
|
||||
command: z.literal("new_tab"),
|
||||
args: z
|
||||
.object({
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
url: BrowserAutomationHttpUrlSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationPageInfoCommandSchema = z.object({
|
||||
command: z.literal("page_info"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSnapshotCommandSchema = z.object({
|
||||
command: z.literal("snapshot"),
|
||||
args: BrowserAutomationTabTargetSchema,
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationClickCommandSchema = z.object({
|
||||
@@ -81,8 +77,6 @@ export const BrowserAutomationWaitCommandSchema = z.object({
|
||||
text: z.string().min(1).optional(),
|
||||
url: z.string().min(1).optional(),
|
||||
timeoutMs: z.number().int().positive().max(30_000).optional(),
|
||||
}).refine((args) => Number(Boolean(args.text)) + Number(Boolean(args.url)) === 1, {
|
||||
message: BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE,
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -111,23 +105,42 @@ export const BrowserAutomationNavigateCommandSchema = z.object({
|
||||
|
||||
export const BrowserAutomationBackCommandSchema = z.object({
|
||||
command: z.literal("back"),
|
||||
args: BrowserAutomationTabTargetSchema,
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationForwardCommandSchema = z.object({
|
||||
command: z.literal("forward"),
|
||||
args: BrowserAutomationTabTargetSchema,
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationReloadCommandSchema = z.object({
|
||||
command: z.literal("reload"),
|
||||
args: BrowserAutomationTabTargetSchema,
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationScreenshotCommandSchema = z.object({
|
||||
command: z.literal("screenshot"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({
|
||||
command: z.literal("full_page_screenshot"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationPdfCommandSchema = z.object({
|
||||
command: z.literal("pdf"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
fullPage: z.boolean().default(false),
|
||||
landscape: z.boolean().optional(),
|
||||
printBackground: z.boolean().default(true),
|
||||
}).default({ printBackground: true }),
|
||||
});
|
||||
|
||||
export const BrowserAutomationDownloadCommandSchema = z.object({
|
||||
command: z.literal("download"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
url: BrowserAutomationHttpUrlSchema,
|
||||
fileName: z.string().min(1).optional(),
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -139,6 +152,28 @@ export const BrowserAutomationUploadCommandSchema = z.object({
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFocusCommandSchema = z.object({
|
||||
command: z.literal("focus"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationClearCommandSchema = z.object({
|
||||
command: z.literal("clear"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationCheckCommandSchema = z.object({
|
||||
command: z.literal("check"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
checked: z.boolean().default(true),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSelectCommandSchema = z.object({
|
||||
command: z.literal("select"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
@@ -166,12 +201,45 @@ export const BrowserAutomationLogsCommandSchema = z.object({
|
||||
command: z.literal("logs"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
maxEntries: z.number().int().positive().max(200).default(50),
|
||||
}).default({ maxEntries: 50 }),
|
||||
});
|
||||
|
||||
export const BrowserAutomationStorageCommandSchema = z.object({
|
||||
command: z.literal("storage"),
|
||||
args: BrowserAutomationTabTargetSchema.default({}),
|
||||
});
|
||||
|
||||
const BrowserAutomationViewportInputSchema = z.object({
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
deviceScaleFactor: z.number().positive().optional(),
|
||||
});
|
||||
|
||||
const BrowserAutomationGeolocationInputSchema = z.object({
|
||||
latitude: z.number().min(-90).max(90),
|
||||
longitude: z.number().min(-180).max(180),
|
||||
accuracy: z.number().positive().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationEnvironmentCommandSchema = z.object({
|
||||
command: z.literal("environment"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
viewport: BrowserAutomationViewportInputSchema.optional(),
|
||||
geolocation: BrowserAutomationGeolocationInputSchema.optional(),
|
||||
}).default({}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSetBackgroundCommandSchema = z.object({
|
||||
command: z.literal("set_background"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
color: z.string().min(1),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationListTabsCommandSchema,
|
||||
BrowserAutomationNewTabCommandSchema,
|
||||
BrowserAutomationPageInfoCommandSchema,
|
||||
BrowserAutomationSnapshotCommandSchema,
|
||||
BrowserAutomationClickCommandSchema,
|
||||
BrowserAutomationFillCommandSchema,
|
||||
@@ -183,15 +251,24 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationForwardCommandSchema,
|
||||
BrowserAutomationReloadCommandSchema,
|
||||
BrowserAutomationScreenshotCommandSchema,
|
||||
BrowserAutomationFullPageScreenshotCommandSchema,
|
||||
BrowserAutomationPdfCommandSchema,
|
||||
BrowserAutomationDownloadCommandSchema,
|
||||
BrowserAutomationUploadCommandSchema,
|
||||
BrowserAutomationFocusCommandSchema,
|
||||
BrowserAutomationClearCommandSchema,
|
||||
BrowserAutomationCheckCommandSchema,
|
||||
BrowserAutomationSelectCommandSchema,
|
||||
BrowserAutomationHoverCommandSchema,
|
||||
BrowserAutomationDragCommandSchema,
|
||||
BrowserAutomationLogsCommandSchema,
|
||||
BrowserAutomationStorageCommandSchema,
|
||||
BrowserAutomationEnvironmentCommandSchema,
|
||||
BrowserAutomationSetBackgroundCommandSchema,
|
||||
]);
|
||||
|
||||
export const BrowserAutomationTabInfoSchema = z.object({
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
url: z.string(),
|
||||
title: z.string(),
|
||||
@@ -208,11 +285,16 @@ export const BrowserAutomationListTabsResultSchema = z.object({
|
||||
|
||||
export const BrowserAutomationNewTabResultSchema = z.object({
|
||||
command: z.literal("new_tab"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
workspaceId: z.string().min(1),
|
||||
url: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationPageInfoResultSchema = z.object({
|
||||
command: z.literal("page_info"),
|
||||
tab: BrowserAutomationTabInfoSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationSnapshotElementSchema = z.object({
|
||||
ref: z.string().regex(/^@e\d+$/),
|
||||
role: z.string(),
|
||||
@@ -224,7 +306,7 @@ export const BrowserAutomationSnapshotElementSchema = z.object({
|
||||
|
||||
export const BrowserAutomationSnapshotResultSchema = z.object({
|
||||
command: z.literal("snapshot"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
url: z.string(),
|
||||
title: z.string(),
|
||||
@@ -233,88 +315,132 @@ export const BrowserAutomationSnapshotResultSchema = z.object({
|
||||
|
||||
export const BrowserAutomationClickResultSchema = z.object({
|
||||
command: z.literal("click"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationFillResultSchema = z.object({
|
||||
command: z.literal("fill"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationWaitResultSchema = z.object({
|
||||
command: z.literal("wait"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
matched: z.enum(["text", "url"]),
|
||||
});
|
||||
|
||||
export const BrowserAutomationTypeResultSchema = z.object({
|
||||
command: z.literal("type"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationKeypressResultSchema = z.object({
|
||||
command: z.literal("keypress"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
key: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationNavigateResultSchema = z.object({
|
||||
command: z.literal("navigate"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
url: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationBackResultSchema = z.object({
|
||||
command: z.literal("back"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationForwardResultSchema = z.object({
|
||||
command: z.literal("forward"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationReloadResultSchema = z.object({
|
||||
command: z.literal("reload"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationScreenshotResultSchema = z.object({
|
||||
command: z.literal("screenshot"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
mimeType: z.literal("image/png"),
|
||||
dataBase64: z.string().min(1),
|
||||
width: z.number().int().nonnegative(),
|
||||
height: z.number().int().nonnegative(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFullPageScreenshotResultSchema = z.object({
|
||||
command: z.literal("full_page_screenshot"),
|
||||
browserId: z.string().min(1),
|
||||
mimeType: z.literal("image/png"),
|
||||
dataBase64: z.string().min(1),
|
||||
width: z.number().int().nonnegative(),
|
||||
height: z.number().int().nonnegative(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationPdfResultSchema = z.object({
|
||||
command: z.literal("pdf"),
|
||||
browserId: z.string().min(1),
|
||||
mimeType: z.literal("application/pdf"),
|
||||
dataBase64: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationDownloadResultSchema = z.object({
|
||||
command: z.literal("download"),
|
||||
browserId: z.string().min(1),
|
||||
url: z.string().min(1),
|
||||
filePath: z.string().min(1),
|
||||
totalBytes: z.number().int().nonnegative().optional(),
|
||||
state: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationUploadResultSchema = z.object({
|
||||
command: z.literal("upload"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
filePaths: z.array(z.string().min(1)).min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFocusResultSchema = z.object({
|
||||
command: z.literal("focus"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationClearResultSchema = z.object({
|
||||
command: z.literal("clear"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationCheckResultSchema = z.object({
|
||||
command: z.literal("check"),
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
checked: z.boolean(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSelectResultSchema = z.object({
|
||||
command: z.literal("select"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
value: z.string(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationHoverResultSchema = z.object({
|
||||
command: z.literal("hover"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationDragResultSchema = z.object({
|
||||
command: z.literal("drag"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
sourceRef: BrowserAutomationRefSchema,
|
||||
targetRef: BrowserAutomationRefSchema,
|
||||
});
|
||||
@@ -339,14 +465,64 @@ export const BrowserAutomationNetworkLogEntrySchema = z.object({
|
||||
|
||||
export const BrowserAutomationLogsResultSchema = z.object({
|
||||
command: z.literal("logs"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
browserId: z.string().min(1),
|
||||
console: z.array(BrowserAutomationConsoleLogEntrySchema),
|
||||
network: z.array(BrowserAutomationNetworkLogEntrySchema),
|
||||
});
|
||||
|
||||
export const BrowserAutomationCookieEntrySchema = z.object({
|
||||
name: z.string(),
|
||||
value: z.string(),
|
||||
domain: z.string().optional(),
|
||||
path: z.string().optional(),
|
||||
secure: z.boolean().optional(),
|
||||
httpOnly: z.boolean().optional(),
|
||||
expirationDate: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationStorageEntrySchema = z.object({
|
||||
key: z.string(),
|
||||
value: z.string(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationStorageResultSchema = z.object({
|
||||
command: z.literal("storage"),
|
||||
browserId: z.string().min(1),
|
||||
url: z.string(),
|
||||
cookies: z.array(BrowserAutomationCookieEntrySchema),
|
||||
localStorage: z.array(BrowserAutomationStorageEntrySchema),
|
||||
sessionStorage: z.array(BrowserAutomationStorageEntrySchema),
|
||||
});
|
||||
|
||||
export const BrowserAutomationViewportResultSchema = z.object({
|
||||
width: z.number().int().nonnegative(),
|
||||
height: z.number().int().nonnegative(),
|
||||
deviceScaleFactor: z.number().positive(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationGeolocationResultSchema = z.object({
|
||||
latitude: z.number(),
|
||||
longitude: z.number(),
|
||||
accuracy: z.number(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationEnvironmentResultSchema = z.object({
|
||||
command: z.literal("environment"),
|
||||
browserId: z.string().min(1),
|
||||
viewport: BrowserAutomationViewportResultSchema,
|
||||
geolocation: BrowserAutomationGeolocationResultSchema.optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSetBackgroundResultSchema = z.object({
|
||||
command: z.literal("set_background"),
|
||||
browserId: z.string().min(1),
|
||||
color: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationListTabsResultSchema,
|
||||
BrowserAutomationNewTabResultSchema,
|
||||
BrowserAutomationPageInfoResultSchema,
|
||||
BrowserAutomationSnapshotResultSchema,
|
||||
BrowserAutomationClickResultSchema,
|
||||
BrowserAutomationFillResultSchema,
|
||||
@@ -358,11 +534,20 @@ export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationForwardResultSchema,
|
||||
BrowserAutomationReloadResultSchema,
|
||||
BrowserAutomationScreenshotResultSchema,
|
||||
BrowserAutomationFullPageScreenshotResultSchema,
|
||||
BrowserAutomationPdfResultSchema,
|
||||
BrowserAutomationDownloadResultSchema,
|
||||
BrowserAutomationUploadResultSchema,
|
||||
BrowserAutomationFocusResultSchema,
|
||||
BrowserAutomationClearResultSchema,
|
||||
BrowserAutomationCheckResultSchema,
|
||||
BrowserAutomationSelectResultSchema,
|
||||
BrowserAutomationHoverResultSchema,
|
||||
BrowserAutomationDragResultSchema,
|
||||
BrowserAutomationLogsResultSchema,
|
||||
BrowserAutomationStorageResultSchema,
|
||||
BrowserAutomationEnvironmentResultSchema,
|
||||
BrowserAutomationSetBackgroundResultSchema,
|
||||
]);
|
||||
|
||||
export const BrowserAutomationErrorSchema = z.object({
|
||||
@@ -371,16 +556,15 @@ export const BrowserAutomationErrorSchema = z.object({
|
||||
retryable: z.boolean().default(false),
|
||||
});
|
||||
|
||||
export const BrowserAutomationExecuteRequestSchema = z
|
||||
.object({
|
||||
type: z.literal("browser.automation.execute.request"),
|
||||
requestId: z.string().min(1),
|
||||
agentId: z.string().min(1).optional(),
|
||||
cwd: z.string().min(1).optional(),
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
command: BrowserAutomationCommandSchema,
|
||||
})
|
||||
.strict();
|
||||
export const BrowserAutomationExecuteRequestSchema = z.object({
|
||||
type: z.literal("browser.automation.execute.request"),
|
||||
requestId: z.string().min(1),
|
||||
agentId: z.string().min(1).optional(),
|
||||
cwd: z.string().min(1).optional(),
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
browserId: z.string().min(1).optional(),
|
||||
command: BrowserAutomationCommandSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationExecuteResponseSchema = z.object({
|
||||
type: z.literal("browser.automation.execute.response"),
|
||||
@@ -407,6 +591,8 @@ export type BrowserAutomationConsoleLogEntry = z.infer<
|
||||
export type BrowserAutomationNetworkLogEntry = z.infer<
|
||||
typeof BrowserAutomationNetworkLogEntrySchema
|
||||
>;
|
||||
export type BrowserAutomationCookieEntry = z.infer<typeof BrowserAutomationCookieEntrySchema>;
|
||||
export type BrowserAutomationStorageEntry = z.infer<typeof BrowserAutomationStorageEntrySchema>;
|
||||
export type BrowserAutomationExecuteRequest = z.infer<typeof BrowserAutomationExecuteRequestSchema>;
|
||||
export type BrowserAutomationExecuteResponse = z.infer<
|
||||
typeof BrowserAutomationExecuteResponseSchema
|
||||
|
||||
@@ -10,8 +10,6 @@ import {
|
||||
} from "./messages.js";
|
||||
|
||||
describe("browser automation protocol integration", () => {
|
||||
const browserId = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
test("desktop automation capability parses in hello without narrowing old clients", () => {
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
@@ -41,7 +39,7 @@ describe("browser automation protocol integration", () => {
|
||||
const parsed = SessionOutboundMessageSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
command: { command: "snapshot", args: { browserId } },
|
||||
command: { command: "page_info", args: { browserId: "browser-1" } },
|
||||
});
|
||||
|
||||
expect(parsed.type).toBe("browser.automation.execute.request");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/relay",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"description": "Paseo relay for bridging daemon and client connections",
|
||||
"files": [
|
||||
"dist",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/server",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"description": "Paseo backend server",
|
||||
"files": [
|
||||
"dist/server",
|
||||
@@ -65,10 +65,10 @@
|
||||
"@agentclientprotocol/sdk": "^0.17.1",
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.3.195",
|
||||
"@anthropic-ai/sdk": "^0.104.2",
|
||||
"@getpaseo/client": "0.1.104-beta.1",
|
||||
"@getpaseo/highlight": "0.1.104-beta.1",
|
||||
"@getpaseo/protocol": "0.1.104-beta.1",
|
||||
"@getpaseo/relay": "0.1.104-beta.1",
|
||||
"@getpaseo/client": "0.1.103",
|
||||
"@getpaseo/highlight": "0.1.103",
|
||||
"@getpaseo/protocol": "0.1.103",
|
||||
"@getpaseo/relay": "0.1.103",
|
||||
"@isaacs/ttlcache": "^2.1.4",
|
||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||
"@opencode-ai/sdk": "1.14.46",
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { realpathSync, rmSync } from "node:fs";
|
||||
import { access, mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
@@ -11,12 +9,10 @@ import { z } from "zod";
|
||||
|
||||
import { createTestLogger } from "../../test-utils/test-logger.js";
|
||||
import { createAgentMcpServer } from "./mcp-server.js";
|
||||
import { createPaseoToolCatalog } from "./tools/paseo-tools.js";
|
||||
import { AgentManager, type ManagedAgent } from "./agent-manager.js";
|
||||
import { AgentStorage, type StoredAgentRecord } from "./agent-storage.js";
|
||||
import { createTestAgentClients } from "../test-utils/fake-agent-client.js";
|
||||
import type { AgentMode, AgentProvider, ProviderSnapshotEntry } from "./agent-sdk-types.js";
|
||||
import type { ProviderSnapshotManager } from "./provider-snapshot-manager.js";
|
||||
import { createProviderSnapshotManagerStub } from "../test-utils/session-stubs.js";
|
||||
import {
|
||||
AgentListItemPayloadSchema,
|
||||
@@ -40,12 +36,9 @@ import { WorkspaceGitServiceImpl } from "../workspace-git-service.js";
|
||||
import type { GitHubService } from "../../services/github-service.js";
|
||||
import type { TerminalManager } from "../../terminal/terminal-manager.js";
|
||||
import { PARENT_AGENT_ID_LABEL } from "@getpaseo/protocol/agent-labels";
|
||||
import type { BrowserToolsBroker, BrowserToolsExecuteInput } from "../browser-tools/broker.js";
|
||||
import type { BrowserToolsResponsePayload } from "../browser-tools/errors.js";
|
||||
|
||||
const REPO_CWD = resolvePath("/tmp/repo");
|
||||
const TARGET_CWD = resolvePath("/tmp/target");
|
||||
const BROWSER_WORKSPACE_ID = "wks_browser_tools";
|
||||
|
||||
interface LooseSafeParseResult {
|
||||
success: boolean;
|
||||
@@ -63,13 +56,6 @@ interface LooseStructuredContent {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
interface LooseContentBlock {
|
||||
type: string;
|
||||
text?: string;
|
||||
data?: string;
|
||||
mimeType?: string;
|
||||
}
|
||||
|
||||
interface RegisteredMcpTool {
|
||||
inputSchema: LooseInputSchema;
|
||||
outputSchema?: unknown;
|
||||
@@ -78,18 +64,18 @@ interface RegisteredMcpTool {
|
||||
extra?: unknown,
|
||||
) => Promise<{
|
||||
structuredContent: LooseStructuredContent;
|
||||
content?: LooseContentBlock[];
|
||||
content?: Array<{ type: string; text?: string }>;
|
||||
}>;
|
||||
handler?: (input: unknown) => Promise<{
|
||||
structuredContent: LooseStructuredContent;
|
||||
content?: LooseContentBlock[];
|
||||
content?: Array<{ type: string; text?: string }>;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface RegisteredMcpToolWithHandler extends RegisteredMcpTool {
|
||||
handler: (input: unknown) => Promise<{
|
||||
structuredContent: LooseStructuredContent;
|
||||
content?: LooseContentBlock[];
|
||||
content?: Array<{ type: string; text?: string }>;
|
||||
}>;
|
||||
}
|
||||
|
||||
@@ -500,77 +486,6 @@ function createGitHubServiceStub(): GitHubService {
|
||||
};
|
||||
}
|
||||
|
||||
class FakeBrowserToolsBroker {
|
||||
public readonly calls: BrowserToolsExecuteInput[] = [];
|
||||
|
||||
public constructor(private readonly response: BrowserToolsResponsePayload) {}
|
||||
|
||||
public async execute(input: BrowserToolsExecuteInput): Promise<BrowserToolsResponsePayload> {
|
||||
this.calls.push(input);
|
||||
return this.response;
|
||||
}
|
||||
}
|
||||
|
||||
class BoundaryAgentManagerFake {
|
||||
private readonly agent = createManagedAgent({
|
||||
id: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: BROWSER_WORKSPACE_ID,
|
||||
});
|
||||
|
||||
public getAgent(agentId: string): ManagedAgent | null {
|
||||
return agentId === this.agent.id ? this.agent : null;
|
||||
}
|
||||
|
||||
public listAgents(): ManagedAgent[] {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
class BoundaryAgentStorageFake {
|
||||
public async list(): Promise<StoredAgentRecord[]> {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
class BoundaryProviderSnapshotManagerFake {
|
||||
public listRegisteredProviderIds(): AgentProvider[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
public hasProvider(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public getProviderLabel(provider: AgentProvider): string {
|
||||
return provider;
|
||||
}
|
||||
|
||||
public async listProviders(): Promise<ProviderSnapshotEntry[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
public async getProvider(): Promise<ProviderSnapshotEntry> {
|
||||
throw new Error("Provider catalog is not used by this boundary test");
|
||||
}
|
||||
|
||||
public async listModels(): Promise<[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
public async listModes(): Promise<[]> {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function connectInMemoryMcpClient(server: Awaited<ReturnType<typeof createAgentMcpServer>>) {
|
||||
const client = new Client({ name: "paseo-test-client", version: "0.0.0" });
|
||||
const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
|
||||
await server.connect(serverTransport);
|
||||
await client.connect(clientTransport);
|
||||
return client;
|
||||
}
|
||||
|
||||
function createStoredSchedule(input: CreateScheduleInput): StoredSchedule {
|
||||
const now = "2026-04-11T00:00:00.000Z";
|
||||
return {
|
||||
@@ -666,153 +581,9 @@ function createPaseoWorktreeForMcpTest(options: {
|
||||
describe("browser MCP tools", () => {
|
||||
const logger = createTestLogger();
|
||||
|
||||
it("calls registered tools through the MCP SDK with listed output schemas", async () => {
|
||||
const agentManager = new BoundaryAgentManagerFake();
|
||||
const agentStorage = new BoundaryAgentStorageFake();
|
||||
const broker = new FakeBrowserToolsBroker({
|
||||
requestId: "req-browser-tabs",
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
});
|
||||
const serverOptions = {
|
||||
agentManager: agentManager as AgentManager,
|
||||
agentStorage: agentStorage as AgentStorage,
|
||||
providerSnapshotManager:
|
||||
new BoundaryProviderSnapshotManagerFake() as unknown as ProviderSnapshotManager,
|
||||
browserToolsBroker: broker as BrowserToolsBroker,
|
||||
callerAgentId: "agent-1",
|
||||
logger,
|
||||
};
|
||||
const server = await createAgentMcpServer(serverOptions);
|
||||
const client = await connectInMemoryMcpClient(server);
|
||||
|
||||
try {
|
||||
const browserResult = await client.callTool({
|
||||
name: "browser_list_tabs",
|
||||
arguments: {},
|
||||
});
|
||||
const listAgentsResult = await client.callTool({
|
||||
name: "list_agents",
|
||||
arguments: {},
|
||||
});
|
||||
|
||||
expect(broker.calls).toEqual([
|
||||
{
|
||||
agentId: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: BROWSER_WORKSPACE_ID,
|
||||
command: { command: "list_tabs", args: {} },
|
||||
},
|
||||
]);
|
||||
expect(browserResult.isError).not.toBe(true);
|
||||
expect(browserResult.structuredContent).toEqual({
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: BROWSER_WORKSPACE_ID },
|
||||
});
|
||||
expect(listAgentsResult.isError).not.toBe(true);
|
||||
expect(listAgentsResult.structuredContent).toEqual({
|
||||
agents: [],
|
||||
});
|
||||
|
||||
const listedTools = await client.listTools();
|
||||
const toolsByName = new Map(listedTools.tools.map((tool) => [tool.name, tool]));
|
||||
const catalog = createPaseoToolCatalog(serverOptions);
|
||||
|
||||
for (const tool of catalog.tools.values()) {
|
||||
if (tool.outputSchema !== undefined) {
|
||||
expect(toolsByName.get(tool.name)?.outputSchema, `${tool.name} outputSchema`).toEqual(
|
||||
expect.objectContaining({ type: "object" }),
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await client.close();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("returns screenshot pixels as image content and keeps structured content metadata-only", async () => {
|
||||
const agentManager = new BoundaryAgentManagerFake();
|
||||
const agentStorage = new BoundaryAgentStorageFake();
|
||||
const broker = new FakeBrowserToolsBroker({
|
||||
requestId: "req-browser-screenshot",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "screenshot",
|
||||
browserId: "11111111-1111-4111-8111-111111111111",
|
||||
mimeType: "image/png",
|
||||
dataBase64: "iVBORw0KGgo=",
|
||||
width: 800,
|
||||
height: 600,
|
||||
},
|
||||
});
|
||||
const server = await createAgentMcpServer({
|
||||
agentManager: agentManager as AgentManager,
|
||||
agentStorage: agentStorage as AgentStorage,
|
||||
providerSnapshotManager:
|
||||
new BoundaryProviderSnapshotManagerFake() as unknown as ProviderSnapshotManager,
|
||||
browserToolsBroker: broker as BrowserToolsBroker,
|
||||
callerAgentId: "agent-1",
|
||||
logger,
|
||||
});
|
||||
const client = await connectInMemoryMcpClient(server);
|
||||
|
||||
try {
|
||||
const response = await client.callTool({
|
||||
name: "browser_screenshot",
|
||||
arguments: { browserId: "11111111-1111-4111-8111-111111111111" },
|
||||
});
|
||||
|
||||
expect(broker.calls).toEqual([
|
||||
{
|
||||
agentId: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: BROWSER_WORKSPACE_ID,
|
||||
command: {
|
||||
command: "screenshot",
|
||||
args: {
|
||||
browserId: "11111111-1111-4111-8111-111111111111",
|
||||
fullPage: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(response.content).toEqual([
|
||||
{ type: "text", text: "Captured browser screenshot (800x600)." },
|
||||
{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
|
||||
]);
|
||||
expect(response.structuredContent).toEqual({
|
||||
ok: true,
|
||||
result: {
|
||||
command: "screenshot",
|
||||
browserId: "11111111-1111-4111-8111-111111111111",
|
||||
mimeType: "image/png",
|
||||
width: 800,
|
||||
height: 600,
|
||||
},
|
||||
context: {
|
||||
agentId: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: BROWSER_WORKSPACE_ID,
|
||||
browserId: "11111111-1111-4111-8111-111111111111",
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(response.structuredContent)).not.toContain("iVBORw0KGgo=");
|
||||
expect(JSON.stringify(response.structuredContent)).not.toContain("dataBase64");
|
||||
} finally {
|
||||
await client.close();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps browser tools registered when browser tools are disabled", async () => {
|
||||
const { agentManager, agentStorage, spies } = createTestDeps();
|
||||
spies.agentManager.getAgent.mockReturnValue({
|
||||
id: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: BROWSER_WORKSPACE_ID,
|
||||
});
|
||||
spies.agentManager.getAgent.mockReturnValue({ id: "agent-1", cwd: REPO_CWD });
|
||||
const execute = vi.fn().mockResolvedValue({
|
||||
requestId: "req-browser-disabled",
|
||||
ok: false,
|
||||
@@ -834,12 +605,12 @@ describe("browser MCP tools", () => {
|
||||
|
||||
const response = await tool.handler({});
|
||||
|
||||
expect(lookupTool(server, "browser_snapshot")).not.toBeUndefined();
|
||||
expect(lookupTool(server, "browser_page_info")).not.toBeUndefined();
|
||||
expect(execute).toHaveBeenCalledWith({
|
||||
agentId: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: BROWSER_WORKSPACE_ID,
|
||||
command: { command: "list_tabs", args: {} },
|
||||
workspaceId: REPO_CWD,
|
||||
command: { command: "list_tabs", args: { workspaceId: REPO_CWD } },
|
||||
});
|
||||
expect(response.structuredContent).toEqual({
|
||||
ok: false,
|
||||
@@ -848,54 +619,11 @@ describe("browser MCP tools", () => {
|
||||
message: "Browser tools are disabled.",
|
||||
retryable: false,
|
||||
},
|
||||
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: BROWSER_WORKSPACE_ID },
|
||||
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD },
|
||||
});
|
||||
});
|
||||
|
||||
it("wires browser tools through the browser tools broker", async () => {
|
||||
const { agentManager, agentStorage, spies } = createTestDeps();
|
||||
spies.agentManager.getAgent.mockReturnValue({
|
||||
id: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: BROWSER_WORKSPACE_ID,
|
||||
});
|
||||
const execute = vi.fn().mockResolvedValue({
|
||||
requestId: "req-browser-tabs",
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
});
|
||||
const server = await createAgentMcpServer({
|
||||
agentManager,
|
||||
agentStorage,
|
||||
providerSnapshotManager: createOpenCodeManager().manager,
|
||||
browserToolsBroker: { execute } as never,
|
||||
callerAgentId: "agent-1",
|
||||
logger,
|
||||
});
|
||||
const tool = registeredTool(server, "browser_list_tabs");
|
||||
|
||||
const response = await tool.handler({});
|
||||
|
||||
expect(execute).toHaveBeenCalledWith({
|
||||
agentId: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: BROWSER_WORKSPACE_ID,
|
||||
command: { command: "list_tabs", args: {} },
|
||||
});
|
||||
expect(response.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: "No Paseo browser tabs are open. Call browser_new_tab to create one.",
|
||||
},
|
||||
]);
|
||||
expect(response.structuredContent).toEqual({
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: BROWSER_WORKSPACE_ID },
|
||||
});
|
||||
});
|
||||
|
||||
it("tells browser callers without a workspace how to proceed before broker execution", async () => {
|
||||
const { agentManager, agentStorage, spies } = createTestDeps();
|
||||
spies.agentManager.getAgent.mockReturnValue({ id: "agent-1", cwd: REPO_CWD });
|
||||
const execute = vi.fn().mockResolvedValue({
|
||||
@@ -915,22 +643,22 @@ describe("browser MCP tools", () => {
|
||||
|
||||
const response = await tool.handler({});
|
||||
|
||||
expect(execute).not.toHaveBeenCalled();
|
||||
expect(execute).toHaveBeenCalledWith({
|
||||
agentId: "agent-1",
|
||||
cwd: REPO_CWD,
|
||||
workspaceId: REPO_CWD,
|
||||
command: { command: "list_tabs", args: { workspaceId: REPO_CWD } },
|
||||
});
|
||||
expect(response.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: "This browser tool needs a workspace. Start the agent from a Paseo workspace before calling browser_new_tab or browser_list_tabs.",
|
||||
text: "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab.",
|
||||
},
|
||||
]);
|
||||
expect(response.structuredContent).toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_denied",
|
||||
message:
|
||||
"This browser tool needs a workspace. Start the agent from a Paseo workspace before calling browser_new_tab or browser_list_tabs.",
|
||||
retryable: false,
|
||||
},
|
||||
context: { agentId: "agent-1", cwd: REPO_CWD },
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
ServerNotification,
|
||||
ServerRequest,
|
||||
} from "@modelcontextprotocol/sdk/types.js";
|
||||
import { z } from "zod";
|
||||
|
||||
import { createPaseoToolCatalog, type PaseoToolHostDependencies } from "./tools/paseo-tools.js";
|
||||
import type { PaseoToolResult } from "./tools/types.js";
|
||||
@@ -13,6 +14,39 @@ export type AgentMcpServerOptions = PaseoToolHostDependencies;
|
||||
|
||||
type McpToolContext = RequestHandlerExtra<ServerRequest, ServerNotification>;
|
||||
|
||||
function isZodSchema(value: unknown): value is z.ZodType {
|
||||
return (
|
||||
typeof value === "object" &&
|
||||
value !== null &&
|
||||
typeof (value as { safeParseAsync?: unknown }).safeParseAsync === "function"
|
||||
);
|
||||
}
|
||||
|
||||
function relaxMcpOutputSchema(outputSchema: unknown): unknown {
|
||||
if (!outputSchema) {
|
||||
return outputSchema;
|
||||
}
|
||||
|
||||
if (isZodSchema(outputSchema)) {
|
||||
return outputSchema instanceof z.ZodObject ? outputSchema.passthrough() : outputSchema;
|
||||
}
|
||||
|
||||
return z.object(outputSchema as z.ZodRawShape).passthrough();
|
||||
}
|
||||
|
||||
function relaxMcpToolOutputSchema<TConfig extends { outputSchema?: unknown }>(
|
||||
config: TConfig,
|
||||
): TConfig {
|
||||
if (config.outputSchema === undefined) {
|
||||
return config;
|
||||
}
|
||||
|
||||
return {
|
||||
...config,
|
||||
outputSchema: relaxMcpOutputSchema(config.outputSchema),
|
||||
} as TConfig;
|
||||
}
|
||||
|
||||
function formatStructuredContentForModel(structuredContent: unknown): string {
|
||||
if (
|
||||
!structuredContent ||
|
||||
@@ -81,12 +115,12 @@ export async function createAgentMcpServer(options: AgentMcpServerOptions): Prom
|
||||
for (const tool of catalog.tools.values()) {
|
||||
server.registerTool(
|
||||
tool.name,
|
||||
{
|
||||
relaxMcpToolOutputSchema({
|
||||
title: tool.title,
|
||||
description: tool.description,
|
||||
inputSchema: tool.inputSchema,
|
||||
outputSchema: tool.outputSchema,
|
||||
},
|
||||
}),
|
||||
async (args: unknown, context?: McpToolContext) =>
|
||||
toMcpToolResult(await catalog.executeTool(tool.name, args, { signal: context?.signal })),
|
||||
);
|
||||
|
||||
@@ -2359,126 +2359,6 @@ describe("Codex app-server provider", () => {
|
||||
rmSync(source, { force: true });
|
||||
});
|
||||
|
||||
test("mcpToolCall image content emits a completed tool call plus assistant markdown image", async () => {
|
||||
const appServer = createFakeCodexAppServer();
|
||||
const session = new CodexAppServerAgentSession(
|
||||
createConfig({ cwd: "/workspace/project" }),
|
||||
null,
|
||||
createTestLogger(),
|
||||
async () => appServer.child,
|
||||
);
|
||||
const events: AgentStreamEvent[] = [];
|
||||
const timelineEvents: Array<Extract<AgentStreamEvent, { type: "timeline" }>> = [];
|
||||
const timelineItemsReceived = new Promise<void>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
unsubscribe();
|
||||
reject(new Error("Timed out waiting for MCP image timeline events"));
|
||||
}, 1000);
|
||||
const unsubscribe = session.subscribe((event) => {
|
||||
events.push(event);
|
||||
if (event.type !== "timeline") {
|
||||
return;
|
||||
}
|
||||
timelineEvents.push(event);
|
||||
if (timelineEvents.length === 2) {
|
||||
clearTimeout(timeout);
|
||||
unsubscribe();
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
const { turnId } = await session.startTurn("capture a browser screenshot");
|
||||
appServer.child.stdout.write(
|
||||
`${JSON.stringify({
|
||||
method: "item/completed",
|
||||
params: {
|
||||
item: {
|
||||
id: "mcp-browser-screenshot",
|
||||
type: "mcpToolCall",
|
||||
status: "completed",
|
||||
server: "paseo",
|
||||
tool: "browser_screenshot",
|
||||
arguments: { browserId: "11111111-1111-4111-8111-111111111111" },
|
||||
result: {
|
||||
content: [
|
||||
{ type: "text", text: "Captured browser screenshot (1x1)." },
|
||||
{ type: "image", data: ONE_BY_ONE_PNG_BASE64, mimeType: "image/png" },
|
||||
],
|
||||
structuredContent: {
|
||||
ok: true,
|
||||
result: {
|
||||
command: "screenshot",
|
||||
browserId: "11111111-1111-4111-8111-111111111111",
|
||||
mimeType: "image/png",
|
||||
width: 1,
|
||||
height: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})}\n`,
|
||||
);
|
||||
|
||||
await timelineItemsReceived;
|
||||
|
||||
expect(timelineEvents).toEqual([
|
||||
{
|
||||
type: "timeline",
|
||||
provider: "codex",
|
||||
turnId,
|
||||
item: {
|
||||
type: "tool_call",
|
||||
callId: "mcp-browser-screenshot",
|
||||
name: "paseo.browser_screenshot",
|
||||
status: "completed",
|
||||
error: null,
|
||||
detail: {
|
||||
type: "unknown",
|
||||
input: { browserId: "11111111-1111-4111-8111-111111111111" },
|
||||
output: {
|
||||
content: [
|
||||
{ type: "text", text: "Captured browser screenshot (1x1)." },
|
||||
{ type: "text", text: "[image]" },
|
||||
],
|
||||
structuredContent: {
|
||||
ok: true,
|
||||
result: {
|
||||
command: "screenshot",
|
||||
browserId: "11111111-1111-4111-8111-111111111111",
|
||||
mimeType: "image/png",
|
||||
width: 1,
|
||||
height: 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "timeline",
|
||||
provider: "codex",
|
||||
turnId,
|
||||
item: expect.objectContaining({ type: "assistant_message" }),
|
||||
},
|
||||
]);
|
||||
const imageEvent = timelineEvents[1];
|
||||
if (imageEvent.item.type !== "assistant_message") {
|
||||
throw new Error("Expected assistant image timeline event");
|
||||
}
|
||||
expect(JSON.stringify(events)).not.toContain(ONE_BY_ONE_PNG_BASE64);
|
||||
const source = markdownImageSource(imageEvent.item.text);
|
||||
expect(source).toMatch(/paseo-attachments[\\/].+\.png$/);
|
||||
expect(existsSync(source)).toBe(true);
|
||||
rmSync(source, { force: true });
|
||||
appServer.assertNoErrors();
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("ignores incomplete imageGeneration thread items without failing the turn", () => {
|
||||
const session = createSession();
|
||||
const events: AgentStreamEvent[] = [];
|
||||
|
||||
@@ -49,7 +49,6 @@ import { curateAgentActivity } from "../activity-curator.js";
|
||||
import {
|
||||
mapCodexToolCallEnvelope,
|
||||
mapCodexToolCallFromThreadItem,
|
||||
splitCodexMcpToolResultImages,
|
||||
} from "./codex/tool-call-mapper.js";
|
||||
import {
|
||||
checkProviderLaunchAvailable,
|
||||
@@ -1697,39 +1696,6 @@ export function threadItemToTimeline(
|
||||
}
|
||||
}
|
||||
|
||||
function mcpToolResultImagesToTimeline(item: unknown): AgentTimelineItem[] {
|
||||
const itemRecord = toObjectRecord(item);
|
||||
if (!itemRecord) {
|
||||
return [];
|
||||
}
|
||||
const normalizedType = normalizeCodexThreadItemType(
|
||||
typeof itemRecord.type === "string" ? itemRecord.type : undefined,
|
||||
);
|
||||
if (normalizedType !== "mcpToolCall") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const { images } = splitCodexMcpToolResultImages(itemRecord.result);
|
||||
return images
|
||||
.map((image) =>
|
||||
renderProviderImageOutputAsAssistantMarkdown(image, {
|
||||
materialize: materializeProviderImage,
|
||||
}),
|
||||
)
|
||||
.filter((timelineItem): timelineItem is AgentTimelineItem => timelineItem !== null);
|
||||
}
|
||||
|
||||
function threadItemToTimelineEntries(
|
||||
item: unknown,
|
||||
options?: { includeUserMessage?: boolean; cwd?: string | null },
|
||||
): AgentTimelineItem[] {
|
||||
const timelineItem = threadItemToTimeline(item, options);
|
||||
if (!timelineItem) {
|
||||
return [];
|
||||
}
|
||||
return [timelineItem, ...mcpToolResultImagesToTimeline(item)];
|
||||
}
|
||||
|
||||
const CodexThreadReadResponseSchema = z
|
||||
.object({
|
||||
thread: z
|
||||
@@ -1769,7 +1735,8 @@ async function loadCodexThreadHistoryTimeline(params: {
|
||||
const timeline: PersistedTimelineEntry[] = [];
|
||||
for (const turn of response.thread.turns) {
|
||||
for (const item of turn.items) {
|
||||
for (const timelineItem of threadItemToTimelineEntries(item, { cwd: params.cwd })) {
|
||||
const timelineItem = threadItemToTimeline(item, { cwd: params.cwd });
|
||||
if (timelineItem) {
|
||||
const timestamp =
|
||||
readCodexHistoryTimestamp(item) ?? readCodexTurnHistoryTimestamp(turn, timelineItem);
|
||||
timeline.push({
|
||||
@@ -4881,15 +4848,10 @@ export class CodexAppServerAgentSession implements AgentSession {
|
||||
}
|
||||
this.warnOnIncompleteEditToolCall(timelineItem, "item_completed", parsed.item);
|
||||
}
|
||||
const imageItems = mcpToolResultImagesToTimeline(parsed.item);
|
||||
this.emitEvent({ type: "timeline", provider: CODEX_PROVIDER, item: timelineItem });
|
||||
if (timelineItem.type === "assistant_message") {
|
||||
this.pendingAssistantMessageBoundary = true;
|
||||
}
|
||||
for (const imageItem of imageItems) {
|
||||
this.emitEvent({ type: "timeline", provider: CODEX_PROVIDER, item: imageItem });
|
||||
this.pendingAssistantMessageBoundary = true;
|
||||
}
|
||||
if (itemId) {
|
||||
this.emittedItemCompletedIds.add(itemId);
|
||||
this.emittedItemStartedIds.delete(itemId);
|
||||
|
||||
@@ -703,44 +703,6 @@ describe("codex tool-call mapper", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("replaces mcp image result blocks with placeholder text in tool output", () => {
|
||||
const item = expectMapped(
|
||||
mapCodexToolCallFromThreadItem({
|
||||
type: "mcpToolCall",
|
||||
id: "codex-browser-screenshot",
|
||||
status: "completed",
|
||||
server: "paseo",
|
||||
tool: "browser_screenshot",
|
||||
arguments: { browserId: "11111111-1111-4111-8111-111111111111" },
|
||||
result: {
|
||||
content: [
|
||||
{ type: "text", text: "Captured browser screenshot (1x1)." },
|
||||
{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" },
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(item).toEqual({
|
||||
type: "tool_call",
|
||||
callId: "codex-browser-screenshot",
|
||||
name: "paseo.browser_screenshot",
|
||||
status: "completed",
|
||||
error: null,
|
||||
detail: {
|
||||
type: "unknown",
|
||||
input: { browserId: "11111111-1111-4111-8111-111111111111" },
|
||||
output: {
|
||||
content: [
|
||||
{ type: "text", text: "Captured browser screenshot (1x1)." },
|
||||
{ type: "text", text: "[image]" },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(item)).not.toContain("iVBORw0KGgo=");
|
||||
});
|
||||
|
||||
it("normalizes codex paseo_voice.speak mcp calls and extracts spoken text", () => {
|
||||
const item = expectMapped(
|
||||
mapCodexToolCallFromThreadItem({
|
||||
|
||||
@@ -76,16 +76,6 @@ interface CodexResolvedToolCall {
|
||||
cwd: string | null;
|
||||
}
|
||||
|
||||
export interface CodexMcpToolResultImage {
|
||||
data: string;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
interface CodexMcpToolResultImagesSplit {
|
||||
images: CodexMcpToolResultImage[];
|
||||
output: unknown;
|
||||
}
|
||||
|
||||
function toToolCallTimelineItem(envelope: CodexResolvedToolCall): ToolCallTimelineItem {
|
||||
const name = envelope.toolKind === "speak" ? ("speak" as const) : envelope.name;
|
||||
const parsedDetail = deriveCodexToolDetail({
|
||||
@@ -606,50 +596,6 @@ function buildMcpToolName(server: string | undefined, tool: string): string {
|
||||
return trimmedTool;
|
||||
}
|
||||
|
||||
function readMcpImageContent(block: unknown): CodexMcpToolResultImage | null {
|
||||
if (!isRecord(block)) {
|
||||
return null;
|
||||
}
|
||||
if (block.type !== "image") {
|
||||
return null;
|
||||
}
|
||||
if (typeof block.data !== "string" || typeof block.mimeType !== "string") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
data: block.data,
|
||||
mimeType: block.mimeType,
|
||||
};
|
||||
}
|
||||
|
||||
export function splitCodexMcpToolResultImages(result: unknown): CodexMcpToolResultImagesSplit {
|
||||
if (!isRecord(result) || !Array.isArray(result.content)) {
|
||||
return { images: [], output: result };
|
||||
}
|
||||
|
||||
const images: CodexMcpToolResultImage[] = [];
|
||||
const content = result.content.map((block) => {
|
||||
const image = readMcpImageContent(block);
|
||||
if (!image) {
|
||||
return block;
|
||||
}
|
||||
images.push(image);
|
||||
return { type: "text", text: "[image]" };
|
||||
});
|
||||
|
||||
if (images.length === 0) {
|
||||
return { images, output: result };
|
||||
}
|
||||
|
||||
return {
|
||||
images,
|
||||
output: {
|
||||
...result,
|
||||
content,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function toNullableObject(value: Record<string, unknown>): Record<string, unknown> | null {
|
||||
return Object.keys(value).length > 0 ? value : null;
|
||||
}
|
||||
@@ -909,7 +855,7 @@ function mapMcpToolCallItem(
|
||||
}
|
||||
const name = buildMcpToolName(item.server, tool);
|
||||
const input = item.arguments ?? null;
|
||||
const { output } = splitCodexMcpToolResultImages(item.result ?? null);
|
||||
const output = item.result ?? null;
|
||||
const error = item.error ?? null;
|
||||
const status = normalizeToolCallStatus(item.status, error, output);
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ export interface PaseoToolConfig {
|
||||
title?: string;
|
||||
description?: string;
|
||||
inputSchema?: z.ZodRawShape | z.ZodType;
|
||||
outputSchema?: z.ZodRawShape;
|
||||
outputSchema?: z.ZodRawShape | z.ZodType;
|
||||
}
|
||||
|
||||
export interface PaseoToolDefinition extends PaseoToolConfig {
|
||||
|
||||
@@ -7,8 +7,6 @@ import type {
|
||||
import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js";
|
||||
import { StaticBrowserToolsPolicy } from "./policy.js";
|
||||
|
||||
const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
class FakeDesktopClient implements BrowserToolsDesktopClient {
|
||||
public readonly receivedRequests: BrowserAutomationExecuteRequest[] = [];
|
||||
|
||||
@@ -45,8 +43,8 @@ function createBroker(options: { enabled: boolean; timeoutMs?: number }): Browse
|
||||
});
|
||||
}
|
||||
|
||||
function snapshotCommand(): BrowserAutomationCommand {
|
||||
return { command: "snapshot", args: { browserId: BROWSER_ID } };
|
||||
function pageInfoCommand(): BrowserAutomationCommand {
|
||||
return { command: "page_info", args: { browserId: "browser-1" } };
|
||||
}
|
||||
|
||||
describe("BrowserToolsBroker", () => {
|
||||
@@ -57,7 +55,7 @@ describe("BrowserToolsBroker", () => {
|
||||
test("disabled returns browser_disabled", async () => {
|
||||
const broker = createBroker({ enabled: false });
|
||||
|
||||
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
|
||||
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
@@ -71,7 +69,7 @@ describe("BrowserToolsBroker", () => {
|
||||
test("no capable desktop returns browser_no_desktop", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
|
||||
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
|
||||
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
@@ -113,7 +111,7 @@ describe("BrowserToolsBroker", () => {
|
||||
broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({
|
||||
command: { command: "list_tabs", args: {} },
|
||||
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
|
||||
@@ -122,7 +120,7 @@ describe("BrowserToolsBroker", () => {
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: { command: "list_tabs", args: {} },
|
||||
command: { command: "list_tabs", args: { workspaceId: "workspace-1" } },
|
||||
},
|
||||
]);
|
||||
expect(broker.getPendingRequestCount()).toBe(1);
|
||||
@@ -135,7 +133,7 @@ describe("BrowserToolsBroker", () => {
|
||||
command: "list_tabs",
|
||||
tabs: [
|
||||
{
|
||||
browserId: BROWSER_ID,
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
@@ -152,7 +150,7 @@ describe("BrowserToolsBroker", () => {
|
||||
command: "list_tabs",
|
||||
tabs: [
|
||||
{
|
||||
browserId: BROWSER_ID,
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
@@ -171,7 +169,7 @@ describe("BrowserToolsBroker", () => {
|
||||
broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
|
||||
@@ -180,7 +178,7 @@ describe("BrowserToolsBroker", () => {
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
command: { command: "snapshot", args: { workspaceId: "workspace-1" } },
|
||||
},
|
||||
]);
|
||||
|
||||
@@ -189,7 +187,7 @@ describe("BrowserToolsBroker", () => {
|
||||
ok: true,
|
||||
result: {
|
||||
command: "snapshot",
|
||||
browserId: BROWSER_ID,
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
@@ -202,7 +200,7 @@ describe("BrowserToolsBroker", () => {
|
||||
ok: true,
|
||||
result: {
|
||||
command: "snapshot",
|
||||
browserId: BROWSER_ID,
|
||||
browserId: "browser-1",
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
@@ -216,7 +214,7 @@ describe("BrowserToolsBroker", () => {
|
||||
const broker = createBroker({ enabled: true, timeoutMs: 50 });
|
||||
broker.registerClient(new FakeDesktopClient("desktop-1"));
|
||||
|
||||
const resultPromise = broker.execute({ command: snapshotCommand() });
|
||||
const resultPromise = broker.execute({ command: pageInfoCommand() });
|
||||
expect(broker.getPendingRequestCount()).toBe(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
@@ -238,7 +236,7 @@ describe("BrowserToolsBroker", () => {
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
const unregister = broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({ command: snapshotCommand() });
|
||||
const resultPromise = broker.execute({ command: pageInfoCommand() });
|
||||
expect(broker.getPendingRequestCount()).toBe(1);
|
||||
|
||||
unregister();
|
||||
@@ -259,7 +257,7 @@ describe("BrowserToolsBroker", () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
broker.registerClient(new FailingDesktopClient());
|
||||
|
||||
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
|
||||
await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
@@ -276,14 +274,14 @@ describe("BrowserToolsBroker", () => {
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({ command: snapshotCommand() });
|
||||
const resultPromise = broker.execute({ command: pageInfoCommand() });
|
||||
|
||||
client.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: `Browser tab ${BROWSER_ID} was not found.`,
|
||||
message: "Browser tab browser-1 was not found.",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
@@ -293,7 +291,7 @@ describe("BrowserToolsBroker", () => {
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: `Browser tab ${BROWSER_ID} was not found.`,
|
||||
message: "Browser tab browser-1 was not found.",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
@@ -305,7 +303,7 @@ describe("BrowserToolsBroker", () => {
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({ command: snapshotCommand() });
|
||||
const resultPromise = broker.execute({ command: pageInfoCommand() });
|
||||
expect(broker.getPendingRequestCount()).toBe(1);
|
||||
|
||||
expect(
|
||||
|
||||
@@ -19,6 +19,7 @@ export interface BrowserToolsExecuteInput {
|
||||
agentId?: string;
|
||||
cwd?: string;
|
||||
workspaceId?: string;
|
||||
browserId?: string;
|
||||
requestId?: string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
@@ -113,6 +114,7 @@ export class BrowserToolsBroker {
|
||||
...(input.agentId ? { agentId: input.agentId } : {}),
|
||||
...(input.cwd ? { cwd: input.cwd } : {}),
|
||||
...(input.workspaceId ? { workspaceId: input.workspaceId } : {}),
|
||||
...(input.browserId ? { browserId: input.browserId } : {}),
|
||||
command: input.command,
|
||||
});
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -587,8 +587,8 @@ describe("PersistedConfigSchema logging config", () => {
|
||||
expect(parsed.log?.format).toBe("json");
|
||||
});
|
||||
|
||||
test("rejects unknown logging config fields", () => {
|
||||
const result = PersistedConfigSchema.safeParse({
|
||||
test("strips unknown logging config fields", () => {
|
||||
const parsed = PersistedConfigSchema.parse({
|
||||
log: {
|
||||
console: {
|
||||
level: "info",
|
||||
@@ -597,7 +597,7 @@ describe("PersistedConfigSchema logging config", () => {
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(parsed.log?.console).toEqual({ level: "info" });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -669,6 +669,42 @@ describe("loadPersistedConfig", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("loads a config with unrecognized persisted fields", () => {
|
||||
const home = createTempHome();
|
||||
const configPath = path.join(home, "config.json");
|
||||
try {
|
||||
writeFileSync(
|
||||
configPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
daemon: {
|
||||
listen: "127.0.0.1:6767",
|
||||
futureDaemonSetting: { enabled: true },
|
||||
relay: {
|
||||
enabled: true,
|
||||
futureRelaySetting: "enabled",
|
||||
},
|
||||
},
|
||||
futureRootSetting: true,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
|
||||
const config = loadPersistedConfig(home);
|
||||
|
||||
expect(config.daemon?.listen).toBe("127.0.0.1:6767");
|
||||
expect(config.daemon?.relay?.enabled).toBe(true);
|
||||
expect((config.daemon as Record<string, unknown>)?.futureDaemonSetting).toBeUndefined();
|
||||
expect((config.daemon?.relay as Record<string, unknown>)?.futureRelaySetting).toBeUndefined();
|
||||
expect((config as Record<string, unknown>).futureRootSetting).toBeUndefined();
|
||||
} finally {
|
||||
rmSync(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("loads a config that still uses the removed providers.openai.voice block", () => {
|
||||
const home = createTempHome();
|
||||
const configPath = path.join(home, "config.json");
|
||||
|
||||
@@ -25,7 +25,7 @@ const LogConfigSchema = z
|
||||
level: LogLevelSchema.optional(),
|
||||
format: LogFormatSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
|
||||
file: z
|
||||
@@ -37,20 +37,20 @@ const LogConfigSchema = z
|
||||
maxSize: z.string().min(1).optional(),
|
||||
maxFiles: z.number().int().positive().optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const OpenAiSpeechEndpointSchema = z
|
||||
.object({
|
||||
apiKey: z.string().trim().min(1).optional(),
|
||||
baseUrl: z.string().trim().min(1).optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const OpenAiProviderSchema = z
|
||||
.object({
|
||||
@@ -59,26 +59,26 @@ const OpenAiProviderSchema = z
|
||||
stt: OpenAiSpeechEndpointSchema.optional(),
|
||||
tts: OpenAiSpeechEndpointSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const LocalSpeechProviderSchema = z
|
||||
.object({
|
||||
modelsDir: z.string().min(1).optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const ProvidersSchema = z
|
||||
.object({
|
||||
openai: OpenAiProviderSchema.optional(),
|
||||
local: LocalSpeechProviderSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const WorktreesConfigSchema = z
|
||||
.object({
|
||||
root: z.string().min(1).optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const BcryptHashSchema = z.string().regex(/^\$2[aby]\$\d{2}\$[./A-Za-z0-9]{53}$/, {
|
||||
message: "Expected a bcrypt hash",
|
||||
@@ -88,7 +88,7 @@ const DaemonAuthSchema = z
|
||||
.object({
|
||||
password: BcryptHashSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const SpeechProviderIdSchema = z
|
||||
.string()
|
||||
@@ -106,10 +106,10 @@ const FeatureDictationSchema = z
|
||||
language: z.string().trim().min(1).optional(),
|
||||
confidenceThreshold: z.number().optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const FeatureVoiceModeSchema = z
|
||||
.object({
|
||||
@@ -119,7 +119,7 @@ const FeatureVoiceModeSchema = z
|
||||
provider: z.string().optional(),
|
||||
model: z.string().min(1).optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
stt: z
|
||||
.object({
|
||||
@@ -127,13 +127,13 @@ const FeatureVoiceModeSchema = z
|
||||
model: z.string().min(1).optional(),
|
||||
language: z.string().trim().min(1).optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
turnDetection: z
|
||||
.object({
|
||||
provider: SpeechProviderIdSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
tts: z
|
||||
.object({
|
||||
@@ -143,17 +143,17 @@ const FeatureVoiceModeSchema = z
|
||||
speakerId: z.number().int().optional(),
|
||||
speed: z.number().optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const FeatureWebUiSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
distDir: z.string().min(1).optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const StructuredGenerationProviderConfigSchema = z
|
||||
.object({
|
||||
@@ -161,13 +161,13 @@ const StructuredGenerationProviderConfigSchema = z
|
||||
model: z.string().min(1).optional(),
|
||||
thinkingOptionId: z.string().min(1).optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const AgentMetadataGenerationSchema = z
|
||||
.object({
|
||||
providers: z.array(StructuredGenerationProviderConfigSchema).optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
const BUILTIN_PROVIDER_IDS = ["claude", "codex", "copilot", "opencode", "pi", "omp"] as const;
|
||||
|
||||
@@ -254,7 +254,7 @@ export const PersistedConfigSchema = z
|
||||
.object({
|
||||
allowedOrigins: z.array(z.string()).optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
relay: z
|
||||
.object({
|
||||
@@ -264,7 +264,7 @@ export const PersistedConfigSchema = z
|
||||
useTls: z.boolean().optional(),
|
||||
publicUseTls: z.boolean().optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
serviceProxy: z
|
||||
.object({
|
||||
@@ -275,11 +275,11 @@ export const PersistedConfigSchema = z
|
||||
listen: z.string().optional(),
|
||||
publicBaseUrl: z.url().optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
auth: DaemonAuthSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.transform(({ allowedHosts, ...daemon }) => {
|
||||
const hostnames = daemon.hostnames ?? allowedHosts;
|
||||
return hostnames === undefined ? daemon : { ...daemon, hostnames };
|
||||
@@ -290,7 +290,7 @@ export const PersistedConfigSchema = z
|
||||
.object({
|
||||
baseUrl: z.string().optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
|
||||
providers: ProvidersSchema.optional(),
|
||||
@@ -300,7 +300,7 @@ export const PersistedConfigSchema = z
|
||||
providers: z.preprocess(normalizeAgentProviders, ProviderOverridesSchema).optional(),
|
||||
metadataGeneration: AgentMetadataGenerationSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
features: z
|
||||
.object({
|
||||
@@ -308,12 +308,12 @@ export const PersistedConfigSchema = z
|
||||
voiceMode: FeatureVoiceModeSchema.optional(),
|
||||
webUi: FeatureWebUiSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.strip()
|
||||
.optional(),
|
||||
|
||||
log: LogConfigSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
.strip();
|
||||
|
||||
type PersistedConfigSchemaOutput = z.infer<typeof PersistedConfigSchema>;
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import type {
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import type pino from "pino";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AgentManager } from "./agent/agent-manager.js";
|
||||
import type { AgentStorage } from "./agent/agent-storage.js";
|
||||
@@ -51,7 +51,6 @@ interface QueuedBrowserRequests {
|
||||
}
|
||||
|
||||
const harnesses: BrowserToolsDaemonHarness[] = [];
|
||||
const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(harnesses.splice(0).map((harness) => harness.stop()));
|
||||
@@ -132,7 +131,7 @@ describe("WebSocketServer browser tools wiring", () => {
|
||||
});
|
||||
|
||||
const resultPromise = harness.broker.execute({
|
||||
command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } },
|
||||
command: { command: "click", args: { ref: "@e1" } },
|
||||
});
|
||||
const request = await resumedDesktop.nextBrowserRequest();
|
||||
resumedDesktop.respondToBrowserRequest({
|
||||
@@ -140,13 +139,13 @@ describe("WebSocketServer browser tools wiring", () => {
|
||||
payload: {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
|
||||
result: { command: "click", browserId: "browser-1", ref: "@e1" },
|
||||
},
|
||||
});
|
||||
|
||||
await expect(resultPromise).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
|
||||
result: { command: "click", browserId: "browser-1", ref: "@e1" },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -228,17 +227,17 @@ function createVoiceAssistantWebSocketServer(params: {
|
||||
}): VoiceAssistantWebSocketServer {
|
||||
const { httpServer, broker } = params;
|
||||
const agentManager = {
|
||||
setAgentAttentionCallback() {},
|
||||
subscribe: () => () => {},
|
||||
getMetricsSnapshot: () => ({
|
||||
setAgentAttentionCallback: vi.fn(),
|
||||
subscribe: vi.fn(() => () => {}),
|
||||
getMetricsSnapshot: vi.fn(() => ({
|
||||
total: 0,
|
||||
byLifecycle: {},
|
||||
withActiveForegroundTurn: 0,
|
||||
timelineStats: { totalItems: 0, maxItemsPerAgent: 0 },
|
||||
}),
|
||||
})),
|
||||
};
|
||||
const daemonConfigStore = {
|
||||
onChange: () => () => {},
|
||||
onChange: vi.fn(() => () => {}),
|
||||
};
|
||||
|
||||
return new VoiceAssistantWebSocketServer(
|
||||
@@ -264,15 +263,15 @@ function createVoiceAssistantWebSocketServer(params: {
|
||||
createStub<LoopService>({}),
|
||||
createStub<ScheduleService>({}),
|
||||
createStub<CheckoutDiffManager>({
|
||||
subscribe: () => {},
|
||||
scheduleRefreshForCwd: () => {},
|
||||
getMetrics: () => ({
|
||||
subscribe: vi.fn(),
|
||||
scheduleRefreshForCwd: vi.fn(),
|
||||
getMetrics: vi.fn(() => ({
|
||||
checkoutDiffTargetCount: 0,
|
||||
checkoutDiffSubscriptionCount: 0,
|
||||
checkoutDiffWatcherCount: 0,
|
||||
checkoutDiffFallbackRefreshTargetCount: 0,
|
||||
}),
|
||||
dispose: () => {},
|
||||
})),
|
||||
dispose: vi.fn(),
|
||||
}),
|
||||
undefined,
|
||||
undefined,
|
||||
@@ -348,12 +347,12 @@ function createBrowserRequestQueue(): QueuedBrowserRequests {
|
||||
|
||||
function createLogger() {
|
||||
const logger = {
|
||||
child: () => logger,
|
||||
trace() {},
|
||||
debug() {},
|
||||
info() {},
|
||||
warn() {},
|
||||
error() {},
|
||||
child: vi.fn(() => logger),
|
||||
trace: vi.fn(),
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
return logger;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/website",
|
||||
"version": "0.1.104-beta.1",
|
||||
"version": "0.1.103",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
Reference in New Issue
Block a user