diff --git a/docs/architecture.md b/docs/architecture.md index ab95f6849..fed447e4b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -138,7 +138,7 @@ Electron wrapper for macOS, Linux, and Windows. > **Window-state v1 limitation:** only the _first_ window of a session restores and persists saved geometry (size/position/maximized). Windows opened via ⌘⇧N / second-instance / "Open in new window" open at the default size, OS-cascaded, and do not persist — this avoids every window stacking on the same restored bounds and fighting over the single window-state store. Lifting this needs per-window state keys. > -> **In-app browser panes are not yet per-window.** The active-browser id (`features/browser-webviews.ts`) and the webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) are process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up. +> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus and agent automation targets are intentionally separate: the workspace-active browser follows the user's focused tab, while the agent-active browser is the default target for browser MCP commands. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up. ### `packages/website` — Marketing site diff --git a/docs/development.md b/docs/development.md index 55624c9e1..99a28dc04 100644 --- a/docs/development.md +++ b/docs/development.md @@ -72,6 +72,10 @@ Starting the service must not create, focus, reveal, or leave behind macOS Simul It launches its own Electron-flavored Expo server and passes that URL to Electron. Override the CDP port with `PASEO_ELECTRON_REMOTE_DEBUGGING_PORT` when `9223` is busy. +When running a dedicated Electron QA instance against a non-default Expo port, set +`EXPO_DEV_URL` explicitly. Desktop main defaults to `http://localhost:8081`, so +`PASEO_PORT=57928` alone starts Metro on 57928 but Electron still loads 8081. + ### React render profiling The app has a gated React render profiler in diff --git a/docs/opencode-global-event-baseline.md b/docs/opencode-global-event-baseline.md index 2239c01b4..8c439ce01 100644 --- a/docs/opencode-global-event-baseline.md +++ b/docs/opencode-global-event-baseline.md @@ -9,7 +9,7 @@ Replace the OpenCode provider's per-directory `/event` stream with OpenCode's `/ ## Environment - `opencode --version`: `1.14.46` -- `which opencode`: `/Users/moboudra/.asdf/installs/nodejs/22.20.0/bin/opencode` +- `which opencode`: `opencode` - `node --version`: `v22.20.0` - `npm --version`: `10.9.3` diff --git a/packages/app/src/app/_layout.tsx b/packages/app/src/app/_layout.tsx index ab3370ec1..b0557f9eb 100644 --- a/packages/app/src/app/_layout.tsx +++ b/packages/app/src/app/_layout.tsx @@ -87,6 +87,7 @@ import { polyfillCrypto } from "@/polyfills/crypto"; import { queryClient } from "@/query/query-client"; import { getHostRuntimeStore, + hasConfiguredLocalDaemonOverride, useHostRegistryLoaded, useHostMutations, useHostRuntimeClient, @@ -313,6 +314,9 @@ async function shouldStartBuiltInDaemon(): Promise { if (!shouldUseDesktopDaemon()) { return false; } + if (hasConfiguredLocalDaemonOverride()) { + return false; + } const settings = await loadDesktopSettings(); return settings.daemon.manageBuiltInDaemon; } diff --git a/packages/app/src/browser-automation/handler.test.ts b/packages/app/src/browser-automation/handler.test.ts new file mode 100644 index 000000000..dd6a634cc --- /dev/null +++ b/packages/app/src/browser-automation/handler.test.ts @@ -0,0 +1,375 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; +import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; +import { mountBrowserAutomationHandler } from "./handler"; +import type { DesktopHostBridge } from "@/desktop/host"; +import { useBrowserStore } from "@/stores/browser-store"; +import { + buildWorkspaceTabPersistenceKey, + useWorkspaceLayoutStore, +} from "@/stores/workspace-layout-store"; + +vi.mock("expo-router", () => ({ + router: { + navigate: vi.fn(), + }, +})); + +vi.mock("@react-native-async-storage/async-storage", () => ({ + default: { + getItem: vi.fn(async () => null), + setItem: vi.fn(async () => undefined), + removeItem: vi.fn(async () => undefined), + }, +})); + +type BrowserAutomationExecuteRequest = Extract< + SessionOutboundMessage, + { type: "browser.automation.execute.request" } +>; +type BrowserAutomationExecuteResponse = Extract< + SessionInboundMessage, + { type: "browser.automation.execute.response" } +>; + +class FakeDaemonClient { + public sentResponses: BrowserAutomationExecuteResponse[] = []; + private handler: ((request: BrowserAutomationExecuteRequest) => void) | null = null; + + public on( + type: "browser.automation.execute.request", + handler: (request: BrowserAutomationExecuteRequest) => void, + ): () => void { + expect(type).toBe("browser.automation.execute.request"); + this.handler = handler; + return () => { + if (this.handler === handler) { + this.handler = null; + } + }; + } + + public sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void { + this.sentResponses.push(response); + } + + public receive(nextRequest: BrowserAutomationExecuteRequest): void { + this.handler?.(nextRequest); + } +} + +function browserAutomationRequest(): BrowserAutomationExecuteRequest { + return { + type: "browser.automation.execute.request", + requestId: "req-1", + command: { command: "list_tabs", args: {} }, + }; +} + +function browserNewTabRequest(): BrowserAutomationExecuteRequest { + return { + type: "browser.automation.execute.request", + requestId: "req-new", + agentId: "agent-1", + workspaceId: "/repo", + command: { command: "new_tab", args: { workspaceId: "/repo", url: "https://example.com" } }, + }; +} + +function emptyListTabsPayload(requestId = "req-new:list_tabs") { + return { + requestId, + ok: true as const, + result: { + command: "list_tabs" as const, + tabs: [], + }, + }; +} + +function currentListTabsPayload(requestId = "req-new:list_tabs") { + return { + requestId, + ok: true as const, + result: { + command: "list_tabs" as const, + tabs: currentBrowserTabs(), + }, + }; +} + +function flushPromises(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +function waitForAsyncWork(): Promise { + return new Promise((resolve) => setTimeout(resolve, 20)); +} + +function currentBrowserTabs() { + return Object.values(useBrowserStore.getState().browsersById).map((browser) => ({ + browserId: browser.browserId, + workspaceId: "/repo", + url: browser.url, + title: browser.title, + isActive: true, + isLoading: false, + })); +} + +describe("mountBrowserAutomationHandler", () => { + beforeEach(() => { + useBrowserStore.setState({ browsersById: {} }); + useWorkspaceLayoutStore.setState({ layoutByWorkspace: {} }); + }); + + test("creates an unfocused workspace browser tab for browser_new_tab", async () => { + const client = new FakeDaemonClient(); + const setWorkspaceActiveBrowser = vi.fn(async () => undefined); + const setAgentActiveBrowser = vi.fn(async () => undefined); + const registerWorkspaceBrowser = vi.fn(async () => undefined); + const ensureResidentBrowserWebview = vi.fn(); + const executeAutomationCommand = vi.fn(async () => currentListTabsPayload()); + const workspaceKey = buildWorkspaceTabPersistenceKey({ + serverId: "server-1", + workspaceId: "/repo", + }); + if (!workspaceKey) throw new Error("expected workspace key"); + const focusedTabId = useWorkspaceLayoutStore + .getState() + .openTabFocused(workspaceKey, { kind: "draft", draftId: "human-draft" }); + if (!focusedTabId) throw new Error("expected focused tab"); + mountBrowserAutomationHandler({ + client, + serverId: "server-1", + getHost: () => + ({ + browser: { + executeAutomationCommand, + registerWorkspaceBrowser, + setWorkspaceActiveBrowser, + setAgentActiveBrowser, + }, + }) satisfies DesktopHostBridge, + ensureResidentBrowserWebview, + }); + + client.receive(browserNewTabRequest()); + await flushPromises(); + + const payload = client.sentResponses[0]?.payload; + expect(payload?.ok).toBe(true); + if (!payload?.ok) throw new Error("expected success"); + expect(payload.result.command).toBe("new_tab"); + if (payload.result.command !== "new_tab") throw new Error("expected new_tab result"); + expect(payload.result.url).toBe("https://example.com"); + expect(useWorkspaceLayoutStore.getState().getWorkspaceTabs(workspaceKey)).toContainEqual( + expect.objectContaining({ target: { kind: "browser", browserId: payload.result.browserId } }), + ); + const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey]; + expect(layout?.root.kind).toBe("pane"); + if (layout?.root.kind !== "pane") throw new Error("expected root pane"); + expect(layout.root.pane.focusedTabId).toBe(focusedTabId); + expect(registerWorkspaceBrowser).toHaveBeenCalledWith({ + browserId: payload.result.browserId, + workspaceId: "/repo", + }); + expect(setAgentActiveBrowser).toHaveBeenCalledWith({ + agentId: "agent-1", + browserId: payload.result.browserId, + }); + expect(setWorkspaceActiveBrowser).not.toHaveBeenCalled(); + expect(ensureResidentBrowserWebview).toHaveBeenCalledWith({ + browserId: payload.result.browserId, + url: "https://example.com", + }); + expect(executeAutomationCommand).toHaveBeenCalledTimes(1); + }); + + test("returns browser_timeout when the resident webview does not register", async () => { + const client = new FakeDaemonClient(); + const ensureResidentBrowserWebview = vi.fn(); + const executeAutomationCommand = vi.fn(async () => emptyListTabsPayload()); + mountBrowserAutomationHandler({ + client, + serverId: "server-1", + getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge, + ensureResidentBrowserWebview, + registrationWaitTimeoutMs: 1, + registrationPollIntervalMs: 1, + }); + + client.receive(browserNewTabRequest()); + await waitForAsyncWork(); + + expect(client.sentResponses[0]?.payload).toMatchObject({ + requestId: "req-new", + ok: false, + error: { + code: "browser_timeout", + retryable: true, + }, + }); + expect(ensureResidentBrowserWebview).toHaveBeenCalledWith( + expect.objectContaining({ url: "https://example.com" }), + ); + expect(client.sentResponses[0]?.payload).not.toMatchObject({ + ok: true, + result: { command: "new_tab" }, + }); + }); + + test("wraps browser_new_tab registration bridge errors in a response", async () => { + const client = new FakeDaemonClient(); + const executeAutomationCommand = vi.fn(async () => { + throw new Error("IPC registration check failed"); + }); + mountBrowserAutomationHandler({ + client, + serverId: "server-1", + getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge, + }); + + client.receive(browserNewTabRequest()); + await flushPromises(); + + expect(client.sentResponses[0]?.payload).toEqual({ + requestId: "req-new", + ok: false, + error: { + code: "browser_unknown_error", + message: "IPC registration check failed", + retryable: false, + }, + }); + }); + + test("sends a success response from the desktop bridge", async () => { + const client = new FakeDaemonClient(); + const executeAutomationCommand = vi.fn(async () => ({ + requestId: "req-1", + ok: true as const, + result: { command: "list_tabs" as const, tabs: [] }, + })); + + mountBrowserAutomationHandler({ + client, + getHost: () => ({ browser: { executeAutomationCommand } }) satisfies DesktopHostBridge, + }); + + client.receive(browserAutomationRequest()); + await flushPromises(); + + expect(executeAutomationCommand).toHaveBeenCalledWith(browserAutomationRequest()); + expect(client.sentResponses).toEqual([ + { + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }, + }, + ]); + }); + + test("missing bridge sends browser_unsupported", async () => { + const client = new FakeDaemonClient(); + mountBrowserAutomationHandler({ client, getHost: () => null }); + + client.receive(browserAutomationRequest()); + await flushPromises(); + + expect(client.sentResponses).toEqual([ + { + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: false, + error: { + code: "browser_unsupported", + message: "Desktop browser automation is not available in this app runtime.", + retryable: false, + }, + }, + }, + ]); + }); + + test("typed bridge errors become failure responses", async () => { + const client = new FakeDaemonClient(); + mountBrowserAutomationHandler({ + client, + getHost: () => ({ + browser: { + executeAutomationCommand: async () => { + throw { + code: "browser_tab_not_found", + message: "Browser tab browser-1 was not found.", + retryable: false, + }; + }, + }, + }), + }); + + client.receive(browserAutomationRequest()); + await flushPromises(); + + expect(client.sentResponses[0]?.payload).toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_tab_not_found", + message: "Browser tab browser-1 was not found.", + retryable: false, + }, + }); + }); + + test("unimplemented preload IPC reports browser_unsupported", async () => { + const client = new FakeDaemonClient(); + mountBrowserAutomationHandler({ + client, + getHost: () => ({ + browser: { + executeAutomationCommand: async () => { + throw new Error('No handler registered for "paseo:browser:execute-automation-command"'); + }, + }, + }), + }); + + client.receive(browserAutomationRequest()); + await flushPromises(); + + expect(client.sentResponses[0]?.payload).toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_unsupported", + message: "Desktop browser automation is not implemented by this desktop build yet.", + retryable: false, + }, + }); + }); + + test("unsubscribe stops handling requests", async () => { + const client = new FakeDaemonClient(); + const executeAutomationCommand = vi.fn(async () => ({ + requestId: "req-1", + ok: true as const, + result: { command: "list_tabs" as const, tabs: [] }, + })); + const unsubscribe = mountBrowserAutomationHandler({ + client, + getHost: () => ({ browser: { executeAutomationCommand } }), + }); + + unsubscribe(); + client.receive(browserAutomationRequest()); + await flushPromises(); + + expect(executeAutomationCommand).not.toHaveBeenCalled(); + expect(client.sentResponses).toEqual([]); + }); +}); diff --git a/packages/app/src/browser-automation/handler.ts b/packages/app/src/browser-automation/handler.ts new file mode 100644 index 000000000..32e675370 --- /dev/null +++ b/packages/app/src/browser-automation/handler.ts @@ -0,0 +1,343 @@ +import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; +import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host"; +import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident"; +import { createWorkspaceBrowser } from "@/stores/browser-store"; +import { + buildWorkspaceTabPersistenceKey, + useWorkspaceLayoutStore, +} from "@/stores/workspace-layout-store"; + +type BrowserAutomationExecuteRequest = Extract< + SessionOutboundMessage, + { type: "browser.automation.execute.request" } +>; +type BrowserAutomationExecuteResponse = Extract< + SessionInboundMessage, + { type: "browser.automation.execute.response" } +>; +type BrowserAutomationResponsePayload = BrowserAutomationExecuteResponse["payload"]; +type BrowserAutomationFailurePayload = Extract; +type BrowserAutomationErrorCode = BrowserAutomationFailurePayload["error"]["code"]; + +interface BrowserAutomationClient { + on( + type: "browser.automation.execute.request", + handler: (message: BrowserAutomationExecuteRequest) => void, + ): () => void; + sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void; +} + +export interface BrowserAutomationHandlerOptions { + client: BrowserAutomationClient; + serverId?: string; + getHost?: () => DesktopHostBridge | null; + ensureResidentBrowserWebview?: typeof ensureResidentBrowserWebviewDefault; + registrationWaitTimeoutMs?: number; + registrationPollIntervalMs?: number; +} + +export function mountBrowserAutomationHandler( + options: BrowserAutomationHandlerOptions, +): () => void { + const getHost = options.getHost ?? getDesktopHost; + return options.client.on("browser.automation.execute.request", (request) => { + void handleBrowserAutomationRequest({ + client: options.client, + getHost, + request, + serverId: options.serverId, + ensureResidentBrowserWebview: + options.ensureResidentBrowserWebview ?? ensureResidentBrowserWebviewDefault, + ...(options.registrationWaitTimeoutMs !== undefined + ? { registrationWaitTimeoutMs: options.registrationWaitTimeoutMs } + : {}), + ...(options.registrationPollIntervalMs !== undefined + ? { registrationPollIntervalMs: options.registrationPollIntervalMs } + : {}), + }); + }); +} + +export function mountBrowserAutomationDaemonClientHandler( + client: unknown, + options?: { serverId?: string }, +): () => void { + return mountBrowserAutomationHandler({ + client: client as BrowserAutomationClient, + ...(options?.serverId ? { serverId: options.serverId } : {}), + }); +} + +async function handleBrowserAutomationRequest(params: { + client: BrowserAutomationHandlerOptions["client"]; + getHost: () => DesktopHostBridge | null; + request: BrowserAutomationExecuteRequest; + serverId?: string; + ensureResidentBrowserWebview: typeof ensureResidentBrowserWebviewDefault; + registrationWaitTimeoutMs?: number; + registrationPollIntervalMs?: number; +}): Promise { + const { + client, + getHost, + request, + serverId, + ensureResidentBrowserWebview, + registrationWaitTimeoutMs, + registrationPollIntervalMs, + } = params; + const browserHost = getHost()?.browser; + const executeAutomationCommand = browserHost?.executeAutomationCommand; + + if (request.command.command === "new_tab") { + try { + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: await openBrowserTabForRequest({ + request, + serverId, + browserHost, + ensureResidentBrowserWebview, + ...(registrationWaitTimeoutMs !== undefined ? { registrationWaitTimeoutMs } : {}), + ...(registrationPollIntervalMs !== undefined ? { registrationPollIntervalMs } : {}), + }), + }); + } catch (error) { + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: normalizeThrownBridgeError(request.requestId, error), + }); + } + return; + } + + await rememberAgentBrowserTarget({ request, browserHost }); + + if (!executeAutomationCommand) { + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: browserAutomationFailure({ + requestId: request.requestId, + code: "browser_unsupported", + message: "Desktop browser automation is not available in this app runtime.", + }), + }); + return; + } + + try { + const payload = await executeAutomationCommand(request); + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: normalizeBridgePayload(request.requestId, payload), + }); + } catch (error) { + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: normalizeThrownBridgeError(request.requestId, error), + }); + } +} + +async function openBrowserTabForRequest(params: { + request: BrowserAutomationExecuteRequest; + serverId?: string; + browserHost: DesktopHostBridge["browser"] | undefined; + ensureResidentBrowserWebview: typeof ensureResidentBrowserWebviewDefault; + registrationWaitTimeoutMs?: number; + registrationPollIntervalMs?: number; +}): Promise { + const { + request, + serverId, + browserHost, + ensureResidentBrowserWebview, + registrationWaitTimeoutMs, + registrationPollIntervalMs, + } = params; + const command = request.command as Extract< + BrowserAutomationExecuteRequest["command"], + { command: "new_tab" } + >; + const workspaceId = request.workspaceId ?? command.args.workspaceId; + if (!serverId || !workspaceId) { + return browserAutomationFailure({ + requestId: request.requestId, + code: "browser_no_tab", + message: "Cannot create a browser tab without a workspace context.", + }); + } + + const url = command.args.url ?? "https://example.com"; + const { browserId, url: normalizedUrl } = createWorkspaceBrowser({ initialUrl: url }); + const workspaceKey = buildWorkspaceTabPersistenceKey({ serverId, workspaceId }); + if (!workspaceKey) { + return browserAutomationFailure({ + requestId: request.requestId, + code: "browser_no_tab", + message: "Cannot create a browser tab without a workspace context.", + }); + } + useWorkspaceLayoutStore.getState().openTabInBackground(workspaceKey, { + kind: "browser", + browserId, + }); + + await browserHost?.registerWorkspaceBrowser?.({ browserId, workspaceId }); + if (request.agentId) { + await browserHost?.setAgentActiveBrowser?.({ agentId: request.agentId, browserId }); + } + + if (browserHost?.executeAutomationCommand) { + ensureResidentBrowserWebview({ browserId, url: normalizedUrl }); + const registered = await waitForBrowserRegistration({ + request, + browserId, + workspaceId, + executeAutomationCommand: browserHost.executeAutomationCommand, + ...(registrationWaitTimeoutMs !== undefined ? { timeoutMs: registrationWaitTimeoutMs } : {}), + ...(registrationPollIntervalMs !== undefined + ? { pollIntervalMs: registrationPollIntervalMs } + : {}), + }); + if (!registered) { + return browserAutomationFailure({ + requestId: request.requestId, + code: "browser_timeout", + message: `Timed out waiting for browser tab ${browserId} to register with desktop automation. Try browser_new_tab again.`, + retryable: true, + }); + } + } + + return { + requestId: request.requestId, + ok: true, + result: { command: "new_tab", browserId, workspaceId, url: normalizedUrl }, + }; +} + +async function waitForBrowserRegistration(params: { + request: BrowserAutomationExecuteRequest; + browserId: string; + workspaceId: string; + executeAutomationCommand: ( + request: BrowserAutomationExecuteRequest, + ) => Promise; + timeoutMs?: number; + pollIntervalMs?: number; +}): Promise { + const deadline = Date.now() + (params.timeoutMs ?? 5_000); + while (Date.now() < deadline) { + const payload = await params.executeAutomationCommand({ + type: "browser.automation.execute.request", + requestId: `${params.request.requestId}:list_tabs`, + agentId: params.request.agentId, + cwd: params.request.cwd, + workspaceId: params.workspaceId, + command: { command: "list_tabs", args: { workspaceId: params.workspaceId } }, + }); + if (payload.ok && payload.result.command === "list_tabs") { + if (payload.result.tabs.some((tab) => tab.browserId === params.browserId)) { + return true; + } + } + await delay(params.pollIntervalMs ?? 100); + } + return false; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function rememberAgentBrowserTarget(input: { + request: BrowserAutomationExecuteRequest; + browserHost: DesktopHostBridge["browser"] | undefined; +}): Promise { + if (!input.request.agentId) { + return; + } + const browserId = readRequestBrowserId(input.request); + if (!browserId) { + return; + } + await input.browserHost?.setAgentActiveBrowser?.({ agentId: input.request.agentId, browserId }); +} + +function readRequestBrowserId(request: BrowserAutomationExecuteRequest): string | null { + if (request.browserId) { + return request.browserId; + } + const args = request.command.args as { browserId?: unknown }; + return typeof args.browserId === "string" && args.browserId.length > 0 ? args.browserId : null; +} + +function normalizeBridgePayload( + requestId: string, + payload: BrowserAutomationResponsePayload, +): BrowserAutomationResponsePayload { + return { ...payload, requestId } as BrowserAutomationResponsePayload; +} + +function normalizeThrownBridgeError( + requestId: string, + error: unknown, +): BrowserAutomationFailurePayload { + const typed = readTypedBrowserAutomationError(error); + if (typed) { + return browserAutomationFailure({ requestId, ...typed }); + } + + const message = error instanceof Error ? error.message : String(error); + if (message.includes("No handler registered")) { + return browserAutomationFailure({ + requestId, + code: "browser_unsupported", + message: "Desktop browser automation is not implemented by this desktop build yet.", + }); + } + + return browserAutomationFailure({ + requestId, + code: "browser_unknown_error", + message: message || "Desktop browser automation failed.", + }); +} + +function readTypedBrowserAutomationError( + value: unknown, +): { code: BrowserAutomationErrorCode; message: string; retryable?: boolean } | null { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return null; + } + const record = value as Record; + if (typeof record.code !== "string" || !record.code.startsWith("browser_")) { + return null; + } + if (typeof record.message !== "string" || record.message.length === 0) { + return null; + } + return { + code: record.code as BrowserAutomationErrorCode, + message: record.message, + ...(typeof record.retryable === "boolean" ? { retryable: record.retryable } : {}), + }; +} + +function browserAutomationFailure(params: { + requestId: string; + code: BrowserAutomationErrorCode; + message: string; + retryable?: boolean; +}): BrowserAutomationFailurePayload { + return { + requestId: params.requestId, + ok: false, + error: { + code: params.code, + message: params.message, + retryable: params.retryable ?? false, + }, + }; +} diff --git a/packages/app/src/components/browser-pane.electron.tsx b/packages/app/src/components/browser-pane.electron.tsx index 99ce5fd4a..7cfa1c2d2 100644 --- a/packages/app/src/components/browser-pane.electron.tsx +++ b/packages/app/src/components/browser-pane.electron.tsx @@ -25,6 +25,11 @@ import { } from "@/desktop/host"; import { isDev } from "@/constants/platform"; import { useBrowserStore, normalizeWorkspaceBrowserUrl } from "@/stores/browser-store"; +import { + prepareBrowserWebview, + releaseResidentBrowserWebview, + takeResidentBrowserWebview, +} from "./browser-webview-resident"; type ElectronWebview = HTMLElement & { canGoBack?: () => boolean; @@ -404,16 +409,16 @@ export function BrowserPane({ initialUrlRef.current, browserErrorLabelsRef.current, ); - const webview = document.createElement("webview") as ElectronWebview; + const residentWebview = takeResidentBrowserWebview(browserId) as ElectronWebview | null; + const webview = residentWebview ?? (document.createElement("webview") as ElectronWebview); webviewRef.current = webview; - webview.setAttribute("partition", `persist:paseo-browser-${browserId}`); - webview.setAttribute("allowpopups", "true"); - webview.setAttribute("spellcheck", "false"); - webview.setAttribute("autosize", "on"); - webview.setAttribute( - "src", - initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current, - ); + void getDesktopHost()?.browser?.registerWorkspaceBrowser?.({ browserId, workspaceId }); + if (!residentWebview) { + prepareBrowserWebview(webview, { + browserId, + initialUrl: initialUnsafeNavigationMessage ? "about:blank" : initialUrlRef.current, + }); + } webview.style.display = "flex"; webview.style.flex = "1"; webview.style.width = "100%"; @@ -528,7 +533,14 @@ export function BrowserPane({ webview.removeEventListener("focus", handleWebviewFocus); webview.removeEventListener("mousedown", handleWebviewFocus); if (host.contains(webview)) { - host.removeChild(webview); + const browserStillExists = Boolean( + useBrowserStore.getState().browsersById[browserIdRef.current], + ); + if (browserStillExists) { + releaseResidentBrowserWebview(browserIdRef.current, webview); + } else { + host.removeChild(webview); + } } if (webviewRef.current === webview) { webviewRef.current = null; diff --git a/packages/app/src/components/browser-webview-resident.browser.test.ts b/packages/app/src/components/browser-webview-resident.browser.test.ts new file mode 100644 index 000000000..48ab1a8c4 --- /dev/null +++ b/packages/app/src/components/browser-webview-resident.browser.test.ts @@ -0,0 +1,60 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + clearResidentBrowserWebviewsForTests, + ensureResidentBrowserWebview, + releaseResidentBrowserWebview, + removeResidentBrowserWebview, + takeResidentBrowserWebview, +} from "./browser-webview-resident"; + +describe("resident browser webviews", () => { + afterEach(() => { + clearResidentBrowserWebviewsForTests(); + }); + + it("keeps a browser webview mounted offscreen and reuses the same node", () => { + const host = document.createElement("div"); + const webview = document.createElement("webview"); + host.appendChild(webview); + document.body.appendChild(host); + + releaseResidentBrowserWebview("browser-a", webview); + + expect(host.children).toHaveLength(0); + expect(webview.isConnected).toBe(true); + expect(webview.style.width).toBe("1280px"); + expect(webview.style.height).toBe("800px"); + + const reused = takeResidentBrowserWebview("browser-a"); + + expect(reused).toBe(webview); + expect(takeResidentBrowserWebview("browser-a")).toBeNull(); + }); + + it("creates a resident webview for an agent-created unfocused tab", () => { + const webview = ensureResidentBrowserWebview({ + browserId: "browser-agent", + url: "https://example.com", + }); + + expect(webview).not.toBeNull(); + expect(webview?.isConnected).toBe(true); + expect(webview?.getAttribute("data-paseo-browser-id")).toBe("browser-agent"); + expect(webview?.getAttribute("partition")).toBe("persist:paseo-browser-browser-agent"); + expect((webview as HTMLUnknownElement & { src?: string })?.src).toContain( + "https://example.com", + ); + }); + + it("removes a resident webview when its browser tab closes", () => { + const webview = ensureResidentBrowserWebview({ + browserId: "browser-closed", + url: "https://example.com", + }); + + removeResidentBrowserWebview("browser-closed"); + + expect(webview?.isConnected).toBe(false); + expect(takeResidentBrowserWebview("browser-closed")).toBeNull(); + }); +}); diff --git a/packages/app/src/components/browser-webview-resident.ts b/packages/app/src/components/browser-webview-resident.ts new file mode 100644 index 000000000..a33ff8c69 --- /dev/null +++ b/packages/app/src/components/browser-webview-resident.ts @@ -0,0 +1,153 @@ +const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews"; +const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id"; +const RESIDENT_VIEWPORT_WIDTH = 1280; +const RESIDENT_VIEWPORT_HEIGHT = 800; + +const residentWebviewsByBrowserId = new Map(); + +interface BrowserWebviewElement extends HTMLElement { + src: string; +} + +function trimNonEmpty(value: string | null | undefined): string | null { + if (typeof value !== "string") { + return null; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function readDocument(): Document | null { + return typeof document === "undefined" ? null : document; +} + +function getResidentBrowserHost(ownerDocument: Document): HTMLElement { + const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID); + if (existing) { + return existing; + } + + const host = ownerDocument.createElement("div"); + host.id = RESIDENT_BROWSER_HOST_ID; + host.setAttribute("aria-hidden", "true"); + host.style.position = "fixed"; + host.style.left = "-20000px"; + host.style.top = "0"; + host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`; + host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`; + host.style.overflow = "hidden"; + host.style.opacity = "0"; + host.style.pointerEvents = "none"; + ownerDocument.body.appendChild(host); + return host; +} + +function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null { + for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) { + if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) { + return element as HTMLElement; + } + } + return null; +} + +function applyResidentWebviewStyle(webview: HTMLElement): void { + webview.style.display = "block"; + webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`; + webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`; + webview.style.border = "0"; + webview.style.background = "transparent"; +} + +export function prepareBrowserWebview( + webview: HTMLElement, + input: { browserId: string; initialUrl?: string | null }, +): void { + webview.setAttribute(BROWSER_ID_ATTRIBUTE, input.browserId); + webview.setAttribute("partition", `persist:paseo-browser-${input.browserId}`); + webview.setAttribute("allowpopups", "true"); + webview.setAttribute("spellcheck", "false"); + webview.setAttribute("autosize", "on"); + if (input.initialUrl) { + (webview as BrowserWebviewElement).src = input.initialUrl; + } +} + +export function ensureResidentBrowserWebview(input: { + browserId: string; + url: string; +}): HTMLElement | null { + const browserId = trimNonEmpty(input.browserId); + if (!browserId) { + return null; + } + const ownerDocument = readDocument(); + if (!ownerDocument) { + return null; + } + + const resident = residentWebviewsByBrowserId.get(browserId) ?? null; + if (resident?.isConnected) { + return resident; + } + + const existing = findBrowserWebview(browserId, ownerDocument); + if (existing) { + return existing; + } + + const webview = ownerDocument.createElement("webview") as BrowserWebviewElement; + prepareBrowserWebview(webview, { browserId, initialUrl: input.url }); + releaseResidentBrowserWebview(browserId, webview); + return webview; +} + +export function takeResidentBrowserWebview(browserId: string): HTMLElement | null { + const normalizedBrowserId = trimNonEmpty(browserId); + if (!normalizedBrowserId) { + return null; + } + + const webview = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null; + if (!webview) { + return null; + } + + residentWebviewsByBrowserId.delete(normalizedBrowserId); + return webview; +} + +export function releaseResidentBrowserWebview(browserId: string, webview: HTMLElement): void { + const normalizedBrowserId = trimNonEmpty(browserId); + if (!normalizedBrowserId) { + webview.remove(); + return; + } + const ownerDocument = readDocument(); + if (!ownerDocument) { + return; + } + + residentWebviewsByBrowserId.set(normalizedBrowserId, webview); + applyResidentWebviewStyle(webview); + getResidentBrowserHost(ownerDocument).appendChild(webview); +} + +export function removeResidentBrowserWebview(browserId: string): void { + const normalizedBrowserId = trimNonEmpty(browserId); + if (!normalizedBrowserId) { + return; + } + + const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null; + residentWebviewsByBrowserId.delete(normalizedBrowserId); + resident?.remove(); +} + +export function clearResidentBrowserWebviewsForTests(): void { + for (const webview of residentWebviewsByBrowserId.values()) { + webview.remove(); + } + residentWebviewsByBrowserId.clear(); + readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove(); +} diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index 3de1e50b4..2af6d692e 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -1,5 +1,15 @@ import { Platform } from "react-native"; import { getElectronHost } from "@/desktop/electron/host"; +import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; + +type BrowserAutomationExecuteRequest = Extract< + SessionOutboundMessage, + { type: "browser.automation.execute.request" } +>; +type BrowserAutomationExecuteResponse = Extract< + SessionInboundMessage, + { type: "browser.automation.execute.response" } +>; export type DesktopNotificationPermission = "granted" | "denied" | "default"; @@ -117,9 +127,17 @@ export interface DesktopBrowserNewTabRequestEvent { } export interface DesktopBrowserBridge { - setWorkspaceActiveBrowser?: (browserId: string | null) => Promise; + registerWorkspaceBrowser?: (input: { browserId: string; workspaceId: string }) => Promise; + setWorkspaceActiveBrowser?: (input: { + workspaceId: string; + browserId: string | null; + }) => Promise; + setAgentActiveBrowser?: (input: { agentId: string; browserId: string | null }) => Promise; openDevTools?: (browserId: string) => Promise; clearPartition?: (browserId: string) => Promise; + executeAutomationCommand?: ( + request: BrowserAutomationExecuteRequest, + ) => Promise; } export interface DesktopInvokeBridge { diff --git a/packages/app/src/runtime/host-runtime.test.ts b/packages/app/src/runtime/host-runtime.test.ts index 28bec8aeb..74af37c4d 100644 --- a/packages/app/src/runtime/host-runtime.test.ts +++ b/packages/app/src/runtime/host-runtime.test.ts @@ -1,4 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.hoisted(() => { + Object.defineProperty(globalThis, "__DEV__", { value: false, configurable: true }); +}); import type { DaemonClient, ConnectionState, @@ -16,6 +20,10 @@ import { type HostRuntimeStorage, } from "./host-runtime"; +vi.mock("@/browser-automation/handler", () => ({ + mountBrowserAutomationDaemonClientHandler: vi.fn(() => () => undefined), +})); + class FakeDaemonClient { private state: ConnectionState = { status: "idle" }; private listeners = new Set<(status: ConnectionState) => void>(); @@ -342,6 +350,18 @@ function onceHostListMatches(store: HostRuntimeStore, predicate: () => boolean): }); } +class BrowserClientLifecycle { + public active: Array<{ serverId: string; connectionId: string }> = []; + + mount(input: { host: HostProfile; connection: HostConnection }): () => void { + const entry = { serverId: input.host.serverId, connectionId: input.connection.id }; + this.active.push(entry); + return () => { + this.active = this.active.filter((current) => current !== entry); + }; + } +} + describe("HostRuntimeController", () => { it("replaces the active relay client when re-pairing changes the daemon public key", async () => { const oldRelay: HostConnection = { @@ -458,6 +478,39 @@ describe("HostRuntimeController", () => { expect(controller.getSnapshot().connectionStatus).toBe("online"); }); + it("keeps browser client lifecycle tied to the active host runtime client", async () => { + const host = makeHost({ preferredConnectionId: "direct:lan:6767" }); + const fakeClient = makeConnectedProbeClient(12); + const lifecycle = new BrowserClientLifecycle(); + const controller = new HostRuntimeController({ + host, + deps: { + createClient: () => new FakeDaemonClient() as unknown as DaemonClient, + connectToDaemon: async ({ host: hostProfile, connection }) => ({ + client: makeConnectedProbeClient(10) as unknown as DaemonClient, + serverId: hostProfile.serverId, + hostname: connection.id, + }), + getClientId: async () => "cid_runtime_stable", + mountClientHandlers: (input) => lifecycle.mount(input), + }, + }); + + await controller.start({ + autoProbe: false, + initialConnection: { + connectionId: "direct:lan:6767", + existingClient: fakeClient as unknown as DaemonClient, + }, + }); + + expect(lifecycle.active).toEqual([{ serverId: "srv_test", connectionId: "direct:lan:6767" }]); + + await controller.stop(); + + expect(lifecycle.active).toEqual([]); + }); + it("adopts the first successful probe on startup", async () => { const host = makeHost({ preferredConnectionId: "direct:lan:6767" }); const clients: FakeDaemonClient[] = []; @@ -1431,7 +1484,7 @@ describe("HostRuntimeStore", () => { entries: [ makeFetchAgentsEntry({ id: "agent-recent", - cwd: "/Users/moboudra/dev/paseo", + cwd: "/workspaces/paseo", updatedAt: "2026-03-04T12:00:00.000Z", title: "Recent agent", }), @@ -1444,7 +1497,7 @@ describe("HostRuntimeStore", () => { entries: [ makeFetchAgentsEntry({ id: "agent-stale-attention", - cwd: "/Users/moboudra/dev/paseo-pr67-review", + cwd: "/workspaces/paseo-pr67-review", updatedAt: "2026-02-20T08:00:00.000Z", title: "Needs triage", requiresAttention: true, @@ -1612,7 +1665,7 @@ describe("HostRuntimeStore", () => { useSessionStore.getState().setAgents(host.serverId, () => { const stale = makeFetchAgentsEntry({ id: "agent-archived", - cwd: "/Users/moboudra/dev/paseo", + cwd: "/workspaces/paseo", updatedAt: "2026-03-30T15:29:00.000Z", archivedAt: null, title: "Stale active copy", diff --git a/packages/app/src/runtime/host-runtime.ts b/packages/app/src/runtime/host-runtime.ts index 9423a9609..23633c3b2 100644 --- a/packages/app/src/runtime/host-runtime.ts +++ b/packages/app/src/runtime/host-runtime.ts @@ -37,6 +37,8 @@ import { buildLocalDaemonTransportUrl, createDesktopLocalDaemonTransportFactory, } from "@/desktop/daemon/desktop-daemon-transport"; +import { getDesktopHost } from "@/desktop/host"; +import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; import { replaceFetchedAgentDirectory } from "@/utils/agent-directory-sync"; import { useSessionStore } from "@/stores/session-store"; import { @@ -45,6 +47,7 @@ import { } from "@/workspace/legacy-daemon-workspaces"; import { invalidateCheckoutGitQueriesForServer } from "@/git/query-keys"; import { queryClient } from "@/query/query-client"; +import { mountBrowserAutomationDaemonClientHandler } from "@/browser-automation/handler"; export type HostRuntimeConnectionStatus = "idle" | "connecting" | "online" | "offline" | "error"; export type HostRegistryStatus = "loading" | "ready"; @@ -138,6 +141,11 @@ export interface HostRuntimeControllerDeps { }>; getClientId: () => Promise; readInitialConnectionHint?: () => InitialDaemonConnectionHint | null; + mountClientHandlers?: (input: { + client: DaemonClient; + host: HostProfile; + connection: HostConnection; + }) => () => void; } export interface HostRuntimeStorage { @@ -514,6 +522,12 @@ function probeIntervalForConnection( } function createDefaultDeps(): HostRuntimeControllerDeps { + const desktopBrowserAutomationAvailable = + typeof getDesktopHost()?.browser?.executeAutomationCommand === "function"; + const browserAutomationCapabilities = desktopBrowserAutomationAvailable + ? { [CLIENT_CAPS.desktopBrowserAutomation]: true } + : undefined; + return { createClient: ({ host, connection, clientId, runtimeGeneration }) => { const localTransportFactory = createDesktopLocalDaemonTransportFactory(); @@ -523,6 +537,7 @@ function createDefaultDeps(): HostRuntimeControllerDeps { clientType: "mobile" as const, appVersion: resolveAppVersion() ?? undefined, runtimeGeneration, + ...(browserAutomationCapabilities ? { capabilities: browserAutomationCapabilities } : {}), }; if (connection.type === "directSocket" || connection.type === "directPipe") { return new DaemonClient({ @@ -560,8 +575,15 @@ function createDefaultDeps(): HostRuntimeControllerDeps { connectToDaemon(connection, { ...(host.serverId ? { serverId: host.serverId } : {}), ...(timeoutMs !== undefined ? { timeoutMs } : {}), + ...(browserAutomationCapabilities ? { capabilities: browserAutomationCapabilities } : {}), }), getClientId: () => getOrCreateClientId(), + mountClientHandlers: ({ client, host }) => { + if (!browserAutomationCapabilities) { + return () => {}; + } + return mountBrowserAutomationDaemonClientHandler(client, { serverId: host.serverId }); + }, }; } @@ -574,6 +596,7 @@ export class HostRuntimeController { private listeners = new Set<() => void>(); private activeClient: DaemonClient | null = null; private unsubscribeClientStatus: (() => void) | null = null; + private unsubscribeClientHandlers: (() => void) | null = null; private probeIntervalHandle: ReturnType | null = null; private started = false; private connectionFirstSeenAt = new Map(); @@ -656,6 +679,10 @@ export class HostRuntimeController { this.unsubscribeClientStatus(); this.unsubscribeClientStatus = null; } + if (this.unsubscribeClientHandlers) { + this.unsubscribeClientHandlers(); + this.unsubscribeClientHandlers = null; + } if (this.activeClient) { const prev = this.activeClient; this.activeClient = null; @@ -1133,6 +1160,10 @@ export class HostRuntimeController { this.unsubscribeClientStatus(); this.unsubscribeClientStatus = null; } + if (this.unsubscribeClientHandlers) { + this.unsubscribeClientHandlers(); + this.unsubscribeClientHandlers = null; + } if (this.activeClient) { const previousClient = this.activeClient; this.activeClient = null; @@ -1206,6 +1237,8 @@ export class HostRuntimeController { } this.activeClient = client; + this.unsubscribeClientHandlers = + this.deps.mountClientHandlers?.({ client, host: this.host, connection }) ?? null; this.applyConnectionEvent({ type: "select_connection", connectionId: connection.id, diff --git a/packages/app/src/screens/settings/browser-tools-card.tsx b/packages/app/src/screens/settings/browser-tools-card.tsx new file mode 100644 index 000000000..9a0c3100b --- /dev/null +++ b/packages/app/src/screens/settings/browser-tools-card.tsx @@ -0,0 +1,70 @@ +import React, { useCallback } from "react"; +import { Text, View } from "react-native"; +import { useMutation } from "@tanstack/react-query"; +import { useTranslation } from "react-i18next"; +import { Switch } from "@/components/ui/switch"; +import { useDaemonConfig } from "@/hooks/use-daemon-config"; +import { useHostRuntimeIsConnected } from "@/runtime/host-runtime"; +import { settingsStyles } from "@/styles/settings"; +import { + createBrowserToolsPatch, + getBrowserToolsCardState, + getBrowserToolsMutationViewState, +} from "./browser-tools-config"; + +export function BrowserToolsOptInCard({ serverId }: { serverId: string }) { + const { t } = useTranslation(); + const isConnected = useHostRuntimeIsConnected(serverId); + const { config, patchConfig } = useDaemonConfig(serverId); + const state = getBrowserToolsCardState({ isConnected, config }); + const mutation = useMutation({ + mutationFn: async (next: boolean) => { + const result = await patchConfig(createBrowserToolsPatch(next)); + if (!result) { + throw new Error(t("workspace.terminal.hostDisconnected")); + } + return result; + }, + }); + const mutationView = getBrowserToolsMutationViewState({ + isPending: mutation.isPending, + error: mutation.error, + }); + + const handleValueChange = useCallback( + (next: boolean) => { + mutation.mutate(next); + }, + [mutation], + ); + + if (!state.isVisible) return null; + + return ( + + + + {state.title} + {state.warning} + {mutationView.loadingText ? ( + + {mutationView.loadingText} + + ) : null} + {mutationView.errorText ? ( + + {mutationView.errorText} + + ) : null} + + + + + ); +} diff --git a/packages/app/src/screens/settings/browser-tools-config.test.ts b/packages/app/src/screens/settings/browser-tools-config.test.ts new file mode 100644 index 000000000..99b9fa3a1 --- /dev/null +++ b/packages/app/src/screens/settings/browser-tools-config.test.ts @@ -0,0 +1,70 @@ +import type { MutableDaemonConfig } from "@getpaseo/protocol/messages"; +import { describe, expect, it } from "vitest"; +import { + BROWSER_TOOLS_WARNING, + createBrowserToolsPatch, + getBrowserToolsCardState, + getBrowserToolsMutationViewState, +} from "./browser-tools-config"; + +function makeConfig(browserToolsEnabled = false): MutableDaemonConfig { + return { + mcp: { injectIntoAgents: false }, + browserTools: { enabled: browserToolsEnabled }, + providers: {}, + metadataGeneration: { providers: [] }, + autoArchiveAfterMerge: false, + enableTerminalAgentHooks: false, + appendSystemPrompt: "", + }; +} + +describe("browser tools opt-in config", () => { + it("shows the card with the logged-in browser state warning when connected", () => { + expect(getBrowserToolsCardState({ isConnected: true, config: makeConfig(false) })).toEqual({ + isVisible: true, + isEnabled: false, + title: "Browser tools", + warning: BROWSER_TOOLS_WARNING, + }); + }); + + it("reads enabled state from daemon config", () => { + expect(getBrowserToolsCardState({ isConnected: true, config: makeConfig(true) })).toMatchObject( + { + isEnabled: true, + }, + ); + }); + + it("hides the card when the host is disconnected", () => { + expect( + getBrowserToolsCardState({ isConnected: false, config: makeConfig(true) }), + ).toMatchObject({ + isVisible: false, + }); + }); + + it("writes daemon.browserTools.enabled when toggled", () => { + expect(createBrowserToolsPatch(true)).toEqual({ browserTools: { enabled: true } }); + expect(createBrowserToolsPatch(false)).toEqual({ browserTools: { enabled: false } }); + }); + + it("shows loading and disables the toggle while browser tool settings save", () => { + expect(getBrowserToolsMutationViewState({ isPending: true, error: null })).toEqual({ + isSwitchDisabled: true, + loadingText: "Updating browser tools…", + errorText: null, + }); + }); + + it("shows the save error when browser tool settings fail", () => { + expect( + getBrowserToolsMutationViewState({ isPending: false, error: new Error("Disk full") }), + ).toEqual({ + isSwitchDisabled: false, + loadingText: null, + errorText: "Disk full", + }); + }); +}); diff --git a/packages/app/src/screens/settings/browser-tools-config.ts b/packages/app/src/screens/settings/browser-tools-config.ts new file mode 100644 index 000000000..c84b92ec3 --- /dev/null +++ b/packages/app/src/screens/settings/browser-tools-config.ts @@ -0,0 +1,49 @@ +import type { MutableDaemonConfig } from "@getpaseo/protocol/messages"; + +export const BROWSER_TOOLS_TITLE = "Browser tools"; +export const BROWSER_TOOLS_WARNING = + "Allow agents to access and control Paseo desktop browser tabs, including logged-in browser state. Only enable this for agents you trust."; + +export interface BrowserToolsCardState { + isVisible: boolean; + isEnabled: boolean; + title: string; + warning: string; +} + +export interface BrowserToolsMutationViewState { + isSwitchDisabled: boolean; + loadingText: string | null; + errorText: string | null; +} + +export function getBrowserToolsCardState(input: { + isConnected: boolean; + config: MutableDaemonConfig | null; +}): BrowserToolsCardState { + return { + isVisible: input.isConnected, + isEnabled: input.config?.browserTools.enabled === true, + title: BROWSER_TOOLS_TITLE, + warning: BROWSER_TOOLS_WARNING, + }; +} + +export function createBrowserToolsPatch(enabled: boolean): Partial { + return { browserTools: { enabled } }; +} + +export function getBrowserToolsMutationViewState(input: { + isPending: boolean; + error: unknown; +}): BrowserToolsMutationViewState { + return { + isSwitchDisabled: input.isPending, + loadingText: input.isPending ? "Updating browser tools…" : null, + errorText: input.error ? toErrorMessage(input.error) : null, + }; +} + +function toErrorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/app/src/screens/settings/host-page.tsx b/packages/app/src/screens/settings/host-page.tsx index ddefea1e9..ca4ddc7a3 100644 --- a/packages/app/src/screens/settings/host-page.tsx +++ b/packages/app/src/screens/settings/host-page.tsx @@ -61,6 +61,7 @@ import { formatLatency } from "@/utils/latency"; import { ICON_SIZE } from "@/styles/theme"; import type { Theme } from "@/styles/theme"; import { getProviderIcon } from "@/components/provider-icons"; +import { BrowserToolsOptInCard } from "./browser-tools-card"; const ThemedArrowUp = withUnistyles(ArrowUp); const ThemedArrowDown = withUnistyles(ArrowDown); @@ -262,6 +263,7 @@ export function HostAgentsPage({ serverId }: { serverId: string }) { {isConnected ? ( + ) : ( diff --git a/packages/app/src/screens/settings/providers-section.test.tsx b/packages/app/src/screens/settings/providers-section.test.tsx index 4e16a5d03..c0bed32b1 100644 --- a/packages/app/src/screens/settings/providers-section.test.tsx +++ b/packages/app/src/screens/settings/providers-section.test.tsx @@ -220,6 +220,7 @@ const disabledCodexEntry: ProviderSnapshotEntry = { function makeConfig(providers: MutableDaemonConfig["providers"] = {}): MutableDaemonConfig { return { mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, providers, metadataGeneration: { providers: [] }, autoArchiveAfterMerge: false, diff --git a/packages/app/src/screens/workspace/workspace-screen.tsx b/packages/app/src/screens/workspace/workspace-screen.tsx index 39dc14391..5fc3133ca 100644 --- a/packages/app/src/screens/workspace/workspace-screen.tsx +++ b/packages/app/src/screens/workspace/workspace-screen.tsx @@ -105,6 +105,7 @@ import type { CheckoutStatusPayload } from "@/git/use-status-query"; import { confirmDialog } from "@/utils/confirm-dialog"; import { useArchiveAgent } from "@/hooks/use-archive-agent"; import { useStableEvent } from "@/hooks/use-stable-event"; +import { removeResidentBrowserWebview } from "@/components/browser-webview-resident"; import { createWorkspaceBrowser, useBrowserStore } from "@/stores/browser-store"; import { getDesktopHost } from "@/desktop/host"; import { buildProviderCommand } from "@/utils/provider-command-templates"; @@ -304,19 +305,22 @@ function decodeSegment(value: string): string { function useSyncWorkspaceActiveBrowser(input: { workspaceLayout: WorkspaceLayout | null; isRouteFocused: boolean; + workspaceId: string; }) { const focusedBrowserId = useMemo( () => getFocusedBrowserId(input.workspaceLayout), [input.workspaceLayout], ); - const desktopActiveBrowserId = input.isRouteFocused ? focusedBrowserId : null; useEffect(() => { if (!getIsElectron()) { return; } - void getDesktopHost()?.browser?.setWorkspaceActiveBrowser?.(desktopActiveBrowserId); - }, [desktopActiveBrowserId]); + void getDesktopHost()?.browser?.setWorkspaceActiveBrowser?.({ + workspaceId: input.workspaceId, + browserId: focusedBrowserId, + }); + }, [focusedBrowserId, input.workspaceId]); } function getFallbackTabOptionLabel( @@ -1876,7 +1880,11 @@ function WorkspaceScreenContent({ () => (workspaceLayout ? collectAllTabs(workspaceLayout.root) : EMPTY_UI_TABS), [workspaceLayout], ); - useSyncWorkspaceActiveBrowser({ workspaceLayout, isRouteFocused }); + useSyncWorkspaceActiveBrowser({ + workspaceLayout, + isRouteFocused, + workspaceId: normalizedWorkspaceId, + }); const openWorkspaceTabInBackground = useWorkspaceLayoutStore( (state) => state.openTabInBackground, ); @@ -1921,6 +1929,7 @@ function WorkspaceScreenContent({ if (input.target?.kind === "browser") { const { browserId } = input.target; useBrowserStore.getState().removeBrowser(browserId); + removeResidentBrowserWebview(browserId); void getDesktopHost()?.browser?.clearPartition?.(browserId); } closeWorkspaceTab(persistenceKey, normalizedTabId); diff --git a/packages/app/src/styles/settings.ts b/packages/app/src/styles/settings.ts index 75b8c8c92..08f685d80 100644 --- a/packages/app/src/styles/settings.ts +++ b/packages/app/src/styles/settings.ts @@ -63,4 +63,9 @@ export const settingsStyles = StyleSheet.create((theme) => ({ fontSize: theme.fontSize.xs, marginTop: theme.spacing[1], }, + rowError: { + color: theme.colors.statusDanger, + fontSize: theme.fontSize.xs, + marginTop: theme.spacing[1], + }, })); diff --git a/packages/app/src/utils/assistant-image-metadata.test.ts b/packages/app/src/utils/assistant-image-metadata.test.ts index d8e8d1569..e6138dc71 100644 --- a/packages/app/src/utils/assistant-image-metadata.test.ts +++ b/packages/app/src/utils/assistant-image-metadata.test.ts @@ -25,7 +25,7 @@ describe("assistant image metadata", () => { setAssistantImageMetadata( { source: "/tmp/paseo-codex-screenshot.png", - workspaceRoot: "/Users/moboudra/dev/paseo", + workspaceRoot: "/workspaces/paseo", serverId: "server-1", }, { width: 1200, height: 800 }, diff --git a/packages/app/src/utils/explorer-paths.test.ts b/packages/app/src/utils/explorer-paths.test.ts index a504525f9..97f06895c 100644 --- a/packages/app/src/utils/explorer-paths.test.ts +++ b/packages/app/src/utils/explorer-paths.test.ts @@ -5,28 +5,28 @@ describe("buildAbsoluteExplorerPath", () => { it("builds a POSIX absolute path from a relative explorer path", () => { expect( buildAbsoluteExplorerPath({ - workspaceRoot: "/Users/moboudra/dev/paseo", + workspaceRoot: "/workspaces/paseo", entryPath: "packages/app/src/components/file-explorer-pane.tsx", }), - ).toBe("/Users/moboudra/dev/paseo/packages/app/src/components/file-explorer-pane.tsx"); + ).toBe("/workspaces/paseo/packages/app/src/components/file-explorer-pane.tsx"); }); it("returns workspace root when entry path points to explorer root", () => { expect( buildAbsoluteExplorerPath({ - workspaceRoot: "/Users/moboudra/dev/paseo", + workspaceRoot: "/workspaces/paseo", entryPath: ".", }), - ).toBe("/Users/moboudra/dev/paseo"); + ).toBe("/workspaces/paseo"); }); it("trims trailing separators from workspace root before joining", () => { expect( buildAbsoluteExplorerPath({ - workspaceRoot: "/Users/moboudra/dev/paseo/", + workspaceRoot: "/workspaces/paseo/", entryPath: "README.md", }), - ).toBe("/Users/moboudra/dev/paseo/README.md"); + ).toBe("/workspaces/paseo/README.md"); }); it("builds a Windows absolute path with backslash separators", () => { @@ -41,7 +41,7 @@ describe("buildAbsoluteExplorerPath", () => { it("passes through an already-absolute entry path", () => { expect( buildAbsoluteExplorerPath({ - workspaceRoot: "/Users/moboudra/dev/paseo", + workspaceRoot: "/workspaces/paseo", entryPath: "/tmp/another/location.txt", }), ).toBe("/tmp/another/location.txt"); diff --git a/packages/app/src/utils/host-routes.test.ts b/packages/app/src/utils/host-routes.test.ts index 7d9754df6..67fb9d856 100644 --- a/packages/app/src/utils/host-routes.test.ts +++ b/packages/app/src/utils/host-routes.test.ts @@ -45,8 +45,8 @@ describe("workspace route parsing", () => { }); it("decodes non-canonical base64url workspace IDs used by older links", () => { - expect(decodeWorkspaceIdFromPathSegment("L1VzZXJzL21vYm91ZHJhL2Rldi9wYXNlby")).toBe( - "/Users/moboudra/dev/paseo", + expect(decodeWorkspaceIdFromPathSegment("L2hvbWUvdXNlci9kZXYvcGFzZW8")).toBe( + "/home/user/dev/paseo", ); }); diff --git a/packages/app/src/utils/test-daemon-connection.ts b/packages/app/src/utils/test-daemon-connection.ts index 5a91e78b8..4f138e3c2 100644 --- a/packages/app/src/utils/test-daemon-connection.ts +++ b/packages/app/src/utils/test-daemon-connection.ts @@ -97,6 +97,7 @@ export class DaemonConnectionTestError extends Error { export async function buildClientConfig( connection: HostConnection, serverId?: string, + options?: { capabilities?: DaemonClientConfig["capabilities"] }, deps: Pick< DaemonConnectionDependencies, "getClientId" | "resolveAppVersion" | "createLocalTransportFactory" | "buildLocalTransportUrl" @@ -110,6 +111,7 @@ export async function buildClientConfig( appVersion: deps.resolveAppVersion() ?? undefined, suppressSendErrors: true, reconnect: { enabled: false }, + ...(options?.capabilities ? { capabilities: options.capabilities } : {}), ...((connection.type === "directSocket" || connection.type === "directPipe") && localTransportFactory ? { transportFactory: localTransportFactory } @@ -221,6 +223,7 @@ export function connectAndProbe( interface ProbeOptions { serverId?: string; timeoutMs?: number; + capabilities?: DaemonClientConfig["capabilities"]; } function resolveTimeout(connection: HostConnection, options?: ProbeOptions): number { @@ -242,6 +245,6 @@ export async function connectToDaemon( options?: ProbeOptions, deps: DaemonConnectionDependencies = defaultDaemonConnectionDependencies, ): Promise<{ client: DaemonProbeClient; serverId: string; hostname: string | null }> { - const config = await buildClientConfig(connection, options?.serverId, deps); + const config = await buildClientConfig(connection, options?.serverId, options, deps); return connectAndProbe(config, resolveTimeout(connection, options), deps); } diff --git a/packages/app/vitest.setup.ts b/packages/app/vitest.setup.ts index 50a5734d0..6fca0ed79 100644 --- a/packages/app/vitest.setup.ts +++ b/packages/app/vitest.setup.ts @@ -1,5 +1,6 @@ // @ts-nocheck import { vi } from "vitest"; +import React from "react"; const globalWithTestShims = globalThis as typeof globalThis & Record; @@ -94,3 +95,33 @@ vi.mock("react-native-svg", () => { vi.mock("expo-linking", () => ({ openURL: vi.fn().mockResolvedValue(undefined), })); + +const RouterPassthrough = ({ children }: { children?: React.ReactNode }) => children; + +vi.mock("expo-router", () => ({ + Redirect: () => null, + Stack: Object.assign(RouterPassthrough, { + Screen: () => null, + Protected: RouterPassthrough, + }), + router: { + back: vi.fn(), + canGoBack: vi.fn(() => false), + navigate: vi.fn(), + push: vi.fn(), + replace: vi.fn(), + setParams: vi.fn(), + }, + useGlobalSearchParams: vi.fn(() => ({})), + useLocalSearchParams: vi.fn(() => ({})), + usePathname: vi.fn(() => "/"), + useRootNavigationState: vi.fn(() => ({ key: "root" })), + useRouter: vi.fn(() => ({ + back: vi.fn(), + canGoBack: vi.fn(() => false), + navigate: vi.fn(), + push: vi.fn(), + replace: vi.fn(), + setParams: vi.fn(), + })), +})); diff --git a/packages/client/src/daemon-client.test.ts b/packages/client/src/daemon-client.test.ts index d0cd496d4..ce927e0d8 100644 --- a/packages/client/src/daemon-client.test.ts +++ b/packages/client/src/daemon-client.test.ts @@ -1,6 +1,7 @@ import { afterEach, expect, expectTypeOf, test, vi } from "vitest"; import { z } from "zod"; import { DaemonClient, type DaemonTransport, type Logger } from "./daemon-client"; +import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; import { decodeFileTransferFrame, encodeFileTransferFrame, @@ -136,6 +137,58 @@ afterEach(async () => { await Promise.all(clients.map((client) => client.close())); clients.length = 0; vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +test("does not infer browser automation capabilities from Electron runtime", async () => { + vi.stubGlobal("navigator", { + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) Paseo/0.1.89 Chrome/146 Electron/41.2.0 Safari/537.36", + }); + const mock = createMockTransport(); + const client = new DaemonClient({ + url: "ws://test", + clientId: "electron_unit_test", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + + const connectPromise = client.connect(); + mock.triggerOpen({ preserveSent: true }); + await connectPromise; + + const hello = z + .object({ + type: z.literal("hello"), + capabilities: z.record(z.unknown()), + }) + .parse(JSON.parse(assertStr(mock.sent[0]))); + expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBeUndefined(); +}); + +test("advertises consumer-provided browser automation capabilities", async () => { + const mock = createMockTransport(); + const client = new DaemonClient({ + url: "ws://test", + clientId: "browser_capability_unit_test", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true }, + }); + clients.push(client); + + const connectPromise = client.connect(); + mock.triggerOpen({ preserveSent: true }); + await connectPromise; + + const hello = z + .object({ + type: z.literal("hello"), + capabilities: z.record(z.unknown()), + }) + .parse(JSON.parse(assertStr(mock.sent[0]))); + expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBe(true); }); const noopLogger: Logger = { @@ -453,6 +506,7 @@ test("advertises client capabilities in hello", async () => { logger, reconnect: { enabled: false }, transportFactory: () => mock.transport, + capabilities: { desktop_browser_automation: true }, }); clients.push(client); @@ -470,6 +524,43 @@ test("advertises client capabilities in hello", async () => { custom_mode_icons: true, reasoning_merge_enum: true, terminal_reflowable_snapshot: true, + desktop_browser_automation: true, + }, + }); +}); + +test("sends typed browser automation execute responses", async () => { + const logger = createMockLogger(); + const mock = createMockTransport(); + + const client = new DaemonClient({ + url: "ws://test", + clientId: "clsk_unit_test", + logger, + reconnect: { enabled: false }, + transportFactory: () => mock.transport, + }); + clients.push(client); + + const connectPromise = client.connect(); + mock.triggerOpen(); + await connectPromise; + + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }, + }); + + expect(parseSentFrame(mock.sent[0])).toEqual({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: true, + result: { command: "list_tabs", tabs: [] }, }, }); }); diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index 18240ceb2..4218cbbce 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -1,5 +1,5 @@ import type { z } from "zod"; -import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; +import { CLIENT_CAPS, type ClientCapability } from "@getpaseo/protocol/client-capabilities"; import { AgentCreateFailedStatusPayloadSchema, AgentCreatedStatusPayloadSchema, @@ -119,6 +119,10 @@ import { } from "./daemon-client-transport.js"; import { DaemonClientRuntimeMetrics } from "./daemon-client-runtime-metrics.js"; import { TerminalStreamRouter, type TerminalStreamEvent } from "./terminal-stream-router.js"; +import type { + BrowserAutomationExecuteRequest, + BrowserAutomationExecuteResponse, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; export interface Logger { debug(obj: object, msg?: string): void; @@ -221,6 +225,8 @@ export type DaemonEvent = | { type: "error"; message: string }; export type DaemonEventHandler = (event: DaemonEvent) => void; +export type BrowserAutomationExecuteRequestMessage = BrowserAutomationExecuteRequest; +export type BrowserAutomationExecuteResponseMessage = BrowserAutomationExecuteResponse; export interface DaemonClientConfig { url: string; @@ -246,6 +252,7 @@ export interface DaemonClientConfig { }; runtimeMetricsIntervalMs?: number; runtimeMetricsWindowMs?: number; + capabilities?: Partial>; } export interface SendMessageOptions { @@ -3850,6 +3857,10 @@ export class DaemonClient { }); } + sendBrowserAutomationExecuteResponse(response: BrowserAutomationExecuteResponse): void { + this.sendSessionMessageStrict(response); + } + async readProjectConfig(repoRoot: string, requestId?: string): Promise { return this.sendCorrelatedSessionRequest({ requestId, @@ -4648,6 +4659,7 @@ export class DaemonClient { [CLIENT_CAPS.customModeIcons]: true, [CLIENT_CAPS.reasoningMergeEnum]: true, [CLIENT_CAPS.terminalReflowableSnapshot]: true, + ...this.config.capabilities, }, ...(this.config.appVersion ? { appVersion: this.config.appVersion } : {}), }), diff --git a/packages/client/src/index.test.ts b/packages/client/src/index.test.ts index 030fe64fb..3483956ca 100644 --- a/packages/client/src/index.test.ts +++ b/packages/client/src/index.test.ts @@ -674,6 +674,7 @@ test("config actions delegate to existing daemon config RPCs", async () => { config: { mcp: { injectIntoAgents: true }, providers: {}, + browserTools: { enabled: false }, metadataGeneration: { providers: [] }, autoArchiveAfterMerge: false, enableTerminalAgentHooks: false, @@ -728,6 +729,7 @@ test("config actions delegate to existing daemon config RPCs", async () => { enabled: false, }, }, + browserTools: { enabled: false }, metadataGeneration: { providers: [] }, autoArchiveAfterMerge: false, enableTerminalAgentHooks: false, diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index c7018eb3f..fe9cc1998 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -30,6 +30,8 @@ export { DaemonClient }; export type { DaemonClientConfig, DaemonEvent, + BrowserAutomationExecuteRequestMessage, + BrowserAutomationExecuteResponseMessage, WebSocketFactory, WebSocketLike, } from "./daemon-client.js"; diff --git a/packages/desktop/scripts/dev-runner.mjs b/packages/desktop/scripts/dev-runner.mjs index 08e27b4d2..51ca14130 100644 --- a/packages/desktop/scripts/dev-runner.mjs +++ b/packages/desktop/scripts/dev-runner.mjs @@ -19,7 +19,6 @@ if (!Number.isInteger(expoPort) || expoPort <= 0) { } const expoDevUrl = process.env.EXPO_DEV_URL || `http://localhost:${expoPort}`; -const daemonEndpoint = process.env.PASEO_DAEMON_ENDPOINT || process.env.PASEO_LISTEN || ""; const colorEnv = { FORCE_COLOR: process.env.FORCE_COLOR || "1", npm_config_color: process.env.npm_config_color || "always", @@ -164,7 +163,6 @@ spawnChild("metro", "npx", ["expo", "start", "--port", String(expoPort)], { BROWSER: "none", APP_VARIANT: "development", PASEO_WEB_PLATFORM: "electron", - EXPO_PUBLIC_LOCAL_DAEMON: daemonEndpoint, }, }); diff --git a/packages/desktop/src/daemon/cli/passthrough.test.ts b/packages/desktop/src/daemon/cli/passthrough.test.ts index da25d0c53..84c728e40 100644 --- a/packages/desktop/src/daemon/cli/passthrough.test.ts +++ b/packages/desktop/src/daemon/cli/passthrough.test.ts @@ -65,6 +65,16 @@ describe("passthrough CLI", () => { ).toBeNull(); }); + it("ignores Electron remote debugging switches", () => { + expect( + parsePassthroughCliArgs({ + argv: ["/usr/bin/Paseo", "--remote-debugging-port=9233"], + isDefaultApp: false, + forceCli: false, + }), + ).toBeNull(); + }); + it("preserves CLI flags for direct app invocations", () => { expect( parsePassthroughCliArgs({ diff --git a/packages/desktop/src/daemon/cli/passthrough.ts b/packages/desktop/src/daemon/cli/passthrough.ts index 4244263c9..17b3422c4 100644 --- a/packages/desktop/src/daemon/cli/passthrough.ts +++ b/packages/desktop/src/daemon/cli/passthrough.ts @@ -2,7 +2,7 @@ import { pathToFileURL } from "node:url"; import { resolvePassthroughCliEntrypoint } from "./entrypoints.js"; const DESKTOP_CLI_ENV = "PASEO_DESKTOP_CLI"; -const IGNORED_ARG_PREFIXES = ["-psn_", "--no-sandbox"]; +const IGNORED_ARG_PREFIXES = ["-psn_", "--no-sandbox", "--remote-debugging-port="]; export type PassthroughCliRunner = (argv: string[]) => Promise; diff --git a/packages/desktop/src/features/browser-automation/ipc.test.ts b/packages/desktop/src/features/browser-automation/ipc.test.ts new file mode 100644 index 000000000..81245483a --- /dev/null +++ b/packages/desktop/src/features/browser-automation/ipc.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; + +import { sanitizeDownloadFileName } from "./ipc.js"; + +describe("browser automation IPC", () => { + it("strips directories from agent-supplied download filenames", () => { + expect( + sanitizeDownloadFileName({ + url: "https://example.com/fallback.txt", + fileName: "../../.ssh/authorized_keys", + }), + ).toBe("authorized_keys"); + }); + + it("falls back to a safe filename when the URL has no basename", () => { + expect(sanitizeDownloadFileName({ url: "https://example.com/" })).toBe("download"); + }); +}); diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts new file mode 100644 index 000000000..92c716d46 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -0,0 +1,185 @@ +import { mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import type { WebContents } from "electron"; +import { ipcMain } from "electron"; +import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import type { + BrowserAutomationConsoleLogEntry, + BrowserAutomationCookieEntry, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import type { TabContents, BrowserRegistry } from "./service.js"; +import { executeAutomationCommand } from "./service.js"; +import { + listRegisteredPaseoBrowserIds, + listRegisteredPaseoBrowserIdsForWorkspace, + getPaseoBrowserWebContents, + getWorkspaceActivePaseoBrowserWebContents, + getWorkspaceActivePaseoBrowserId, + getAgentActivePaseoBrowserId, + getPaseoBrowserWorkspaceId, +} from "../browser-webviews/index.js"; + +const MAX_CONSOLE_MESSAGES_PER_TAB = 200; +const consoleMessagesByContentsId = new Map(); +const observedContentsIds = new Set(); + +interface IpcHandlerRegistry { + handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void; +} + +function adaptWebContents(contents: WebContents): TabContents { + observeConsoleMessages(contents); + return { + id: contents.id, + getURL: () => contents.getURL(), + getTitle: () => contents.getTitle(), + canGoBack: () => contents.canGoBack(), + canGoForward: () => contents.canGoForward(), + isLoading: () => contents.isLoading(), + isDestroyed: () => contents.isDestroyed(), + executeJavaScript: (code: string) => contents.executeJavaScript(code), + loadURL: (url: string) => contents.loadURL(url), + goBack: () => contents.goBack(), + goForward: () => contents.goForward(), + reload: () => contents.reload(), + capturePage: () => contents.capturePage(), + getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [], + getCookies: async (url: string) => + (await contents.session.cookies.get({ url })).map(normalizeCookie), + sendDebugCommand: async (command: string, params?: Record) => { + if (!contents.debugger.isAttached()) { + contents.debugger.attach("1.3"); + } + return contents.debugger.sendCommand(command, params ?? {}); + }, + printToPDF: async (options?: Record) => contents.printToPDF(options ?? {}), + downloadURL: (input) => downloadWithContents(contents, input), + }; +} + +function downloadWithContents( + contents: WebContents, + input: { url: string; fileName?: string }, +): Promise<{ filePath: string; totalBytes?: number; state: string }> { + const downloadDir = join(tmpdir(), "paseo-browser-downloads"); + mkdirSync(downloadDir, { recursive: true }); + const filePath = join(downloadDir, sanitizeDownloadFileName(input)); + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + contents.session.off("will-download", onDownload); + reject(new Error(`Timed out waiting for browser download: ${input.url}`)); + }, 30_000); + function onDownload(_event: Electron.Event, item: Electron.DownloadItem): void { + if (item.getURL() !== input.url) { + return; + } + clearTimeout(timeout); + contents.session.off("will-download", onDownload); + item.setSavePath(filePath); + item.once("done", (_doneEvent, state) => { + resolve({ filePath, totalBytes: item.getTotalBytes(), state }); + }); + } + contents.session.on("will-download", onDownload); + contents.downloadURL(input.url); + }); +} + +export function sanitizeDownloadFileName(input: { url: string; fileName?: string }): string { + const requestedName = input.fileName ?? basename(new URL(input.url).pathname); + return basename(requestedName) || "download"; +} + +function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry { + return { + name: cookie.name, + value: cookie.value, + ...(cookie.domain ? { domain: cookie.domain } : {}), + ...(cookie.path ? { path: cookie.path } : {}), + secure: cookie.secure, + httpOnly: cookie.httpOnly, + ...(typeof cookie.expirationDate === "number" ? { expirationDate: cookie.expirationDate } : {}), + }; +} + +function observeConsoleMessages(contents: WebContents): void { + if (observedContentsIds.has(contents.id)) { + return; + } + observedContentsIds.add(contents.id); + contents.on("console-message", (_event, level, message, line, sourceId) => { + const entry = normalizeConsoleMessage({ level, message, line, sourceId }); + const messages = consoleMessagesByContentsId.get(contents.id) ?? []; + messages.push(entry); + consoleMessagesByContentsId.set(contents.id, messages.slice(-MAX_CONSOLE_MESSAGES_PER_TAB)); + }); + contents.once("destroyed", () => { + observedContentsIds.delete(contents.id); + consoleMessagesByContentsId.delete(contents.id); + }); +} + +function normalizeConsoleMessage(input: { + level: unknown; + message: unknown; + line: unknown; + sourceId: unknown; +}): BrowserAutomationConsoleLogEntry { + return { + level: typeof input.level === "string" ? input.level : String(input.level ?? "log"), + message: typeof input.message === "string" ? input.message : String(input.message ?? ""), + ...(typeof input.sourceId === "string" && input.sourceId.length > 0 + ? { source: input.sourceId } + : {}), + ...(typeof input.line === "number" ? { line: input.line } : {}), + timestamp: Date.now(), + }; +} + +function createRegistry(): BrowserRegistry { + return { + listRegisteredBrowserIds: listRegisteredPaseoBrowserIds, + listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace, + getTabContents(browserId: string): TabContents | null { + const contents = getPaseoBrowserWebContents(browserId); + return contents ? adaptWebContents(contents) : null; + }, + getBrowserWorkspaceId: getPaseoBrowserWorkspaceId, + getWorkspaceActiveTabContents(workspaceId: string): TabContents | null { + const contents = getWorkspaceActivePaseoBrowserWebContents(workspaceId); + return contents ? adaptWebContents(contents) : null; + }, + getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId, + getAgentActiveBrowserId: getAgentActivePaseoBrowserId, + }; +} + +export function registerBrowserAutomationIpc(options?: { ipc?: IpcHandlerRegistry }): void { + const ipc = options?.ipc ?? ipcMain; + const registry = createRegistry(); + + ipc.handle("paseo:browser:execute-automation-command", async (_event, rawRequest: unknown) => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse(rawRequest); + if (!parsed.success) { + return { + requestId: readRequestId(rawRequest), + ok: false as const, + error: { + code: "browser_unsupported" as const, + message: `Invalid automation request: ${parsed.error.message}`, + retryable: false, + }, + }; + } + return executeAutomationCommand(parsed.data, registry); + }); +} + +function readRequestId(rawRequest: unknown): string { + if (typeof rawRequest !== "object" || rawRequest === null || Array.isArray(rawRequest)) { + return "unknown"; + } + const requestId = (rawRequest as Record).requestId; + return typeof requestId === "string" && requestId.length > 0 ? requestId : "unknown"; +} diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts new file mode 100644 index 000000000..641c4105f --- /dev/null +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -0,0 +1,1913 @@ +import { resolve as resolvePath } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { BrowserSnapshotEngine } from "./snapshot-engine.js"; +import type { TabContents, BrowserRegistry } from "./service.js"; +import { executeAutomationCommand } from "./service.js"; + +function fakeTab(overrides: Partial & { id: number }): TabContents { + return { + getURL: () => "https://example.com", + getTitle: () => "Example", + canGoBack: () => false, + canGoForward: () => false, + isLoading: () => false, + isDestroyed: () => false, + executeJavaScript: async () => "[]", + loadURL: async () => {}, + goBack: () => {}, + goForward: () => {}, + reload: () => {}, + capturePage: async () => ({ + toPNG: () => new Uint8Array([137, 80, 78, 71]), + getSize: () => ({ width: 10, height: 5 }), + }), + ...overrides, + }; +} + +function createRegistry(overrides: Partial = {}): BrowserRegistry { + return { + listRegisteredBrowserIds: () => [], + listRegisteredBrowserIdsForWorkspace: () => [], + getTabContents: () => null, + getBrowserWorkspaceId: () => null, + getWorkspaceActiveTabContents: () => null, + getWorkspaceActiveBrowserId: () => null, + getAgentActiveBrowserId: () => null, + ...overrides, + }; +} + +function hasScriptWith(scripts: string[], first: string, second: string): boolean { + for (const script of scripts) { + if (script.includes(first) && script.includes(second)) { + return true; + } + } + return false; +} + +const TAB_A = fakeTab({ id: 1, getURL: () => "https://a.com", getTitle: () => "Tab A" }); +const TAB_B = fakeTab({ id: 2, getURL: () => "https://b.com", getTitle: () => "Tab B" }); + +describe("executeAutomationCommand", () => { + describe("list_tabs", () => { + it("returns registered tabs with url/title/active data", () => { + const registry = createRegistry({ + listRegisteredBrowserIds: () => ["a", "b"], + getTabContents: (id) => { + if (id === "a") return TAB_A; + if (id === "b") return TAB_B; + return null; + }, + getBrowserWorkspaceId: (id) => (id === "a" || id === "b" ? "workspace-a" : null), + getWorkspaceActiveBrowserId: () => "a", + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r1", + command: { command: "list_tabs", args: {} }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r1", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: "a", + workspaceId: "workspace-a", + url: "https://a.com", + title: "Tab A", + isActive: false, + isLoading: false, + canGoBack: false, + canGoForward: false, + }, + { + browserId: "b", + workspaceId: "workspace-a", + url: "https://b.com", + title: "Tab B", + isActive: false, + isLoading: false, + canGoBack: false, + canGoForward: false, + }, + ], + }, + }); + }); + + it("skips destroyed tabs", () => { + const destroyedTab = fakeTab({ id: 3, isDestroyed: () => true }); + const registry = createRegistry({ + listRegisteredBrowserIds: () => ["a", "dead"], + getTabContents: (id) => { + if (id === "a") return TAB_A; + if (id === "dead") return destroyedTab; + return null; + }, + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r2", + command: { command: "list_tabs", args: {} }, + }, + registry, + ); + + expect(result.ok).toBe(true); + if (!result.ok) throw new Error("expected success"); + expect(result.result.command).toBe("list_tabs"); + expect(result.result.tabs).toHaveLength(1); + expect(result.result.tabs[0]?.browserId).toBe("a"); + }); + + it("returns empty list when no tabs registered", () => { + const registry = createRegistry(); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r3", + command: { command: "list_tabs", args: {} }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r3", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }); + }); + + it("lists only tabs owned by the requested workspace", () => { + const registry = createRegistry({ + listRegisteredBrowserIdsForWorkspace: (workspaceId) => { + if (workspaceId === "workspace-a") return ["a"]; + if (workspaceId === "workspace-b") return ["b"]; + return []; + }, + getTabContents: (id) => { + if (id === "a") return TAB_A; + if (id === "b") return TAB_B; + return null; + }, + getBrowserWorkspaceId: (id) => { + if (id === "a") return "workspace-a"; + if (id === "b") return "workspace-b"; + return null; + }, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : "b"), + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-workspace-list", + workspaceId: "workspace-a", + command: { command: "list_tabs", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(result.ok).toBe(true); + if (!result.ok) throw new Error("expected success"); + expect(result.result.command).toBe("list_tabs"); + expect(result.result.tabs).toHaveLength(1); + expect(result.result.tabs[0]?.browserId).toBe("a"); + expect(result.result.tabs[0]?.workspaceId).toBe("workspace-a"); + }); + }); + + describe("page_info", () => { + it("uses explicit browserId", () => { + const registry = createRegistry({ + getTabContents: (id) => (id === "b" ? TAB_B : null), + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r4", + browserId: "b", + command: { command: "page_info", args: { browserId: "b" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r4", + ok: true, + result: { + command: "page_info", + tab: { + browserId: "b", + url: "https://b.com", + title: "Tab B", + isActive: false, + isLoading: false, + canGoBack: false, + canGoForward: false, + }, + }, + }); + }); + + it("uses the top-level browserId when command args omit it", () => { + const registry = createRegistry({ + getTabContents: (id) => (id === "b" ? TAB_B : null), + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-top-level-browser-id", + browserId: "b", + command: { command: "page_info", args: {} }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-top-level-browser-id", + ok: true, + result: { + command: "page_info", + tab: { + browserId: "b", + url: "https://b.com", + title: "Tab B", + isActive: false, + isLoading: false, + canGoBack: false, + canGoForward: false, + }, + }, + }); + }); + + it("uses active workspace browser when browserId omitted", () => { + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? TAB_A : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r5", + workspaceId: "workspace-a", + command: { command: "page_info", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r5", + ok: true, + result: { + command: "page_info", + tab: { + browserId: "a", + workspaceId: "workspace-a", + url: "https://a.com", + title: "Tab A", + isActive: true, + isLoading: false, + canGoBack: false, + canGoForward: false, + }, + }, + }); + }); + + it("uses the agent active browser before the human-focused workspace browser", () => { + const registry = createRegistry({ + getTabContents: (id) => { + if (id === "agent-browser") return TAB_B; + if (id === "human-browser") return TAB_A; + return null; + }, + getBrowserWorkspaceId: (id) => + id === "agent-browser" || id === "human-browser" ? "workspace-a" : null, + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? TAB_A : null, + getWorkspaceActiveBrowserId: (workspaceId) => + workspaceId === "workspace-a" ? "human-browser" : null, + getAgentActiveBrowserId: (agentId) => (agentId === "agent-1" ? "agent-browser" : null), + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-agent-active", + agentId: "agent-1", + workspaceId: "workspace-a", + command: { command: "page_info", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-agent-active", + ok: true, + result: { + command: "page_info", + tab: { + browserId: "agent-browser", + workspaceId: "workspace-a", + url: "https://b.com", + title: "Tab B", + isActive: false, + isLoading: false, + canGoBack: false, + canGoForward: false, + }, + }, + }); + }); + + it("returns canGoBack/canGoForward when available", () => { + const tabWithNav = fakeTab({ + id: 10, + canGoBack: () => true, + canGoForward: () => true, + }); + const registry = createRegistry({ + getTabContents: () => tabWithNav, + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-nav", + browserId: "x", + command: { command: "page_info", args: { browserId: "x" } }, + }, + registry, + ); + + expect(result.ok).toBe(true); + if (!result.ok) throw new Error("expected success"); + expect(result.result.command).toBe("page_info"); + expect(result.result.tab.canGoBack).toBe(true); + expect(result.result.tab.canGoForward).toBe(true); + }); + + it("returns browser_no_tab when no active workspace browser", () => { + const registry = createRegistry({ + getWorkspaceActiveTabContents: () => null, + getWorkspaceActiveBrowserId: () => null, + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r6", + command: { command: "page_info", args: {} }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r6", + ok: false, + error: { + code: "browser_no_tab", + message: "No active browser tab in workspace", + retryable: false, + }, + }); + }); + + it("returns browser_tab_not_found for missing explicit browserId", () => { + const registry = createRegistry({ getTabContents: () => null }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r7", + browserId: "missing", + command: { command: "page_info", args: { browserId: "missing" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r7", + ok: false, + error: { + code: "browser_tab_not_found", + message: "No browser tab found for ID: missing", + retryable: false, + }, + }); + }); + + it("returns browser_tab_not_found when explicit browserId belongs to another workspace", () => { + const registry = createRegistry({ + getTabContents: (id) => (id === "b" ? TAB_B : null), + getBrowserWorkspaceId: (id) => (id === "b" ? "workspace-b" : null), + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-cross-workspace-browser-id", + workspaceId: "workspace-a", + browserId: "b", + command: { + command: "page_info", + args: { workspaceId: "workspace-a", browserId: "b" }, + }, + }, + registry, + ); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected failure"); + expect(result.error.code).toBe("browser_tab_not_found"); + }); + + it("returns browser_tab_closed for destroyed tab", () => { + const destroyedTab = fakeTab({ id: 99, isDestroyed: () => true }); + const registry = createRegistry({ + getTabContents: () => destroyedTab, + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r8", + browserId: "dead", + command: { command: "page_info", args: { browserId: "dead" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r8", + ok: false, + error: { + code: "browser_tab_closed", + message: "Browser tab dead has been closed", + retryable: false, + }, + }); + }); + + it("returns browser_tab_closed for destroyed active workspace tab", () => { + const destroyedTab = fakeTab({ id: 99, isDestroyed: () => true }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? destroyedTab : null, + getWorkspaceActiveBrowserId: (workspaceId) => + workspaceId === "workspace-a" ? "dead" : null, + }); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-active-dead", + workspaceId: "workspace-a", + command: { command: "page_info", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected failure"); + expect(result.error.code).toBe("browser_tab_closed"); + }); + }); + + describe("set_background", () => { + it("sets the active tab page background color", async () => { + const scripts: string[] = []; + const tab = fakeTab({ + id: 99, + executeJavaScript: async (script) => { + scripts.push(script); + return true; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: () => tab, + getWorkspaceActiveBrowserId: () => "browser-1", + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-bg", + workspaceId: "workspace-a", + command: { + command: "set_background", + args: { workspaceId: "workspace-a", color: "red" }, + }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-bg", + ok: true, + result: { command: "set_background", browserId: "browser-1", color: "red" }, + }); + expect(hasScriptWith(scripts, "document.body.style.background", "red")).toBe(true); + }); + }); + + describe("snapshot", () => { + it("returns snapshot refs for the active workspace browser", async () => { + const tab = fakeTab({ + id: 5, + getURL: () => "https://example.com/form", + getTitle: () => "Fixture", + executeJavaScript: async () => + JSON.stringify([ + { + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + { + role: "button", + tagName: "button", + text: "Greet", + selector: "button", + attributes: {}, + }, + ]), + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-snapshot", + workspaceId: "workspace-a", + command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-snapshot", + ok: true, + result: { + command: "snapshot", + browserId: "a", + workspaceId: "workspace-a", + url: "https://example.com/form", + title: "Fixture", + elements: [ + { + ref: "@e1", + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + { + ref: "@e2", + role: "button", + tagName: "button", + text: "Greet", + selector: "button", + attributes: {}, + }, + ], + }, + }); + }); + }); + + describe("click and fill", () => { + it("fills and clicks refs from the latest snapshot", async () => { + const executedScripts: string[] = []; + const tab = fakeTab({ + id: 6, + getURL: () => "https://example.com/form", + getTitle: () => "Fixture", + executeJavaScript: async (script) => { + executedScripts.push(script); + if (script.includes("CANDIDATE_SELECTOR")) { + return JSON.stringify([ + { + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + { + role: "button", + tagName: "button", + text: "Greet", + selector: "#greet", + attributes: { id: "greet" }, + }, + ]); + } + return true; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-snapshot-for-actions", + workspaceId: "workspace-a", + command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + const fillResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-fill", + workspaceId: "workspace-a", + command: { + command: "fill", + args: { workspaceId: "workspace-a", ref: "@e1", value: "Ada" }, + }, + }, + registry, + ); + const clickResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-click", + workspaceId: "workspace-a", + command: { command: "click", args: { workspaceId: "workspace-a", ref: "@e2" } }, + }, + registry, + ); + + expect(fillResult).toEqual({ + requestId: "r-fill", + ok: true, + result: { command: "fill", browserId: "a", ref: "@e1" }, + }); + expect(clickResult).toEqual({ + requestId: "r-click", + ok: true, + result: { command: "click", browserId: "a", ref: "@e2" }, + }); + let filledName = false; + let clickedGreet = false; + for (const script of executedScripts) { + filledName ||= script.includes("#name") && script.includes("Ada"); + clickedGreet ||= script.includes("#greet") && script.includes("click"); + } + expect(filledName).toBe(true); + expect(clickedGreet).toBe(true); + }); + + it("returns browser_stale_ref when the page has navigated since the snapshot", async () => { + let currentUrl = "https://example.com/form"; + const tab = fakeTab({ + id: 7, + getURL: () => currentUrl, + executeJavaScript: async () => + JSON.stringify([ + { + role: "button", + tagName: "button", + text: "Greet", + selector: "#greet", + attributes: { id: "greet" }, + }, + ]), + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-snapshot-before-nav", + workspaceId: "workspace-a", + command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + currentUrl = "https://example.com/next"; + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-stale-click", + workspaceId: "workspace-a", + command: { command: "click", args: { workspaceId: "workspace-a", ref: "@e1" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-stale-click", + ok: false, + error: { + code: "browser_stale_ref", + message: "Browser element reference @e1 is stale. Take a new snapshot and try again.", + retryable: false, + }, + }); + }); + + it("returns browser_stale_ref when a same-URL DOM change removes the ref", async () => { + const tab = fakeTab({ + id: 26, + getURL: () => "https://example.com/form", + executeJavaScript: async (script) => { + if (script.includes("CANDIDATE_SELECTOR")) { + return JSON.stringify([ + { + role: "button", + tagName: "button", + text: "Greet", + selector: "#greet", + attributes: { id: "greet" }, + }, + ]); + } + return false; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-snapshot-before-dom-removal", + workspaceId: "workspace-a", + command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-dom-removed-click", + workspaceId: "workspace-a", + command: { command: "click", args: { workspaceId: "workspace-a", ref: "@e1" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-dom-removed-click", + ok: false, + error: { + code: "browser_stale_ref", + message: "Browser element reference @e1 is stale. Take a new snapshot and try again.", + retryable: false, + }, + }); + }); + + it("focuses, clears, checks, and selects snapshot refs", async () => { + const executedScripts: string[] = []; + const tab = fakeTab({ + id: 17, + getURL: () => "https://example.com/controls", + executeJavaScript: async (script) => { + executedScripts.push(script); + if (script.includes("CANDIDATE_SELECTOR")) { + return JSON.stringify([ + { + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + { + role: "checkbox", + tagName: "input", + text: "Subscribe", + selector: "#subscribe", + attributes: { id: "subscribe", type: "checkbox" }, + }, + { + role: "combobox", + tagName: "select", + text: "Country", + selector: "#country", + attributes: { id: "country" }, + }, + { + role: "button", + tagName: "button", + text: "Preview", + selector: "#preview", + attributes: { id: "preview" }, + }, + { + role: "generic", + tagName: "div", + text: "Drop zone", + selector: "#drop-zone", + attributes: { id: "drop-zone", tabindex: "0" }, + }, + ]); + } + return true; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-snapshot-for-controls", + workspaceId: "workspace-a", + command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + const focusResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-focus", + workspaceId: "workspace-a", + command: { command: "focus", args: { workspaceId: "workspace-a", ref: "@e1" } }, + }, + registry, + ); + const clearResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-clear", + workspaceId: "workspace-a", + command: { command: "clear", args: { workspaceId: "workspace-a", ref: "@e1" } }, + }, + registry, + ); + const checkResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-check", + workspaceId: "workspace-a", + command: { + command: "check", + args: { workspaceId: "workspace-a", ref: "@e2", checked: true }, + }, + }, + registry, + ); + const selectResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-select", + workspaceId: "workspace-a", + command: { + command: "select", + args: { workspaceId: "workspace-a", ref: "@e3", value: "us" }, + }, + }, + registry, + ); + const hoverResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-hover", + workspaceId: "workspace-a", + command: { command: "hover", args: { workspaceId: "workspace-a", ref: "@e4" } }, + }, + registry, + ); + const dragResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-drag", + workspaceId: "workspace-a", + command: { + command: "drag", + args: { workspaceId: "workspace-a", sourceRef: "@e4", targetRef: "@e5" }, + }, + }, + registry, + ); + + expect(focusResult).toEqual({ + requestId: "r-focus", + ok: true, + result: { command: "focus", browserId: "a", ref: "@e1" }, + }); + expect(clearResult).toEqual({ + requestId: "r-clear", + ok: true, + result: { command: "clear", browserId: "a", ref: "@e1" }, + }); + expect(checkResult).toEqual({ + requestId: "r-check", + ok: true, + result: { command: "check", browserId: "a", ref: "@e2", checked: true }, + }); + expect(selectResult).toEqual({ + requestId: "r-select", + ok: true, + result: { command: "select", browserId: "a", ref: "@e3", value: "us" }, + }); + expect(hoverResult).toEqual({ + requestId: "r-hover", + ok: true, + result: { command: "hover", browserId: "a", ref: "@e4" }, + }); + expect(dragResult).toEqual({ + requestId: "r-drag", + ok: true, + result: { command: "drag", browserId: "a", sourceRef: "@e4", targetRef: "@e5" }, + }); + expect(hasScriptWith(executedScripts, "#name", "focus")).toBe(true); + expect(hasScriptWith(executedScripts, "#name", "deleteContent")).toBe(true); + expect(hasScriptWith(executedScripts, "#subscribe", "checked")).toBe(true); + expect(hasScriptWith(executedScripts, "#country", "us")).toBe(true); + expect(hasScriptWith(executedScripts, "#preview", "mouseover")).toBe(true); + expect(hasScriptWith(executedScripts, "#drop-zone", "dragover")).toBe(true); + }); + }); + + describe("wait", () => { + it("waits until page text appears", async () => { + let reads = 0; + const tab = fakeTab({ + id: 8, + getURL: () => "https://example.com/wait", + executeJavaScript: async () => { + reads += 1; + return reads >= 2 ? "Loading\nReady" : "Loading"; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-wait-text", + workspaceId: "workspace-a", + command: { + command: "wait", + args: { workspaceId: "workspace-a", text: "Ready", timeoutMs: 100 }, + }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-wait-text", + ok: true, + result: { command: "wait", browserId: "a", matched: "text" }, + }); + }); + + it("returns browser_timeout when waited-for text does not appear", async () => { + const tab = fakeTab({ + id: 9, + getURL: () => "https://example.com/wait", + executeJavaScript: async () => "Loading", + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-wait-timeout", + workspaceId: "workspace-a", + command: { + command: "wait", + args: { workspaceId: "workspace-a", text: "Ready", timeoutMs: 1 }, + }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-wait-timeout", + ok: false, + error: { + code: "browser_timeout", + message: "Timed out waiting for browser text: Ready", + retryable: true, + }, + }); + }); + }); + + describe("type and keypress", () => { + it("types text into a snapshot ref and dispatches keypress", async () => { + const executedScripts: string[] = []; + const tab = fakeTab({ + id: 10, + getURL: () => "https://example.com/type", + executeJavaScript: async (script) => { + executedScripts.push(script); + if (script.includes("CANDIDATE_SELECTOR")) { + return JSON.stringify([ + { + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name" }, + }, + ]); + } + return true; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-snapshot-for-type", + workspaceId: "workspace-a", + command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + const typeResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-type", + workspaceId: "workspace-a", + command: { + command: "type", + args: { workspaceId: "workspace-a", ref: "@e1", text: "Ada" }, + }, + }, + registry, + ); + const keyResult = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-keypress", + workspaceId: "workspace-a", + command: { + command: "keypress", + args: { workspaceId: "workspace-a", ref: "@e1", key: "Enter" }, + }, + }, + registry, + ); + + expect(typeResult).toEqual({ + requestId: "r-type", + ok: true, + result: { command: "type", browserId: "a", ref: "@e1" }, + }); + expect(keyResult).toEqual({ + requestId: "r-keypress", + ok: true, + result: { command: "keypress", browserId: "a", key: "Enter", ref: "@e1" }, + }); + let typedAda = false; + let pressedEnter = false; + for (const script of executedScripts) { + typedAda ||= script.includes("#name") && script.includes("Ada"); + pressedEnter ||= script.includes("#name") && script.includes("Enter"); + } + expect(typedAda).toBe(true); + expect(pressedEnter).toBe(true); + }); + }); + + describe("logs", () => { + it("returns recent console messages and network performance entries", async () => { + const tab = fakeTab({ + id: 19, + getConsoleMessages: () => [ + { level: "info", message: "first", timestamp: 1 }, + { level: "error", message: "second", source: "fixture", line: 7, timestamp: 2 }, + ], + executeJavaScript: async () => + JSON.stringify([ + { + url: "https://example.com/app.js", + type: "script", + startTime: 3, + duration: 4, + transferSize: 100, + }, + ]), + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-logs", + workspaceId: "workspace-a", + command: { command: "logs", args: { workspaceId: "workspace-a", maxEntries: 1 } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-logs", + ok: true, + result: { + command: "logs", + browserId: "a", + console: [ + { level: "error", message: "second", source: "fixture", line: 7, timestamp: 2 }, + ], + network: [ + { + url: "https://example.com/app.js", + type: "script", + startTime: 3, + duration: 4, + transferSize: 100, + }, + ], + }, + }); + }); + }); + + describe("storage", () => { + it("returns cookies, localStorage, and sessionStorage for the target tab", async () => { + const tab = fakeTab({ + id: 20, + getURL: () => "https://example.com/storage", + getCookies: async () => [ + { name: "theme", value: "dark", domain: "example.com", httpOnly: true }, + ], + executeJavaScript: async () => + JSON.stringify({ + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }), + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-storage", + workspaceId: "workspace-a", + command: { command: "storage", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-storage", + ok: true, + result: { + command: "storage", + browserId: "a", + url: "https://example.com/storage", + cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }, + }); + }); + }); + + describe("environment", () => { + it("sets viewport and geolocation for the target tab", async () => { + const debugCommands: Array<{ command: string; params?: Record }> = []; + const scripts: string[] = []; + const tab = fakeTab({ + id: 21, + sendDebugCommand: async (command, params) => { + debugCommands.push({ command, params }); + }, + executeJavaScript: async (script) => { + scripts.push(script); + if (script.includes("window.innerWidth")) { + return JSON.stringify({ width: 390, height: 844, deviceScaleFactor: 3 }); + } + return true; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-environment", + workspaceId: "workspace-a", + command: { + command: "environment", + args: { + workspaceId: "workspace-a", + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + }, + registry, + ); + + expect(debugCommands).toEqual([ + { + command: "Emulation.setDeviceMetricsOverride", + params: { width: 390, height: 844, deviceScaleFactor: 3, mobile: false }, + }, + { + command: "Emulation.setGeolocationOverride", + params: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + ]); + expect(hasScriptWith(scripts, "navigator", "geolocation")).toBe(true); + expect(result).toEqual({ + requestId: "r-environment", + ok: true, + result: { + command: "environment", + browserId: "a", + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }); + }); + }); + + describe("full-page screenshot and PDF", () => { + it("captures a full-page screenshot through CDP", async () => { + const debugCommands: Array<{ command: string; params?: Record }> = []; + const tab = fakeTab({ + id: 22, + sendDebugCommand: async (command, params) => { + debugCommands.push({ command, params }); + if (command === "Page.getLayoutMetrics") { + return { contentSize: { width: 390.2, height: 1200.1 } }; + } + return { data: "iVBORw0KGgo=" }; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-full-page", + workspaceId: "workspace-a", + command: { command: "full_page_screenshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(debugCommands).toEqual([ + { command: "Page.getLayoutMetrics", params: undefined }, + { + command: "Page.captureScreenshot", + params: { + format: "png", + captureBeyondViewport: true, + clip: { x: 0, y: 0, width: 391, height: 1201, scale: 1 }, + }, + }, + ]); + expect(result).toEqual({ + requestId: "r-full-page", + ok: true, + result: { + command: "full_page_screenshot", + browserId: "a", + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 391, + height: 1201, + }, + }); + }); + + it("exports the target tab as PDF", async () => { + const printOptions: Record[] = []; + const tab = fakeTab({ + id: 23, + printToPDF: async (options) => { + printOptions.push(options ?? {}); + return new Uint8Array([0x25, 0x50, 0x44, 0x46]); + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-pdf", + workspaceId: "workspace-a", + command: { + command: "pdf", + args: { workspaceId: "workspace-a", landscape: true, printBackground: false }, + }, + }, + registry, + ); + + expect(printOptions).toEqual([{ printBackground: false, landscape: true }]); + expect(result).toEqual({ + requestId: "r-pdf", + ok: true, + result: { + command: "pdf", + browserId: "a", + mimeType: "application/pdf", + dataBase64: "JVBERg==", + }, + }); + }); + }); + + describe("download and upload", () => { + it("downloads a URL through the target tab", async () => { + const tab = fakeTab({ + id: 24, + downloadURL: async (input) => ({ + filePath: `/tmp/${input.fileName ?? "download"}`, + totalBytes: 5, + state: "completed", + }), + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-download", + workspaceId: "workspace-a", + command: { + command: "download", + args: { + workspaceId: "workspace-a", + url: "https://example.com/file.txt", + fileName: "file.txt", + }, + }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-download", + ok: true, + result: { + command: "download", + browserId: "a", + url: "https://example.com/file.txt", + filePath: "/tmp/file.txt", + totalBytes: 5, + state: "completed", + }, + }); + }); + + it("rejects downloads for non-http URLs", async () => { + const downloadedUrls: string[] = []; + const tab = fakeTab({ + id: 27, + downloadURL: async (input) => { + downloadedUrls.push(input.url); + return { filePath: "/tmp/download", state: "completed" }; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-download-file", + workspaceId: "workspace-a", + command: { + command: "download", + args: { workspaceId: "workspace-a", url: "file:///tmp/secret.txt" }, + }, + }, + registry, + ); + + expect(downloadedUrls).toEqual([]); + expect(result).toEqual({ + requestId: "r-download-file", + ok: false, + error: { + code: "browser_denied", + message: "Browser download only supports http and https URLs.", + retryable: false, + }, + }); + }); + + it("sets workspace files on a file input ref through CDP", async () => { + const debugCommands: Array<{ command: string; params?: Record }> = []; + const workspaceRoot = "/tmp/paseo-workspace-a"; + const tab = fakeTab({ + id: 25, + executeJavaScript: async () => + JSON.stringify([ + { + role: "textbox", + tagName: "input", + text: "", + selector: "#file", + attributes: { id: "file", type: "file" }, + }, + ]), + sendDebugCommand: async (command, params) => { + debugCommands.push({ command, params }); + if (command === "DOM.getDocument") { + return { root: { nodeId: 1 } }; + } + if (command === "DOM.querySelector") { + return { nodeId: 2 }; + } + return {}; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === workspaceRoot ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === workspaceRoot ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? workspaceRoot : null), + }); + const snapshotEngine = new BrowserSnapshotEngine(); + const snapshot = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-snapshot-upload", + workspaceId: workspaceRoot, + command: { command: "snapshot", args: { workspaceId: workspaceRoot } }, + }, + registry, + { snapshotEngine }, + ); + if (!snapshot.ok || snapshot.result.command !== "snapshot") { + throw new Error("snapshot failed"); + } + const ref = snapshot.result.elements[0]?.ref; + if (!ref) { + throw new Error("missing upload ref"); + } + const uploadPath = resolvePath(workspaceRoot, "uploads/file.txt"); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-upload", + workspaceId: workspaceRoot, + command: { + command: "upload", + args: { workspaceId: workspaceRoot, ref, filePaths: ["uploads/file.txt"] }, + }, + }, + registry, + { snapshotEngine }, + ); + + expect(debugCommands.at(-1)).toEqual({ + command: "DOM.setFileInputFiles", + params: { nodeId: 2, files: [uploadPath] }, + }); + expect(result).toEqual({ + requestId: "r-upload", + ok: true, + result: { + command: "upload", + browserId: "a", + ref, + filePaths: [uploadPath], + }, + }); + }); + + it("rejects upload paths outside the workspace", async () => { + const workspaceRoot = "/tmp/paseo-workspace-a"; + const debugCommands: Array<{ command: string; params?: Record }> = []; + const tab = fakeTab({ + id: 26, + executeJavaScript: async () => + JSON.stringify([ + { + role: "textbox", + tagName: "input", + text: "", + selector: "#file", + attributes: { id: "file", type: "file" }, + }, + ]), + sendDebugCommand: async (command, params) => { + debugCommands.push({ command, params }); + if (command === "DOM.getDocument") { + return { root: { nodeId: 1 } }; + } + if (command === "DOM.querySelector") { + return { nodeId: 2 }; + } + return {}; + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === workspaceRoot ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === workspaceRoot ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? workspaceRoot : null), + }); + const snapshotEngine = new BrowserSnapshotEngine(); + const snapshot = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-snapshot-upload-outside", + workspaceId: workspaceRoot, + command: { command: "snapshot", args: { workspaceId: workspaceRoot } }, + }, + registry, + { snapshotEngine }, + ); + if (!snapshot.ok || snapshot.result.command !== "snapshot") { + throw new Error("snapshot failed"); + } + const ref = snapshot.result.elements[0]?.ref; + if (!ref) { + throw new Error("missing upload ref"); + } + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-upload-outside", + workspaceId: workspaceRoot, + command: { + command: "upload", + args: { workspaceId: workspaceRoot, ref, filePaths: ["../secret.txt"] }, + }, + }, + registry, + { snapshotEngine }, + ); + + expect(debugCommands).not.toContainEqual( + expect.objectContaining({ command: "DOM.setFileInputFiles" }), + ); + expect(result).toEqual({ + requestId: "r-upload-outside", + ok: false, + error: { + code: "browser_unsupported", + message: "browser_upload only accepts files inside the agent workspace.", + retryable: false, + }, + }); + }); + }); + + describe("navigation", () => { + it("navigates the active workspace browser to a URL", async () => { + const navigatedUrls: string[] = []; + const tab = fakeTab({ + id: 11, + loadURL: async (url) => { + navigatedUrls.push(url); + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-navigate", + workspaceId: "workspace-a", + command: { + command: "navigate", + args: { workspaceId: "workspace-a", url: "https://example.com/next" }, + }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-navigate", + ok: true, + result: { command: "navigate", browserId: "a", url: "https://example.com/next" }, + }); + expect(navigatedUrls).toEqual(["https://example.com/next"]); + }); + + it("rejects navigation to local files and script URLs", async () => { + const navigatedUrls: string[] = []; + const tab = fakeTab({ + id: 13, + loadURL: async (url) => { + navigatedUrls.push(url); + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + for (const [requestId, url] of [ + ["r-navigate-file", "file:///tmp/secret.txt"], + ["r-navigate-js", "javascript:alert(1)"], + ] as const) { + await expect( + executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId, + workspaceId: "workspace-a", + command: { command: "navigate", args: { workspaceId: "workspace-a", url } }, + }, + registry, + ), + ).resolves.toEqual({ + requestId, + ok: false, + error: { + code: "browser_denied", + message: "Browser navigation only supports http and https URLs.", + retryable: false, + }, + }); + } + + expect(navigatedUrls).toEqual([]); + }); + + it("dispatches back, forward, and reload to the active browser", async () => { + const actions: string[] = []; + const tab = fakeTab({ + id: 12, + goBack: () => actions.push("back"), + goForward: () => actions.push("forward"), + reload: () => actions.push("reload"), + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const back = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-back", + workspaceId: "workspace-a", + command: { command: "back", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + const forward = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-forward", + workspaceId: "workspace-a", + command: { command: "forward", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + const reload = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-reload", + workspaceId: "workspace-a", + command: { command: "reload", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(back).toEqual({ + requestId: "r-back", + ok: true, + result: { command: "back", browserId: "a" }, + }); + expect(forward).toEqual({ + requestId: "r-forward", + ok: true, + result: { command: "forward", browserId: "a" }, + }); + expect(reload).toEqual({ + requestId: "r-reload", + ok: true, + result: { command: "reload", browserId: "a" }, + }); + expect(actions).toEqual(["back", "forward", "reload"]); + }); + }); + + describe("screenshot", () => { + it("captures a viewport screenshot through CDP when available", async () => { + let captureParams: Record | undefined; + const tab = fakeTab({ + id: 13, + executeJavaScript: async () => ({ width: 321, height: 123 }), + sendDebugCommand: async (command, params) => { + if (command === "Page.captureScreenshot") { + captureParams = params; + return { data: "iVBORw0KGgo=" }; + } + throw new Error(`Unexpected CDP command ${command}`); + }, + capturePage: async () => { + throw new Error("capturePage should not be used when CDP is available"); + }, + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-screenshot-cdp", + workspaceId: "workspace-a", + command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-screenshot-cdp", + ok: true, + result: { + command: "screenshot", + browserId: "a", + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 321, + height: 123, + }, + }); + expect(captureParams).toEqual({ format: "png", fromSurface: false }); + }); + + it("captures a PNG screenshot from the active browser", async () => { + const tab = fakeTab({ + id: 13, + capturePage: async () => ({ + toPNG: () => new Uint8Array([137, 80, 78, 71, 1, 2, 3]), + getSize: () => ({ width: 640, height: 480 }), + }), + }); + const registry = createRegistry({ + getWorkspaceActiveTabContents: (workspaceId) => + workspaceId === "workspace-a" ? tab : null, + getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), + getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), + }); + + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r-screenshot", + workspaceId: "workspace-a", + command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r-screenshot", + ok: true, + result: { + command: "screenshot", + browserId: "a", + mimeType: "image/png", + dataBase64: "iVBORwECAw==", + width: 640, + height: 480, + }, + }); + }); + }); + + describe("unsupported command", () => { + it("returns browser_unsupported for unknown commands", () => { + const registry = createRegistry(); + + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "r9", + // Cast to bypass discriminated union — tests forward-compat fallback + command: { command: "future_click", args: {} } as never, + }, + registry, + ); + + expect(result).toEqual({ + requestId: "r9", + ok: false, + error: { + code: "browser_unsupported", + message: "Unsupported command: future_click", + retryable: false, + }, + }); + }); + }); +}); diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts new file mode 100644 index 000000000..97b10afcb --- /dev/null +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -0,0 +1,1562 @@ +import { isAbsolute, relative, resolve as resolvePath } from "node:path"; + +import type { + BrowserAutomationCommand, + BrowserAutomationConsoleLogEntry, + BrowserAutomationCookieEntry, + BrowserAutomationErrorCode, + BrowserAutomationExecuteResponse, + BrowserAutomationExecuteRequest, + BrowserAutomationNetworkLogEntry, + BrowserAutomationStorageEntry, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import { BrowserSnapshotEngine } from "./snapshot-engine.js"; + +export interface TabContents { + readonly id: number; + getURL(): string; + getTitle(): string; + canGoBack(): boolean; + canGoForward(): boolean; + isLoading(): boolean; + isDestroyed(): boolean; + executeJavaScript(code: string): Promise; + loadURL(url: string): Promise; + goBack(): void; + goForward(): void; + reload(): void; + capturePage(): Promise; + getConsoleMessages?(): BrowserAutomationConsoleLogEntry[]; + getCookies?(url: string): Promise; + sendDebugCommand?(command: string, params?: Record): Promise; + printToPDF?(options?: Record): Promise; + downloadURL?(input: { url: string; fileName?: string }): Promise<{ + filePath: string; + totalBytes?: number; + state: string; + }>; +} + +export interface TabImage { + toPNG(): Uint8Array; + getSize(): { width: number; height: number }; +} + +export interface BrowserRegistry { + listRegisteredBrowserIds(): string[]; + listRegisteredBrowserIdsForWorkspace(workspaceId: string): string[]; + getTabContents(browserId: string): TabContents | null; + getBrowserWorkspaceId(browserId: string): string | null; + getWorkspaceActiveTabContents(workspaceId: string): TabContents | null; + getWorkspaceActiveBrowserId(workspaceId: string): string | null; + getAgentActiveBrowserId(agentId: string): string | null; +} + +export type AutomationCommandPayload = BrowserAutomationExecuteResponse["payload"]; +type FailurePayload = Extract; + +const defaultSnapshotEngine = new BrowserSnapshotEngine(); +const DEFAULT_WAIT_TIMEOUT_MS = 5_000; +const WAIT_POLL_INTERVAL_MS = 25; +const ALLOWED_PAGE_URL_PROTOCOLS = new Set(["http:", "https:"]); + +function fail( + requestId: string, + code: BrowserAutomationErrorCode, + message: string, + retryable = false, +): FailurePayload { + return { requestId, ok: false, error: { code, message, retryable } }; +} + +function tabInfoFromContents( + browserId: string, + contents: TabContents, + activeBrowserId: string | null, + workspaceId: string | null, +) { + return { + browserId, + ...(workspaceId ? { workspaceId } : {}), + url: contents.getURL(), + title: contents.getTitle(), + isActive: activeBrowserId === browserId, + isLoading: contents.isLoading(), + canGoBack: contents.canGoBack(), + canGoForward: contents.canGoForward(), + }; +} + +export function executeAutomationCommand( + rawRequest: BrowserAutomationExecuteRequest, + registry: BrowserRegistry, + options?: { snapshotEngine?: BrowserSnapshotEngine }, +): AutomationCommandPayload | Promise { + const request = resolveAgentBrowserTarget(rawRequest, registry); + const { requestId, command } = request; + const workspaceId = request.workspaceId ?? command.args.workspaceId; + const snapshotEngine = options?.snapshotEngine ?? defaultSnapshotEngine; + const handler = commandHandlers[command.command]; + + if (!handler) { + return fail( + requestId, + "browser_unsupported", + `Unsupported command: ${(command as { command: string }).command}`, + ); + } + + return handler({ request, command, requestId, workspaceId, registry, snapshotEngine }); +} + +function resolveAgentBrowserTarget( + request: BrowserAutomationExecuteRequest, + registry: BrowserRegistry, +): BrowserAutomationExecuteRequest { + if (request.browserId || readCommandBrowserId(request.command)) { + return request; + } + if (!request.agentId) { + return request; + } + + const agentBrowserId = registry.getAgentActiveBrowserId(request.agentId); + if (!agentBrowserId) { + return request; + } + + return { ...request, browserId: agentBrowserId }; +} + +function readCommandBrowserId(command: BrowserAutomationCommand): string | undefined { + const args = command.args as { browserId?: unknown }; + return typeof args.browserId === "string" && args.browserId.length > 0 + ? args.browserId + : undefined; +} + +interface CommandHandlerContext { + request: BrowserAutomationExecuteRequest; + command: BrowserAutomationCommand; + requestId: string; + workspaceId: string | undefined; + registry: BrowserRegistry; + snapshotEngine: BrowserSnapshotEngine; +} + +type CommandHandler = ( + context: CommandHandlerContext, +) => AutomationCommandPayload | Promise; + +const commandHandlers: Partial> = { + list_tabs: ({ requestId, workspaceId, registry }) => + executeListTabs(requestId, workspaceId, registry), + page_info: ({ request, command, requestId, workspaceId, registry }) => { + const pageInfoCommand = command as Extract; + return executePageInfo( + requestId, + workspaceId, + pageInfoCommand.args.browserId ?? request.browserId, + registry, + ); + }, + snapshot: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const snapshotCommand = command as Extract; + return executeSnapshot( + requestId, + workspaceId, + snapshotCommand.args.browserId ?? request.browserId, + registry, + snapshotEngine, + ); + }, + click: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const clickCommand = command as Extract; + return executeClick( + requestId, + workspaceId, + clickCommand.args.browserId ?? request.browserId, + clickCommand.args.ref, + registry, + snapshotEngine, + ); + }, + fill: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const fillCommand = command as Extract; + return executeFill( + requestId, + workspaceId, + fillCommand.args.browserId ?? request.browserId, + fillCommand.args.ref, + fillCommand.args.value, + registry, + snapshotEngine, + ); + }, + wait: ({ request, command, requestId, workspaceId, registry }) => { + const waitCommand = command as Extract; + return executeWait( + requestId, + workspaceId, + waitCommand.args.browserId ?? request.browserId, + { + text: waitCommand.args.text, + url: waitCommand.args.url, + timeoutMs: waitCommand.args.timeoutMs, + }, + registry, + ); + }, + type: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const typeCommand = command as Extract; + return executeType( + requestId, + workspaceId, + typeCommand.args.browserId ?? request.browserId, + typeCommand.args.ref, + typeCommand.args.text, + registry, + snapshotEngine, + ); + }, + keypress: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const keypressCommand = command as Extract; + return executeKeypress( + requestId, + workspaceId, + keypressCommand.args.browserId ?? request.browserId, + keypressCommand.args.ref, + keypressCommand.args.key, + registry, + snapshotEngine, + ); + }, + navigate: ({ request, command, requestId, workspaceId, registry }) => { + const navigateCommand = command as Extract; + return executeNavigate( + requestId, + workspaceId, + navigateCommand.args.browserId ?? request.browserId, + navigateCommand.args.url, + registry, + ); + }, + back: ({ request, command, requestId, workspaceId, registry }) => { + const backCommand = command as Extract; + return executeNavigationAction( + requestId, + workspaceId, + backCommand.args.browserId ?? request.browserId, + "back", + registry, + ); + }, + forward: ({ request, command, requestId, workspaceId, registry }) => { + const forwardCommand = command as Extract; + return executeNavigationAction( + requestId, + workspaceId, + forwardCommand.args.browserId ?? request.browserId, + "forward", + registry, + ); + }, + reload: ({ request, command, requestId, workspaceId, registry }) => { + const reloadCommand = command as Extract; + return executeNavigationAction( + requestId, + workspaceId, + reloadCommand.args.browserId ?? request.browserId, + "reload", + registry, + ); + }, + screenshot: ({ request, command, requestId, workspaceId, registry }) => { + const screenshotCommand = command as Extract< + BrowserAutomationCommand, + { command: "screenshot" } + >; + return executeScreenshot( + requestId, + workspaceId, + screenshotCommand.args.browserId ?? request.browserId, + registry, + ); + }, + full_page_screenshot: ({ request, command, requestId, workspaceId, registry }) => { + const screenshotCommand = command as Extract< + BrowserAutomationCommand, + { command: "full_page_screenshot" } + >; + return executeFullPageScreenshot( + requestId, + workspaceId, + screenshotCommand.args.browserId ?? request.browserId, + registry, + ); + }, + pdf: ({ request, command, requestId, workspaceId, registry }) => { + const pdfCommand = command as Extract; + return executePdf( + requestId, + workspaceId, + pdfCommand.args.browserId ?? request.browserId, + { landscape: pdfCommand.args.landscape, printBackground: pdfCommand.args.printBackground }, + registry, + ); + }, + download: ({ request, command, requestId, workspaceId, registry }) => { + const downloadCommand = command as Extract; + return executeDownload( + requestId, + workspaceId, + downloadCommand.args.browserId ?? request.browserId, + { url: downloadCommand.args.url, fileName: downloadCommand.args.fileName }, + registry, + ); + }, + upload: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const uploadCommand = command as Extract; + return executeUpload( + requestId, + workspaceId, + uploadCommand.args.browserId ?? request.browserId, + { ref: uploadCommand.args.ref, filePaths: uploadCommand.args.filePaths }, + registry, + snapshotEngine, + ); + }, + focus: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const focusCommand = command as Extract; + return executeFocus( + requestId, + workspaceId, + focusCommand.args.browserId ?? request.browserId, + focusCommand.args.ref, + registry, + snapshotEngine, + ); + }, + clear: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const clearCommand = command as Extract; + return executeClear( + requestId, + workspaceId, + clearCommand.args.browserId ?? request.browserId, + clearCommand.args.ref, + registry, + snapshotEngine, + ); + }, + check: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const checkCommand = command as Extract; + return executeCheck( + requestId, + workspaceId, + checkCommand.args.browserId ?? request.browserId, + checkCommand.args.ref, + checkCommand.args.checked, + registry, + snapshotEngine, + ); + }, + select: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const selectCommand = command as Extract; + return executeSelect( + requestId, + workspaceId, + selectCommand.args.browserId ?? request.browserId, + selectCommand.args.ref, + selectCommand.args.value, + registry, + snapshotEngine, + ); + }, + hover: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const hoverCommand = command as Extract; + return executeHover( + requestId, + workspaceId, + hoverCommand.args.browserId ?? request.browserId, + hoverCommand.args.ref, + registry, + snapshotEngine, + ); + }, + drag: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + const dragCommand = command as Extract; + return executeDrag( + requestId, + workspaceId, + dragCommand.args.browserId ?? request.browserId, + dragCommand.args.sourceRef, + dragCommand.args.targetRef, + registry, + snapshotEngine, + ); + }, + logs: ({ request, command, requestId, workspaceId, registry }) => { + const logsCommand = command as Extract; + return executeLogs( + requestId, + workspaceId, + logsCommand.args.browserId ?? request.browserId, + logsCommand.args.maxEntries, + registry, + ); + }, + storage: ({ request, command, requestId, workspaceId, registry }) => { + const storageCommand = command as Extract; + return executeStorage( + requestId, + workspaceId, + storageCommand.args.browserId ?? request.browserId, + registry, + ); + }, + environment: ({ request, command, requestId, workspaceId, registry }) => { + const environmentCommand = command as Extract< + BrowserAutomationCommand, + { command: "environment" } + >; + return executeEnvironment( + requestId, + workspaceId, + environmentCommand.args.browserId ?? request.browserId, + { + viewport: environmentCommand.args.viewport, + geolocation: environmentCommand.args.geolocation, + }, + registry, + ); + }, + set_background: ({ request, command, requestId, workspaceId, registry }) => { + const setBackgroundCommand = command as Extract< + BrowserAutomationCommand, + { command: "set_background" } + >; + return executeSetBackground( + requestId, + workspaceId, + setBackgroundCommand.args.browserId ?? request.browserId, + setBackgroundCommand.args.color, + registry, + ); + }, +}; + +interface ResolvedTabTarget { + browserId: string; + contents: TabContents; +} + +function executeListTabs( + requestId: string, + workspaceId: string | undefined, + registry: BrowserRegistry, +): AutomationCommandPayload { + const browserIds = workspaceId + ? registry.listRegisteredBrowserIdsForWorkspace(workspaceId) + : registry.listRegisteredBrowserIds(); + const activeBrowserId = workspaceId ? registry.getWorkspaceActiveBrowserId(workspaceId) : null; + const tabs: Array> = []; + + for (const browserId of browserIds) { + const contents = registry.getTabContents(browserId); + if (contents && !contents.isDestroyed()) { + tabs.push( + tabInfoFromContents( + browserId, + contents, + activeBrowserId, + registry.getBrowserWorkspaceId(browserId), + ), + ); + } + } + + return { requestId, ok: true, result: { command: "list_tabs", tabs } }; +} + +function executePageInfo( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + registry: BrowserRegistry, +): AutomationCommandPayload { + const target = resolveTabTarget({ + requestId, + workspaceId, + browserId, + registry, + }); + if ("ok" in target) { + return target; + } + + return { + requestId, + ok: true, + result: { + command: "page_info", + tab: tabInfoFromContents( + target.browserId, + target.contents, + workspaceId ? registry.getWorkspaceActiveBrowserId(workspaceId) : null, + registry.getBrowserWorkspaceId(target.browserId), + ), + }, + }; +} + +async function executeSnapshot( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ + requestId, + workspaceId, + browserId, + registry, + }); + if ("ok" in target) { + return target; + } + + const elements = await snapshotEngine.snapshot({ + browserId: target.browserId, + page: target.contents, + }); + + return { + requestId, + ok: true, + result: { + command: "snapshot", + browserId: target.browserId, + ...(registry.getBrowserWorkspaceId(target.browserId) + ? { workspaceId: registry.getBrowserWorkspaceId(target.browserId) ?? undefined } + : {}), + url: target.contents.getURL(), + title: target.contents.getTitle(), + elements, + }, + }; +} + +async function executeClick( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.click({ + browserId: target.browserId, + page: target.contents, + ref, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { requestId, ok: true, result: { command: "click", browserId: target.browserId, ref } }; +} + +async function executeFill( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string, + value: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.fill({ + browserId: target.browserId, + page: target.contents, + ref, + value, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { requestId, ok: true, result: { command: "fill", browserId: target.browserId, ref } }; +} + +async function executeFocus( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.focus({ + browserId: target.browserId, + page: target.contents, + ref, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { requestId, ok: true, result: { command: "focus", browserId: target.browserId, ref } }; +} + +async function executeClear( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.clear({ + browserId: target.browserId, + page: target.contents, + ref, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { requestId, ok: true, result: { command: "clear", browserId: target.browserId, ref } }; +} + +async function executeCheck( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string, + checked: boolean, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.check({ + browserId: target.browserId, + page: target.contents, + ref, + checked, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { + requestId, + ok: true, + result: { command: "check", browserId: target.browserId, ref, checked }, + }; +} + +async function executeSelect( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string, + value: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.select({ + browserId: target.browserId, + page: target.contents, + ref, + value, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { + requestId, + ok: true, + result: { command: "select", browserId: target.browserId, ref, value }, + }; +} + +async function executeHover( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.hover({ + browserId: target.browserId, + page: target.contents, + ref, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { requestId, ok: true, result: { command: "hover", browserId: target.browserId, ref } }; +} + +async function executeDrag( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + sourceRef: string, + targetRef: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.drag({ + browserId: target.browserId, + page: target.contents, + sourceRef, + targetRef, + }); + if (!result.ok) { + return staleRefFailure(requestId, `${sourceRef}/${targetRef}`); + } + return { + requestId, + ok: true, + result: { command: "drag", browserId: target.browserId, sourceRef, targetRef }, + }; +} + +async function executeLogs( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + maxEntries: number, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const consoleMessages = target.contents.getConsoleMessages?.() ?? []; + const networkEntries = parseNetworkEntries( + await target.contents.executeJavaScript(NETWORK_PERFORMANCE_SCRIPT), + ); + return { + requestId, + ok: true, + result: { + command: "logs", + browserId: target.browserId, + console: consoleMessages.slice(-maxEntries), + network: networkEntries.slice(-maxEntries), + }, + }; +} + +async function executeStorage( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const url = target.contents.getURL(); + const cookies = (await target.contents.getCookies?.(url)) ?? []; + const storage = parseStorageState(await target.contents.executeJavaScript(STORAGE_STATE_SCRIPT)); + return { + requestId, + ok: true, + result: { + command: "storage", + browserId: target.browserId, + url, + cookies, + localStorage: storage.localStorage, + sessionStorage: storage.sessionStorage, + }, + }; +} + +async function executeEnvironment( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + environment: { + viewport?: { width: number; height: number; deviceScaleFactor?: number }; + geolocation?: { latitude: number; longitude: number; accuracy?: number }; + }, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + if (environment.viewport) { + await target.contents.sendDebugCommand?.("Emulation.setDeviceMetricsOverride", { + width: environment.viewport.width, + height: environment.viewport.height, + deviceScaleFactor: environment.viewport.deviceScaleFactor ?? 1, + mobile: false, + }); + } + if (environment.geolocation) { + const geolocation = { + latitude: environment.geolocation.latitude, + longitude: environment.geolocation.longitude, + accuracy: environment.geolocation.accuracy ?? 1, + }; + await target.contents.sendDebugCommand?.("Emulation.setGeolocationOverride", geolocation); + await target.contents.executeJavaScript(buildGeolocationShimScript(geolocation)); + } + const viewport = parseViewport(await target.contents.executeJavaScript(VIEWPORT_SCRIPT)); + return { + requestId, + ok: true, + result: { + command: "environment", + browserId: target.browserId, + viewport, + ...(environment.geolocation + ? { + geolocation: { + ...environment.geolocation, + accuracy: environment.geolocation.accuracy ?? 1, + }, + } + : {}), + }, + }; +} + +function staleRefFailure(requestId: string, ref: string): FailurePayload { + return fail( + requestId, + "browser_stale_ref", + `Browser element reference ${ref} is stale. Take a new snapshot and try again.`, + ); +} + +async function executeWait( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + condition: { text?: string; url?: string; timeoutMs?: number }, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + + if (!condition.text && !condition.url) { + return fail(requestId, "browser_unsupported", "browser_wait requires text or url"); + } + + const timeoutMs = condition.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; + const deadline = Date.now() + timeoutMs; + do { + if (condition.url && target.contents.getURL().includes(condition.url)) { + return { + requestId, + ok: true, + result: { command: "wait", browserId: target.browserId, matched: "url" }, + }; + } + if (condition.text) { + const pageText = await target.contents.executeJavaScript("document.body.innerText || ''"); + if (typeof pageText === "string" && pageText.includes(condition.text)) { + return { + requestId, + ok: true, + result: { command: "wait", browserId: target.browserId, matched: "text" }, + }; + } + } + await delay(WAIT_POLL_INTERVAL_MS); + } while (Date.now() < deadline); + + if (condition.text) { + return fail( + requestId, + "browser_timeout", + `Timed out waiting for browser text: ${condition.text}`, + true, + ); + } + if (condition.url) { + return fail( + requestId, + "browser_timeout", + `Timed out waiting for browser URL: ${condition.url}`, + true, + ); + } + return fail(requestId, "browser_unsupported", "browser_wait requires text or url"); +} + +async function executeSetBackground( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + color: string, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + + await target.contents.executeJavaScript(buildSetBackgroundScript(color)); + return { + requestId, + ok: true, + result: { command: "set_background", browserId: target.browserId, color }, + }; +} + +function buildSetBackgroundScript(color: string): string { + return String.raw`(() => { + const color = ${JSON.stringify(color)}; + document.documentElement.style.background = color; + if (document.body) { + document.body.style.background = color; + document.body.style.backgroundColor = color; + document.body.style.minHeight = '100vh'; + } + return true; + })()`; +} + +async function executeType( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string | undefined, + text: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.typeText({ + browserId: target.browserId, + page: target.contents, + ...(ref ? { ref } : {}), + text, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref ?? "@e0"); + } + return { + requestId, + ok: true, + result: { command: "type", browserId: target.browserId, ...(ref ? { ref } : {}) }, + }; +} + +async function executeKeypress( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + ref: string | undefined, + key: string, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + const result = await snapshotEngine.keypress({ + browserId: target.browserId, + page: target.contents, + ...(ref ? { ref } : {}), + key, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref ?? "@e0"); + } + return { + requestId, + ok: true, + result: { command: "keypress", browserId: target.browserId, key, ...(ref ? { ref } : {}) }, + }; +} + +async function executeNavigate( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + url: string, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + if (!isAllowedPageUrl(url)) { + return fail( + requestId, + "browser_denied", + "Browser navigation only supports http and https URLs.", + ); + } + await target.contents.loadURL(url); + return { requestId, ok: true, result: { command: "navigate", browserId: target.browserId, url } }; +} + +function executeNavigationAction( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + action: "back" | "forward" | "reload", + registry: BrowserRegistry, +): AutomationCommandPayload { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + if (action === "back") { + target.contents.goBack(); + return { requestId, ok: true, result: { command: "back", browserId: target.browserId } }; + } + if (action === "forward") { + target.contents.goForward(); + return { requestId, ok: true, result: { command: "forward", browserId: target.browserId } }; + } + target.contents.reload(); + return { requestId, ok: true, result: { command: "reload", browserId: target.browserId } }; +} + +async function executeScreenshot( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + if (target.contents.sendDebugCommand) { + const screenshot = (await target.contents.sendDebugCommand("Page.captureScreenshot", { + format: "png", + fromSurface: false, + })) as CdpCaptureScreenshotResult; + if (!screenshot.data) { + return fail(requestId, "browser_unsupported", "browser_screenshot returned no data"); + } + const viewport = await getViewportSize(target.contents); + return { + requestId, + ok: true, + result: { + command: "screenshot", + browserId: target.browserId, + mimeType: "image/png", + dataBase64: screenshot.data, + width: viewport.width, + height: viewport.height, + }, + }; + } + const image = await target.contents.capturePage(); + const size = image.getSize(); + return { + requestId, + ok: true, + result: { + command: "screenshot", + browserId: target.browserId, + mimeType: "image/png", + dataBase64: Buffer.from(image.toPNG()).toString("base64"), + width: size.width, + height: size.height, + }, + }; +} + +interface CdpLayoutMetrics { + cssLayoutViewport?: { + clientWidth?: number; + clientHeight?: number; + }; + layoutViewport?: { + clientWidth?: number; + clientHeight?: number; + }; + cssContentSize?: { + width?: number; + height?: number; + }; + contentSize?: { + width?: number; + height?: number; + }; +} + +interface CdpCaptureScreenshotResult { + data?: string; +} + +async function getViewportSize(contents: TabContents): Promise<{ width: number; height: number }> { + const result = await contents.executeJavaScript( + "({ width: Math.round(window.innerWidth), height: Math.round(window.innerHeight) })", + ); + if (!result || typeof result !== "object") { + return { width: 0, height: 0 }; + } + const record = result as { width?: unknown; height?: unknown }; + return { + width: typeof record.width === "number" && Number.isFinite(record.width) ? record.width : 0, + height: typeof record.height === "number" && Number.isFinite(record.height) ? record.height : 0, + }; +} + +async function getCdpLayoutMetrics(contents: TabContents): Promise<{ + viewportWidth: number; + viewportHeight: number; + contentWidth: number; + contentHeight: number; +}> { + if (!contents.sendDebugCommand) { + return { viewportWidth: 0, viewportHeight: 0, contentWidth: 0, contentHeight: 0 }; + } + const metrics = (await contents.sendDebugCommand("Page.getLayoutMetrics")) as CdpLayoutMetrics; + const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport; + const contentSize = metrics.cssContentSize ?? metrics.contentSize; + return { + viewportWidth: Math.ceil(viewport?.clientWidth ?? 0), + viewportHeight: Math.ceil(viewport?.clientHeight ?? 0), + contentWidth: Math.ceil(contentSize?.width ?? 0), + contentHeight: Math.ceil(contentSize?.height ?? 0), + }; +} + +async function executeFullPageScreenshot( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + if (!target.contents.sendDebugCommand) { + return fail(requestId, "browser_unsupported", "browser_full_page_screenshot requires CDP"); + } + const metrics = await getCdpLayoutMetrics(target.contents); + const width = metrics.contentWidth; + const height = metrics.contentHeight; + const screenshot = (await target.contents.sendDebugCommand("Page.captureScreenshot", { + format: "png", + captureBeyondViewport: true, + clip: { x: 0, y: 0, width, height, scale: 1 }, + })) as CdpCaptureScreenshotResult; + if (!screenshot.data) { + return fail(requestId, "browser_unsupported", "browser_full_page_screenshot returned no data"); + } + return { + requestId, + ok: true, + result: { + command: "full_page_screenshot", + browserId: target.browserId, + mimeType: "image/png", + dataBase64: screenshot.data, + width, + height, + }, + }; +} + +async function executePdf( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + options: { landscape?: boolean; printBackground: boolean }, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + if (!target.contents.printToPDF) { + return fail(requestId, "browser_unsupported", "browser_pdf requires PDF support"); + } + const pdf = await target.contents.printToPDF({ + printBackground: options.printBackground, + ...(options.landscape !== undefined ? { landscape: options.landscape } : {}), + }); + return { + requestId, + ok: true, + result: { + command: "pdf", + browserId: target.browserId, + mimeType: "application/pdf", + dataBase64: Buffer.from(pdf).toString("base64"), + }, + }; +} + +async function executeDownload( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + input: { url: string; fileName?: string }, + registry: BrowserRegistry, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + if (!isAllowedPageUrl(input.url)) { + return fail(requestId, "browser_denied", "Browser download only supports http and https URLs."); + } + if (!target.contents.downloadURL) { + return fail(requestId, "browser_unsupported", "browser_download requires download support"); + } + const download = await target.contents.downloadURL(input); + return { + requestId, + ok: true, + result: { + command: "download", + browserId: target.browserId, + url: input.url, + filePath: download.filePath, + ...(download.totalBytes !== undefined ? { totalBytes: download.totalBytes } : {}), + state: download.state, + }, + }; +} + +function isAllowedPageUrl(value: string): boolean { + try { + return ALLOWED_PAGE_URL_PROTOCOLS.has(new URL(value).protocol); + } catch { + return false; + } +} + +async function executeUpload( + requestId: string, + workspaceId: string | undefined, + browserId: string | undefined, + input: { ref: string; filePaths: string[] }, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + if (!target.contents.sendDebugCommand) { + return fail(requestId, "browser_unsupported", "browser_upload requires CDP"); + } + const resolved = snapshotEngine.selectorForRef({ + browserId: target.browserId, + page: target.contents, + ref: input.ref, + }); + if (!resolved.ok) { + return staleRefFailure(requestId, input.ref); + } + const document = (await target.contents.sendDebugCommand("DOM.getDocument", { + depth: -1, + pierce: true, + })) as { root?: { nodeId?: number } }; + const rootNodeId = document.root?.nodeId; + if (typeof rootNodeId !== "number") { + return fail(requestId, "browser_unsupported", "browser_upload could not read DOM"); + } + const queried = (await target.contents.sendDebugCommand("DOM.querySelector", { + nodeId: rootNodeId, + selector: resolved.selector, + })) as { nodeId?: number }; + if (typeof queried.nodeId !== "number" || queried.nodeId <= 0) { + return staleRefFailure(requestId, input.ref); + } + const workspaceRoot = resolveUploadWorkspaceRoot({ workspaceId, target, registry }); + if (!workspaceRoot) { + return fail(requestId, "browser_unsupported", "browser_upload requires a workspace target"); + } + const filePaths = resolveWorkspaceFilePaths(input.filePaths, workspaceRoot); + if (!filePaths) { + return fail( + requestId, + "browser_unsupported", + "browser_upload only accepts files inside the agent workspace.", + ); + } + + await target.contents.sendDebugCommand("DOM.setFileInputFiles", { + nodeId: queried.nodeId, + files: filePaths, + }); + return { + requestId, + ok: true, + result: { + command: "upload", + browserId: target.browserId, + ref: input.ref, + filePaths, + }, + }; +} + +function resolveUploadWorkspaceRoot(params: { + workspaceId: string | undefined; + target: ResolvedTabTarget; + registry: BrowserRegistry; +}): string | null { + const workspaceRoot = + params.workspaceId ?? params.registry.getBrowserWorkspaceId(params.target.browserId); + return workspaceRoot ? resolvePath(workspaceRoot) : null; +} + +function resolveWorkspaceFilePaths(filePaths: string[], workspaceRoot: string): string[] | null { + const resolvedPaths = filePaths.map((filePath) => + isAbsolute(filePath) ? resolvePath(filePath) : resolvePath(workspaceRoot, filePath), + ); + if (resolvedPaths.some((filePath) => !isPathInsideDirectory(filePath, workspaceRoot))) { + return null; + } + return resolvedPaths; +} + +function isPathInsideDirectory(filePath: string, directory: string): boolean { + const relativePath = relative(directory, filePath); + return relativePath === "" || (!relativePath.startsWith("..") && !isAbsolute(relativePath)); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function parseNetworkEntries(value: unknown): BrowserAutomationNetworkLogEntry[] { + const parsed = typeof value === "string" ? JSON.parse(value) : value; + if (!Array.isArray(parsed)) { + return []; + } + return parsed.flatMap((entry): BrowserAutomationNetworkLogEntry[] => { + if (!entry || typeof entry !== "object") { + return []; + } + const record = entry as Record; + const url = readString(record.url); + const startTime = readNumber(record.startTime); + const duration = readNumber(record.duration); + if (!url || startTime === null || duration === null) { + return []; + } + return [ + { + url, + ...(readString(record.method) ? { method: readString(record.method) ?? undefined } : {}), + ...(readNumber(record.status) !== null + ? { status: readNumber(record.status) ?? undefined } + : {}), + ...(readString(record.type) ? { type: readString(record.type) ?? undefined } : {}), + startTime, + duration, + ...(readNumber(record.transferSize) !== null + ? { transferSize: readNumber(record.transferSize) ?? undefined } + : {}), + }, + ]; + }); +} + +function parseStorageState(value: unknown): { + localStorage: BrowserAutomationStorageEntry[]; + sessionStorage: BrowserAutomationStorageEntry[]; +} { + const parsed = typeof value === "string" ? JSON.parse(value) : value; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return { localStorage: [], sessionStorage: [] }; + } + const record = parsed as Record; + return { + localStorage: parseStorageEntries(record.localStorage), + sessionStorage: parseStorageEntries(record.sessionStorage), + }; +} + +function parseStorageEntries(value: unknown): BrowserAutomationStorageEntry[] { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap((entry): BrowserAutomationStorageEntry[] => { + if (!entry || typeof entry !== "object") { + return []; + } + const record = entry as Record; + const key = readString(record.key); + const itemValue = readString(record.value); + return key !== null && itemValue !== null ? [{ key, value: itemValue }] : []; + }); +} + +function readString(value: unknown): string | null { + return typeof value === "string" ? value : null; +} + +function readNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function parseViewport(value: unknown): { + width: number; + height: number; + deviceScaleFactor: number; +} { + const parsed = typeof value === "string" ? JSON.parse(value) : value; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return { width: 0, height: 0, deviceScaleFactor: 1 }; + } + const record = parsed as Record; + return { + width: readNumber(record.width) ?? 0, + height: readNumber(record.height) ?? 0, + deviceScaleFactor: readNumber(record.deviceScaleFactor) ?? 1, + }; +} + +function buildGeolocationShimScript(geolocation: { + latitude: number; + longitude: number; + accuracy: number; +}): string { + return String.raw`(() => { + const coords = ${JSON.stringify({ ...geolocation, altitude: null, altitudeAccuracy: null, heading: null, speed: null })}; + const position = { coords, timestamp: Date.now() }; + Object.defineProperty(navigator, 'geolocation', { + configurable: true, + value: { + getCurrentPosition(success) { setTimeout(() => success(position), 0); }, + watchPosition(success) { setTimeout(() => success(position), 0); return 1; }, + clearWatch() {}, + }, + }); + return true; + })()`; +} + +const NETWORK_PERFORMANCE_SCRIPT = String.raw`(() => { + const entries = performance.getEntriesByType('resource') + .concat(performance.getEntriesByType('navigation')) + .slice(-200) + .map((entry) => ({ + url: entry.name, + method: entry.initiatorType === 'navigation' ? 'GET' : undefined, + type: entry.initiatorType, + startTime: entry.startTime, + duration: entry.duration, + transferSize: typeof entry.transferSize === 'number' ? entry.transferSize : undefined, + })); + return JSON.stringify(entries); +})()`; + +const STORAGE_STATE_SCRIPT = String.raw`(() => { + function entriesFor(storage) { + const entries = []; + for (let index = 0; index < storage.length; index += 1) { + const key = storage.key(index); + if (key !== null) entries.push({ key, value: storage.getItem(key) || '' }); + } + return entries; + } + function safeEntries(readStorage) { + try { + return entriesFor(readStorage()); + } catch { + return []; + } + } + return JSON.stringify({ + localStorage: safeEntries(() => window.localStorage), + sessionStorage: safeEntries(() => window.sessionStorage), + }); +})()`; + +const VIEWPORT_SCRIPT = String.raw`(() => JSON.stringify({ + width: window.innerWidth, + height: window.innerHeight, + deviceScaleFactor: window.devicePixelRatio || 1, +}))()`; + +function resolveTabTarget(input: { + requestId: string; + workspaceId: string | undefined; + browserId: string | undefined; + registry: BrowserRegistry; +}): ResolvedTabTarget | FailurePayload { + const { requestId, workspaceId, browserId, registry } = input; + let contents: TabContents | null; + let resolvedBrowserId: string; + + if (browserId) { + if (workspaceId && registry.getBrowserWorkspaceId(browserId) !== workspaceId) { + return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`); + } + contents = registry.getTabContents(browserId); + resolvedBrowserId = browserId; + if (!contents) { + return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`); + } + } else { + if (!workspaceId) { + return fail(requestId, "browser_no_tab", "No active browser tab in workspace"); + } + contents = registry.getWorkspaceActiveTabContents(workspaceId); + const activeId = registry.getWorkspaceActiveBrowserId(workspaceId); + if (!contents || !activeId) { + return fail(requestId, "browser_no_tab", "No active browser tab in workspace"); + } + resolvedBrowserId = activeId; + } + + if (contents.isDestroyed()) { + return fail( + requestId, + "browser_tab_closed", + `Browser tab ${resolvedBrowserId} has been closed`, + ); + } + + return { browserId: resolvedBrowserId, contents }; +} diff --git a/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts b/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts new file mode 100644 index 000000000..22bd60936 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; +import { BrowserSnapshotEngine, type SnapshotPage } from "./snapshot-engine.js"; + +class SnapshotFixture implements SnapshotPage { + public currentUrl = "https://example.com/form"; + public actionResult: unknown = true; + + public getURL(): string { + return this.currentUrl; + } + + public async executeJavaScript(code: string): Promise { + if (code.includes("CANDIDATE_SELECTOR")) { + return JSON.stringify([ + { + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + { + role: "button", + tagName: "button", + text: "Drop", + selector: "#drop", + attributes: { id: "drop" }, + }, + ]); + } + return this.actionResult; + } +} + +describe("BrowserSnapshotEngine", () => { + it("treats a false result from a ref action script as a stale ref", async () => { + const page = new SnapshotFixture(); + const engine = new BrowserSnapshotEngine(); + await engine.snapshot({ browserId: "browser-1", page }); + + page.actionResult = false; + + await expect(engine.click({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({ + ok: false, + reason: "stale_ref", + }); + await expect(engine.focus({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({ + ok: false, + reason: "stale_ref", + }); + }); + + it("treats a false result from optional ref text/key actions as a stale ref", async () => { + const page = new SnapshotFixture(); + const engine = new BrowserSnapshotEngine(); + await engine.snapshot({ browserId: "browser-1", page }); + + page.actionResult = false; + + await expect( + engine.typeText({ browserId: "browser-1", page, ref: "@e1", text: "Ada" }), + ).resolves.toEqual({ ok: false, reason: "stale_ref" }); + await expect( + engine.keypress({ browserId: "browser-1", page, ref: "@e1", key: "Enter" }), + ).resolves.toEqual({ ok: false, reason: "stale_ref" }); + }); + + it("treats a false result from drag as a stale ref", async () => { + const page = new SnapshotFixture(); + const engine = new BrowserSnapshotEngine(); + await engine.snapshot({ browserId: "browser-1", page }); + + page.actionResult = false; + + await expect( + engine.drag({ browserId: "browser-1", page, sourceRef: "@e1", targetRef: "@e2" }), + ).resolves.toEqual({ ok: false, reason: "stale_ref" }); + }); +}); diff --git a/packages/desktop/src/features/browser-automation/snapshot-engine.ts b/packages/desktop/src/features/browser-automation/snapshot-engine.ts new file mode 100644 index 000000000..4d3c09745 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/snapshot-engine.ts @@ -0,0 +1,570 @@ +export interface SnapshotPage { + getURL(): string; + executeJavaScript(code: string): Promise; +} + +export interface BrowserSnapshotElement extends RawSnapshotElement { + ref: string; +} + +interface RawSnapshotElement { + role: string; + tagName: string; + text: string; + selector: string; + attributes: Record; +} + +interface BrowserRefState { + nextRefNumber: number; + url: string; + refs: Map; +} + +export type BrowserRefActionResult = + | { ok: true } + | { ok: false; reason: "stale_ref" | "missing_ref" }; +type BrowserRefFailure = Extract; + +type BrowserRefResolveResult = { ok: true; element: RawSnapshotElement } | BrowserRefFailure; + +export class BrowserSnapshotEngine { + private readonly statesByBrowserId = new Map(); + + async snapshot(input: { + browserId: string; + page: SnapshotPage; + }): Promise { + const rawElements = parseRawSnapshotElements( + await input.page.executeJavaScript(SNAPSHOT_SCRIPT), + ); + const state = { + nextRefNumber: 1, + url: input.page.getURL(), + refs: new Map(), + }; + const elements = rawElements.map((element) => { + const ref = `@e${state.nextRefNumber++}`; + state.refs.set(ref, element); + return { + ref, + role: element.role, + tagName: element.tagName, + text: element.text, + selector: element.selector, + attributes: element.attributes, + }; + }); + this.statesByBrowserId.set(input.browserId, state); + return elements; + } + + async click(input: { + browserId: string; + page: SnapshotPage; + ref: string; + }): Promise { + return this.runRefScript(input, (selector) => buildClickScript(selector)); + } + + async fill(input: { + browserId: string; + page: SnapshotPage; + ref: string; + value: string; + }): Promise { + return this.runRefScript(input, (selector) => buildFillScript(selector, input.value)); + } + + async typeText(input: { + browserId: string; + page: SnapshotPage; + ref?: string; + text: string; + }): Promise { + const selector = this.resolveOptionalRef(input); + if (!selector.ok) { + return selector; + } + const result = await input.page.executeJavaScript( + buildTypeScript(selector.selector, input.text), + ); + return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; + } + + async keypress(input: { + browserId: string; + page: SnapshotPage; + ref?: string; + key: string; + }): Promise { + const selector = this.resolveOptionalRef(input); + if (!selector.ok) { + return selector; + } + const result = await input.page.executeJavaScript( + buildKeypressScript(selector.selector, input.key), + ); + return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; + } + + async focus(input: { + browserId: string; + page: SnapshotPage; + ref: string; + }): Promise { + return this.runRefScript(input, (selector) => buildFocusScript(selector)); + } + + async clear(input: { + browserId: string; + page: SnapshotPage; + ref: string; + }): Promise { + return this.runRefScript(input, (selector) => buildClearScript(selector)); + } + + async check(input: { + browserId: string; + page: SnapshotPage; + ref: string; + checked: boolean; + }): Promise { + return this.runRefScript(input, (selector) => buildCheckScript(selector, input.checked)); + } + + async select(input: { + browserId: string; + page: SnapshotPage; + ref: string; + value: string; + }): Promise { + return this.runRefScript(input, (selector) => buildSelectScript(selector, input.value)); + } + + async hover(input: { + browserId: string; + page: SnapshotPage; + ref: string; + }): Promise { + return this.runRefScript(input, (selector) => buildHoverScript(selector)); + } + + async drag(input: { + browserId: string; + page: SnapshotPage; + sourceRef: string; + targetRef: string; + }): Promise { + const source = this.resolveRef({ + browserId: input.browserId, + page: input.page, + ref: input.sourceRef, + }); + if (!source.ok) { + return source; + } + const target = this.resolveRef({ + browserId: input.browserId, + page: input.page, + ref: input.targetRef, + }); + if (!target.ok) { + return target; + } + const result = await input.page.executeJavaScript( + buildDragScript(source.element.selector, target.element.selector), + ); + return result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; + } + + clearBrowser(browserId: string): void { + this.statesByBrowserId.delete(browserId); + } + + selectorForRef(input: { + browserId: string; + page: SnapshotPage; + ref: string; + }): { ok: true; selector: string } | BrowserRefFailure { + const resolved = this.resolveRef(input); + if (!resolved.ok) { + return resolved; + } + return { ok: true, selector: resolved.element.selector }; + } + + private async runRefScript( + input: { browserId: string; page: SnapshotPage; ref: string }, + buildScript: (selector: string) => string, + ): Promise { + const resolved = this.resolveRef(input); + if (!resolved.ok) { + return resolved; + } + const result = await input.page.executeJavaScript(buildScript(resolved.element.selector)); + return result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; + } + + private resolveRef(input: { + browserId: string; + page: SnapshotPage; + ref: string; + }): BrowserRefResolveResult { + const state = this.statesByBrowserId.get(input.browserId); + if (!state || state.url !== input.page.getURL()) { + return { ok: false, reason: "stale_ref" }; + } + const element = state.refs.get(input.ref); + if (!element) { + return { ok: false, reason: "missing_ref" }; + } + return { ok: true, element }; + } + + private resolveOptionalRef(input: { + browserId: string; + page: SnapshotPage; + ref?: string; + }): { ok: true; selector: string | undefined } | BrowserRefFailure { + if (!input.ref) { + return { ok: true, selector: undefined }; + } + const resolved = this.resolveRef({ + browserId: input.browserId, + page: input.page, + ref: input.ref, + }); + if (!resolved.ok) { + return resolved; + } + return { ok: true, selector: resolved.element.selector }; + } +} + +function buildClickScript(selector: string): string { + return String.raw`(() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!element) return false; + element.scrollIntoView({ block: 'center', inline: 'center' }); + element.click(); + return true; + })()`; +} + +function buildFillScript(selector: string, value: string): string { + return String.raw`(() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!element) return false; + element.scrollIntoView({ block: 'center', inline: 'center' }); + element.focus(); + const nextValue = ${JSON.stringify(value)}; + if ('value' in element) { + element.value = nextValue; + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + return true; + } + element.textContent = nextValue; + element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: nextValue })); + return true; + })()`; +} + +function buildTypeScript(selector: string | undefined, text: string): string { + return String.raw`(() => { + const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"}; + if (!element) return false; + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + element.focus?.(); + const text = ${JSON.stringify(text)}; + if ('value' in element) { + element.value = String(element.value || '') + text; + element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })); + element.dispatchEvent(new Event('change', { bubbles: true })); + return true; + } + element.textContent = String(element.textContent || '') + text; + element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })); + return true; + })()`; +} + +function buildKeypressScript(selector: string | undefined, key: string): string { + return String.raw`(() => { + const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"}; + if (!element) return false; + element.focus?.(); + const key = ${JSON.stringify(key)}; + const eventInit = { bubbles: true, cancelable: true, key }; + element.dispatchEvent(new KeyboardEvent('keydown', eventInit)); + element.dispatchEvent(new KeyboardEvent('keypress', eventInit)); + element.dispatchEvent(new KeyboardEvent('keyup', eventInit)); + return true; + })()`; +} + +function buildFocusScript(selector: string): string { + return String.raw`(() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!element) return false; + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + element.focus?.(); + return document.activeElement === element; + })()`; +} + +function buildClearScript(selector: string): string { + return String.raw`(() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!element) return false; + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + element.focus?.(); + if ('value' in element) { + element.value = ''; + element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' })); + element.dispatchEvent(new Event('change', { bubbles: true })); + return true; + } + element.textContent = ''; + element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' })); + return true; + })()`; +} + +function buildCheckScript(selector: string, checked: boolean): string { + return String.raw`(() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!element) return false; + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + element.focus?.(); + const nextChecked = ${JSON.stringify(checked)}; + if ('checked' in element) { + element.checked = nextChecked; + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + return true; + } + if (element.getAttribute('role') === 'checkbox' || element.getAttribute('role') === 'radio') { + element.setAttribute('aria-checked', String(nextChecked)); + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + return true; + } + return false; + })()`; +} + +function buildSelectScript(selector: string, value: string): string { + return String.raw`(() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!element) return false; + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + element.focus?.(); + const nextValue = ${JSON.stringify(value)}; + if ('value' in element) { + element.value = nextValue; + element.dispatchEvent(new Event('input', { bubbles: true })); + element.dispatchEvent(new Event('change', { bubbles: true })); + return true; + } + return false; + })()`; +} + +function buildHoverScript(selector: string): string { + return String.raw`(() => { + const element = document.querySelector(${JSON.stringify(selector)}); + if (!element) return false; + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + const rect = element.getBoundingClientRect(); + const eventInit = { + bubbles: true, + cancelable: true, + clientX: rect.left + rect.width / 2, + clientY: rect.top + rect.height / 2, + screenX: window.screenX + rect.left + rect.width / 2, + screenY: window.screenY + rect.top + rect.height / 2, + view: window, + }; + element.dispatchEvent(new MouseEvent('mouseover', eventInit)); + element.dispatchEvent(new MouseEvent('mouseenter', eventInit)); + element.dispatchEvent(new MouseEvent('mousemove', eventInit)); + return true; + })()`; +} + +function buildDragScript(sourceSelector: string, targetSelector: string): string { + return String.raw`(() => { + const source = document.querySelector(${JSON.stringify(sourceSelector)}); + const target = document.querySelector(${JSON.stringify(targetSelector)}); + if (!source || !target) return false; + source.scrollIntoView?.({ block: 'center', inline: 'center' }); + target.scrollIntoView?.({ block: 'center', inline: 'center' }); + const data = new DataTransfer(); + const sourceRect = source.getBoundingClientRect(); + const targetRect = target.getBoundingClientRect(); + function eventInit(rect) { + return { + bubbles: true, + cancelable: true, + clientX: rect.left + rect.width / 2, + clientY: rect.top + rect.height / 2, + screenX: window.screenX + rect.left + rect.width / 2, + screenY: window.screenY + rect.top + rect.height / 2, + dataTransfer: data, + view: window, + }; + } + source.dispatchEvent(new MouseEvent('mousedown', eventInit(sourceRect))); + source.dispatchEvent(new DragEvent('dragstart', eventInit(sourceRect))); + target.dispatchEvent(new DragEvent('dragenter', eventInit(targetRect))); + target.dispatchEvent(new DragEvent('dragover', eventInit(targetRect))); + target.dispatchEvent(new DragEvent('drop', eventInit(targetRect))); + source.dispatchEvent(new DragEvent('dragend', eventInit(sourceRect))); + target.dispatchEvent(new MouseEvent('mouseup', eventInit(targetRect))); + return true; + })()`; +} + +function parseRawSnapshotElements(value: unknown): RawSnapshotElement[] { + const parsed = typeof value === "string" ? JSON.parse(value) : value; + if (!Array.isArray(parsed)) { + return []; + } + return parsed.flatMap((item): RawSnapshotElement[] => { + if (!item || typeof item !== "object") { + return []; + } + const record = item as Record; + const selector = readString(record.selector); + if (!selector) { + return []; + } + return [ + { + role: readString(record.role) || "generic", + tagName: (readString(record.tagName) || "element").toLowerCase(), + text: readString(record.text) || "", + selector, + attributes: readAttributes(record.attributes), + }, + ]; + }); +} + +function readString(value: unknown): string | null { + return typeof value === "string" ? value : null; +} + +function readAttributes(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return {}; + } + const result: Record = {}; + for (const [key, attributeValue] of Object.entries(value)) { + if (typeof attributeValue === "string") { + result[key] = attributeValue; + } + } + return result; +} + +const SNAPSHOT_SCRIPT = String.raw`(() => { + const MAX_ELEMENTS = 200; + const CANDIDATE_SELECTOR = [ + 'a[href]', + 'button', + 'input', + 'textarea', + 'select', + 'summary', + '[role]', + '[tabindex]:not([tabindex="-1"])', + '[contenteditable=""]', + '[contenteditable="true"]' + ].join(','); + + function cssEscape(value) { + if (window.CSS && typeof window.CSS.escape === 'function') { + return window.CSS.escape(value); + } + return String(value).replace(/[^a-zA-Z0-9_-]/g, '\\$&'); + } + + function isVisible(element) { + const style = window.getComputedStyle(element); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { + return false; + } + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + } + + function roleFor(element) { + const explicit = element.getAttribute('role'); + if (explicit) return explicit; + const tag = element.tagName.toLowerCase(); + if (tag === 'a') return 'link'; + if (tag === 'button') return 'button'; + if (tag === 'select') return 'combobox'; + if (tag === 'textarea') return 'textbox'; + if (tag === 'summary') return 'button'; + if (tag === 'input') { + const type = (element.getAttribute('type') || 'text').toLowerCase(); + if (type === 'checkbox') return 'checkbox'; + if (type === 'radio') return 'radio'; + if (type === 'button' || type === 'submit' || type === 'reset') return 'button'; + return 'textbox'; + } + return 'generic'; + } + + function textFor(element) { + const tag = element.tagName.toLowerCase(); + const pieces = [ + element.getAttribute('aria-label'), + element.getAttribute('alt'), + element.getAttribute('title'), + tag === 'input' ? element.getAttribute('placeholder') : null, + tag === 'input' || tag === 'textarea' ? element.value : null, + element.innerText, + element.textContent + ]; + const text = pieces.find((piece) => typeof piece === 'string' && piece.trim().length > 0); + return (text || '').replace(/\s+/g, ' ').trim().slice(0, 300); + } + + function selectorFor(element) { + if (element.id) return '#' + cssEscape(element.id); + const parts = []; + let current = element; + while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) { + const tag = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (!parent) break; + const siblings = Array.from(parent.children).filter((sibling) => sibling.tagName === current.tagName); + const index = siblings.indexOf(current) + 1; + parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag); + current = parent; + } + return parts.length > 0 ? parts.join(' > ') : element.tagName.toLowerCase(); + } + + return JSON.stringify(Array.from(document.querySelectorAll(CANDIDATE_SELECTOR)) + .filter(isVisible) + .slice(0, MAX_ELEMENTS) + .map((element) => ({ + role: roleFor(element), + tagName: element.tagName.toLowerCase(), + text: textFor(element), + selector: selectorFor(element), + attributes: { + ...(element.id ? { id: element.id } : {}), + ...(element.getAttribute('name') ? { name: element.getAttribute('name') } : {}), + ...(element.getAttribute('type') ? { type: element.getAttribute('type') } : {}), + ...(element.getAttribute('href') ? { href: element.getAttribute('href') } : {}), + ...(element.getAttribute('aria-label') ? { 'aria-label': element.getAttribute('aria-label') } : {}) + } + }))); +})()`; diff --git a/packages/desktop/src/features/browser-webviews/index.ts b/packages/desktop/src/features/browser-webviews/index.ts index c6c575b8b..09fc9561b 100644 --- a/packages/desktop/src/features/browser-webviews/index.ts +++ b/packages/desktop/src/features/browser-webviews/index.ts @@ -4,11 +4,12 @@ import { handleBrowserWindowOpenRequest, isAllowedBrowserWebviewUrl, } from "./window-open.js"; +import { PaseoBrowserWebviewRegistry, type BrowserWorkspaceRegistration } from "./registry.js"; export { BROWSER_NEW_TAB_REQUEST_EVENT, handleBrowserWindowOpenRequest }; +export type { BrowserWorkspaceRegistration }; -const browserIdsByWebContentsId = new Map(); -let workspaceActiveBrowserId: string | null = null; +const browserRegistry = new PaseoBrowserWebviewRegistry(); function getBrowserIdFromWebviewPartition(partition: string | undefined): string | null { const prefix = "persist:paseo-browser-"; @@ -30,16 +31,15 @@ export function readBrowserIdFromWebviewAttach(input: { } export function listRegisteredPaseoBrowserIds(): string[] { - return Array.from(new Set(browserIdsByWebContentsId.values())).sort(); + return browserRegistry + .listBrowserIds() + .filter((browserId) => getPaseoBrowserWebContents(browserId)); } export function registerPaseoBrowserWebContents(contents: WebContents, browserId: string): void { - browserIdsByWebContentsId.set(contents.id, browserId); + browserRegistry.registerWebContents({ webContentsId: contents.id, browserId }); contents.once("destroyed", () => { - browserIdsByWebContentsId.delete(contents.id); - if (workspaceActiveBrowserId === browserId) { - workspaceActiveBrowserId = null; - } + browserRegistry.unregisterWebContents(contents.id); }); } @@ -47,29 +47,71 @@ export function getPaseoBrowserIdForWebContents(contents: WebContents | null): s if (!contents || contents.isDestroyed()) { return null; } - return browserIdsByWebContentsId.get(contents.id) ?? null; + return browserRegistry.getBrowserIdForWebContents(contents.id); } -export function setWorkspaceActivePaseoBrowserId(browserId: string | null): void { - workspaceActiveBrowserId = browserId; +export function registerPaseoBrowserWorkspace(input: BrowserWorkspaceRegistration): void { + browserRegistry.registerWorkspace(input); +} + +export function getPaseoBrowserWorkspaceId(browserId: string): string | null { + return browserRegistry.getWorkspaceId(browserId); +} + +export function listRegisteredPaseoBrowserIdsForWorkspace(workspaceId: string): string[] { + return browserRegistry + .listBrowserIdsForWorkspace(workspaceId) + .filter((browserId) => getPaseoBrowserWebContents(browserId)); +} + +export function setWorkspaceActivePaseoBrowserId(input: { + workspaceId: string; + browserId: string | null; +}): void { + browserRegistry.setWorkspaceActiveBrowser(input); +} + +export function getWorkspaceActivePaseoBrowserId(workspaceId: string): string | null { + return browserRegistry.getWorkspaceActiveBrowserId(workspaceId); +} + +export function setAgentActivePaseoBrowserId(input: { + agentId: string; + browserId: string | null; +}): void { + browserRegistry.setAgentActiveBrowser(input); +} + +export function getAgentActivePaseoBrowserId(agentId: string): string | null { + return browserRegistry.getAgentActiveBrowserId(agentId); } export function getPaseoBrowserWebContents(browserId: string): WebContents | null { - for (const [contentsId, registeredBrowserId] of browserIdsByWebContentsId) { - if (registeredBrowserId !== browserId) continue; - const contents = allWebContents.fromId(contentsId); - if (contents && !contents.isDestroyed()) { - return contents; - } + const contentsId = browserRegistry.getWebContentsIdForBrowser(browserId); + if (contentsId === null) { + return null; } + const contents = allWebContents.fromId(contentsId); + if (contents && !contents.isDestroyed()) { + return contents; + } + browserRegistry.unregisterWebContents(contentsId); return null; } -export function getWorkspaceActivePaseoBrowserWebContents(): WebContents | null { - if (!workspaceActiveBrowserId) { - return null; - } - return getPaseoBrowserWebContents(workspaceActiveBrowserId); +export function getWorkspaceActivePaseoBrowserWebContents(workspaceId: string): WebContents | null { + const activeBrowserId = getWorkspaceActivePaseoBrowserId(workspaceId); + return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null; +} + +export function getAgentActivePaseoBrowserWebContents(agentId: string): WebContents | null { + const activeBrowserId = getAgentActivePaseoBrowserId(agentId); + return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null; +} + +export function getMostRecentWorkspaceActivePaseoBrowserWebContents(): WebContents | null { + const browserId = browserRegistry.getMostRecentWorkspaceActiveBrowserId(); + return browserId ? getPaseoBrowserWebContents(browserId) : null; } function preventUnsafeBrowserWebviewNavigation( diff --git a/packages/desktop/src/features/browser-webviews/registry.test.ts b/packages/desktop/src/features/browser-webviews/registry.test.ts new file mode 100644 index 000000000..02bd40090 --- /dev/null +++ b/packages/desktop/src/features/browser-webviews/registry.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { PaseoBrowserWebviewRegistry } from "./registry.js"; + +describe("PaseoBrowserWebviewRegistry", () => { + it("keeps one authoritative webContents target per browserId", () => { + const registry = new PaseoBrowserWebviewRegistry(); + + registry.registerWebContents({ webContentsId: 1, browserId: "browser-a" }); + registry.registerWorkspace({ browserId: "browser-a", workspaceId: "workspace-a" }); + registry.setWorkspaceActiveBrowser({ workspaceId: "workspace-a", browserId: "browser-a" }); + registry.setAgentActiveBrowser({ agentId: "agent-a", browserId: "browser-a" }); + registry.registerWebContents({ webContentsId: 2, browserId: "browser-a" }); + + expect(registry.getBrowserIdForWebContents(1)).toBeNull(); + expect(registry.getBrowserIdForWebContents(2)).toBe("browser-a"); + expect(registry.getWebContentsIdForBrowser("browser-a")).toBe(2); + expect(registry.getWorkspaceId("browser-a")).toBe("workspace-a"); + expect(registry.getWorkspaceActiveBrowserId("workspace-a")).toBe("browser-a"); + expect(registry.getAgentActiveBrowserId("agent-a")).toBe("browser-a"); + }); + + it("ignores stale destroy events after a duplicate browserId moved", () => { + const registry = new PaseoBrowserWebviewRegistry(); + + registry.registerWebContents({ webContentsId: 1, browserId: "browser-a" }); + registry.registerWebContents({ webContentsId: 2, browserId: "browser-a" }); + registry.unregisterWebContents(1); + + expect(registry.getWebContentsIdForBrowser("browser-a")).toBe(2); + }); +}); diff --git a/packages/desktop/src/features/browser-webviews/registry.ts b/packages/desktop/src/features/browser-webviews/registry.ts new file mode 100644 index 000000000..a9ccd4966 --- /dev/null +++ b/packages/desktop/src/features/browser-webviews/registry.ts @@ -0,0 +1,108 @@ +export interface BrowserWorkspaceRegistration { + browserId: string; + workspaceId: string; +} + +export class PaseoBrowserWebviewRegistry { + private readonly browserIdsByWebContentsId = new Map(); + private readonly webContentsIdsByBrowserId = new Map(); + private readonly workspaceIdsByBrowserId = new Map(); + private readonly activeBrowserIdsByWorkspaceId = new Map(); + private readonly activeBrowserIdsByAgentId = new Map(); + + public registerWebContents(input: { webContentsId: number; browserId: string }): void { + const previousWebContentsId = this.webContentsIdsByBrowserId.get(input.browserId) ?? null; + if (previousWebContentsId !== null && previousWebContentsId !== input.webContentsId) { + this.browserIdsByWebContentsId.delete(previousWebContentsId); + } + + this.browserIdsByWebContentsId.set(input.webContentsId, input.browserId); + this.webContentsIdsByBrowserId.set(input.browserId, input.webContentsId); + } + + public unregisterWebContents(webContentsId: number): void { + const browserId = this.browserIdsByWebContentsId.get(webContentsId) ?? null; + if (!browserId) { + return; + } + + this.browserIdsByWebContentsId.delete(webContentsId); + if (this.webContentsIdsByBrowserId.get(browserId) !== webContentsId) { + return; + } + + this.webContentsIdsByBrowserId.delete(browserId); + this.workspaceIdsByBrowserId.delete(browserId); + this.deleteActiveBrowserReferences(browserId); + } + + public getBrowserIdForWebContents(webContentsId: number): string | null { + return this.browserIdsByWebContentsId.get(webContentsId) ?? null; + } + + public getWebContentsIdForBrowser(browserId: string): number | null { + return this.webContentsIdsByBrowserId.get(browserId) ?? null; + } + + public listBrowserIds(): string[] { + return Array.from(this.webContentsIdsByBrowserId.keys()).sort(); + } + + public registerWorkspace(input: BrowserWorkspaceRegistration): void { + this.workspaceIdsByBrowserId.set(input.browserId, input.workspaceId); + } + + public getWorkspaceId(browserId: string): string | null { + return this.workspaceIdsByBrowserId.get(browserId) ?? null; + } + + public listBrowserIdsForWorkspace(workspaceId: string): string[] { + return this.listBrowserIds().filter( + (browserId) => this.workspaceIdsByBrowserId.get(browserId) === workspaceId, + ); + } + + public setWorkspaceActiveBrowser(input: { workspaceId: string; browserId: string | null }): void { + if (input.browserId) { + this.workspaceIdsByBrowserId.set(input.browserId, input.workspaceId); + this.activeBrowserIdsByWorkspaceId.delete(input.workspaceId); + this.activeBrowserIdsByWorkspaceId.set(input.workspaceId, input.browserId); + return; + } + this.activeBrowserIdsByWorkspaceId.delete(input.workspaceId); + } + + public getWorkspaceActiveBrowserId(workspaceId: string): string | null { + return this.activeBrowserIdsByWorkspaceId.get(workspaceId) ?? null; + } + + public getMostRecentWorkspaceActiveBrowserId(): string | null { + return Array.from(this.activeBrowserIdsByWorkspaceId.values()).at(-1) ?? null; + } + + public setAgentActiveBrowser(input: { agentId: string; browserId: string | null }): void { + if (input.browserId) { + this.activeBrowserIdsByAgentId.delete(input.agentId); + this.activeBrowserIdsByAgentId.set(input.agentId, input.browserId); + return; + } + this.activeBrowserIdsByAgentId.delete(input.agentId); + } + + public getAgentActiveBrowserId(agentId: string): string | null { + return this.activeBrowserIdsByAgentId.get(agentId) ?? null; + } + + private deleteActiveBrowserReferences(browserId: string): void { + for (const [workspaceId, activeBrowserId] of this.activeBrowserIdsByWorkspaceId) { + if (activeBrowserId === browserId) { + this.activeBrowserIdsByWorkspaceId.delete(workspaceId); + } + } + for (const [agentId, activeBrowserId] of this.activeBrowserIdsByAgentId) { + if (activeBrowserId === browserId) { + this.activeBrowserIdsByAgentId.delete(agentId); + } + } + } +} diff --git a/packages/desktop/src/features/menu.ts b/packages/desktop/src/features/menu.ts index 6e56e8efd..c086c2a24 100644 --- a/packages/desktop/src/features/menu.ts +++ b/packages/desktop/src/features/menu.ts @@ -1,5 +1,5 @@ import { app, Menu, BrowserWindow, ipcMain } from "electron"; -import { getWorkspaceActivePaseoBrowserWebContents } from "./browser-webviews/index.js"; +import { getMostRecentWorkspaceActivePaseoBrowserWebContents } from "./browser-webviews/index.js"; interface ShowContextMenuInput { kind?: "terminal"; @@ -20,7 +20,7 @@ function withBrowserWindow( } function getReloadTargetBrowserWebContents(): Electron.WebContents | null { - return getWorkspaceActivePaseoBrowserWebContents(); + return getMostRecentWorkspaceActivePaseoBrowserWebContents(); } function reloadFocusedContentsOrWindow(win: BrowserWindow, options?: { ignoreCache?: boolean }) { diff --git a/packages/desktop/src/main.ts b/packages/desktop/src/main.ts index ac9ba73eb..3b83ea004 100644 --- a/packages/desktop/src/main.ts +++ b/packages/desktop/src/main.ts @@ -53,7 +53,9 @@ import { listRegisteredPaseoBrowserIds, readBrowserIdFromWebviewAttach, registerBrowserWebviewNavigationGuards, + registerPaseoBrowserWorkspace, registerPaseoBrowserWebContents, + setAgentActivePaseoBrowserId, setWorkspaceActivePaseoBrowserId, } from "./features/browser-webviews/index.js"; import { parseOpenProjectPathFromArgv } from "./open-project-routing.js"; @@ -70,6 +72,7 @@ import { } from "./daemon/quit-lifecycle.js"; import { runDesktopStartup } from "./desktop-startup.js"; import { autoUpdateInstalledSkills } from "./integrations/skills/index.js"; +import { registerBrowserAutomationIpc } from "./features/browser-automation/ipc.js"; const DEV_SERVER_URL = process.env.EXPO_DEV_URL ?? "http://localhost:8081"; const APP_SCHEME = "paseo"; @@ -109,6 +112,50 @@ const DESKTOP_SMOKE_ENV = "PASEO_DESKTOP_SMOKE"; const DESKTOP_SMOKE_STOP_REQUEST = "paseo-smoke-stop"; app.setName(APP_NAME); +function readBrowserWorkspaceInput( + input: unknown, +): { browserId: string; workspaceId: string } | null { + if (typeof input !== "object" || input === null || Array.isArray(input)) { + return null; + } + const record = input as Record; + if (typeof record.browserId !== "string" || record.browserId.trim().length === 0) { + return null; + } + if (typeof record.workspaceId !== "string" || record.workspaceId.trim().length === 0) { + return null; + } + return { browserId: record.browserId.trim(), workspaceId: record.workspaceId.trim() }; +} + +function readActiveBrowserInput( + input: unknown, +): { workspaceId: string; browserId: string | null } | null { + if (typeof input !== "object" || input === null || Array.isArray(input)) { + return null; + } + const record = input as Record; + if (typeof record.workspaceId !== "string" || record.workspaceId.trim().length === 0) { + return null; + } + const browserId = typeof record.browserId === "string" ? record.browserId.trim() : null; + return { workspaceId: record.workspaceId.trim(), browserId: browserId || null }; +} + +function readAgentActiveBrowserInput( + input: unknown, +): { agentId: string; browserId: string | null } | null { + if (typeof input !== "object" || input === null || Array.isArray(input)) { + return null; + } + const record = input as Record; + if (typeof record.agentId !== "string" || record.agentId.trim().length === 0) { + return null; + } + const browserId = typeof record.browserId === "string" ? record.browserId.trim() : null; + return { agentId: record.agentId.trim(), browserId: browserId || null }; +} + const pendingBrowserWebviewIds: string[] = []; function isBrowserRefreshInput(input: Electron.Input): boolean { @@ -254,8 +301,25 @@ ipcMain.handle("paseo:get-pending-open-project", (event) => { return result; }); -ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, browserId: unknown) => { - setWorkspaceActivePaseoBrowserId(typeof browserId === "string" ? browserId : null); +ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: unknown) => { + const input = readBrowserWorkspaceInput(rawInput); + if (input) { + registerPaseoBrowserWorkspace(input); + } +}); + +ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput: unknown) => { + const input = readActiveBrowserInput(rawInput); + if (input) { + setWorkspaceActivePaseoBrowserId(input); + } +}); + +ipcMain.handle("paseo:browser:set-agent-active-browser", (_event, rawInput: unknown) => { + const input = readAgentActiveBrowserInput(rawInput); + if (input) { + setAgentActivePaseoBrowserId(input); + } }); ipcMain.handle("paseo:browser:open-devtools", (_event, browserId: unknown) => { @@ -696,6 +760,7 @@ async function bootstrap(): Promise { registerNotificationHandlers(); registerOpenerHandlers(); registerEditorTargetHandlers(); + registerBrowserAutomationIpc(); // In-app "Open in new window": opens a window that lands on the given project // via the same open-project flow as a CLI launch (no move, no ownership). diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index 60e78874b..d0bbd5dfe 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -74,11 +74,17 @@ contextBridge.exposeInMainWorld("paseoDesktop", { ipcRenderer.invoke("paseo:menu:showContextMenu", input), }, browser: { - setWorkspaceActiveBrowser: (browserId: string | null) => - ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", browserId), + registerWorkspaceBrowser: (input: { browserId: string; workspaceId: string }) => + ipcRenderer.invoke("paseo:browser:register-workspace-browser", input), + setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) => + ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input), + setAgentActiveBrowser: (input: { agentId: string; browserId: string | null }) => + ipcRenderer.invoke("paseo:browser:set-agent-active-browser", input), openDevTools: (browserId: string) => ipcRenderer.invoke("paseo:browser:open-devtools", browserId), clearPartition: (browserId: string) => ipcRenderer.invoke("paseo:browser:clear-partition", browserId), + executeAutomationCommand: (request: Record) => + ipcRenderer.invoke("paseo:browser:execute-automation-command", request), }, }); diff --git a/packages/protocol/src/browser-automation/rpc-schemas.test.ts b/packages/protocol/src/browser-automation/rpc-schemas.test.ts new file mode 100644 index 000000000..52dba84aa --- /dev/null +++ b/packages/protocol/src/browser-automation/rpc-schemas.test.ts @@ -0,0 +1,750 @@ +import { describe, expect, test } from "vitest"; + +import { + BrowserAutomationExecuteRequestSchema, + BrowserAutomationExecuteResponseSchema, +} from "./rpc-schemas.js"; + +describe("browser automation execute RPC schemas", () => { + test("rejects navigate and download commands for non-http URLs", () => { + for (const command of ["navigate", "download"] as const) { + expect(() => + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: `req-${command}`, + command: { + command, + args: { url: "file:///tmp/secret.txt" }, + }, + }), + ).toThrow(); + } + }); + + test("parses list tabs requests with top-level correlation and typed command args", () => { + const parsed = BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-1", + workspaceId: "workspace-1", + command: { + command: "list_tabs", + args: { workspaceId: "workspace-1" }, + }, + }); + + expect(parsed).toEqual({ + type: "browser.automation.execute.request", + requestId: "req-1", + workspaceId: "workspace-1", + command: { + command: "list_tabs", + args: { workspaceId: "workspace-1" }, + }, + }); + }); + + test("parses new tab requests and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-new-tab", + workspaceId: "workspace-1", + command: { + command: "new_tab", + args: { workspaceId: "workspace-1", url: "https://example.com" }, + }, + }).command, + ).toEqual({ + command: "new_tab", + args: { workspaceId: "workspace-1", url: "https://example.com" }, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-new-tab", + ok: true, + result: { + command: "new_tab", + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com", + }, + }, + }).payload, + ).toEqual({ + requestId: "req-new-tab", + ok: true, + result: { + command: "new_tab", + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com", + }, + }); + }); + + test("parses page info responses with result data under payload", () => { + const parsed = BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: true, + result: { + command: "page_info", + tab: { + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + }, + }, + }, + }); + + expect(parsed.payload).toEqual({ + requestId: "req-1", + ok: true, + result: { + command: "page_info", + tab: { + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + isActive: false, + isLoading: false, + }, + }, + }); + }); + + test("parses snapshot requests and ref-bearing snapshot responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-snapshot", + workspaceId: "workspace-1", + command: { + command: "snapshot", + args: { workspaceId: "workspace-1", browserId: "browser-1" }, + }, + }), + ).toEqual({ + type: "browser.automation.execute.request", + requestId: "req-snapshot", + workspaceId: "workspace-1", + command: { + command: "snapshot", + args: { workspaceId: "workspace-1", browserId: "browser-1" }, + }, + }); + + const parsed = BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-snapshot", + ok: true, + result: { + command: "snapshot", + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com/form", + title: "Fixture", + elements: [ + { + ref: "@e1", + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + ], + }, + }, + }); + + expect(parsed.payload).toEqual({ + requestId: "req-snapshot", + ok: true, + result: { + command: "snapshot", + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com/form", + title: "Fixture", + elements: [ + { + ref: "@e1", + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name", type: "text" }, + }, + ], + }, + }); + }); + + test("parses click and fill ref commands", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-click", + command: { + command: "click", + args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" }, + }, + }).command, + ).toEqual({ + command: "click", + args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" }, + }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-fill", + command: { + command: "fill", + args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" }, + }, + }).command, + ).toEqual({ + command: "fill", + args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" }, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-fill", + ok: true, + result: { command: "fill", browserId: "browser-1", ref: "@e2" }, + }, + }).payload, + ).toEqual({ + requestId: "req-fill", + ok: true, + result: { command: "fill", browserId: "browser-1", ref: "@e2" }, + }); + }); + + test("parses wait text commands and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-wait", + command: { + command: "wait", + args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 }, + }, + }).command, + ).toEqual({ + command: "wait", + args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 }, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-wait", + ok: true, + result: { command: "wait", browserId: "browser-1", matched: "text" }, + }, + }).payload, + ).toEqual({ + requestId: "req-wait", + ok: true, + result: { command: "wait", browserId: "browser-1", matched: "text" }, + }); + }); + + test("parses set background commands and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-bg", + command: { + command: "set_background", + args: { workspaceId: "workspace-1", color: "red" }, + }, + }).command, + ).toEqual({ + command: "set_background", + args: { workspaceId: "workspace-1", color: "red" }, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-bg", + ok: true, + result: { command: "set_background", browserId: "browser-1", color: "red" }, + }, + }).payload, + ).toEqual({ + requestId: "req-bg", + ok: true, + result: { command: "set_background", browserId: "browser-1", color: "red" }, + }); + }); + + test("parses type and keypress commands", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-type", + command: { command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } }, + }).command, + ).toEqual({ command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-keypress", + command: { command: "keypress", args: { browserId: "browser-1", key: "Enter" } }, + }).command, + ).toEqual({ command: "keypress", args: { browserId: "browser-1", key: "Enter" } }); + }); + + test("parses navigation commands", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-nav", + command: { + command: "navigate", + args: { browserId: "browser-1", url: "https://example.com/next" }, + }, + }).command, + ).toEqual({ + command: "navigate", + args: { browserId: "browser-1", url: "https://example.com/next" }, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-nav", + ok: true, + result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" }, + }, + }).payload, + ).toEqual({ + requestId: "req-nav", + ok: true, + result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" }, + }); + }); + + test("parses screenshot responses", () => { + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-shot", + ok: true, + result: { + command: "screenshot", + browserId: "browser-1", + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 100, + height: 50, + }, + }, + }).payload, + ).toEqual({ + requestId: "req-shot", + ok: true, + result: { + command: "screenshot", + browserId: "browser-1", + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 100, + height: 50, + }, + }); + }); + + test("parses form control commands and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-focus", + command: { command: "focus", args: { browserId: "browser-1", ref: "@e1" } }, + }).command, + ).toEqual({ command: "focus", args: { browserId: "browser-1", ref: "@e1" } }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-clear", + command: { command: "clear", args: { browserId: "browser-1", ref: "@e1" } }, + }).command, + ).toEqual({ command: "clear", args: { browserId: "browser-1", ref: "@e1" } }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-check", + command: { command: "check", args: { browserId: "browser-1", ref: "@e2" } }, + }).command, + ).toEqual({ command: "check", args: { browserId: "browser-1", ref: "@e2", checked: true } }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-select", + command: { command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } }, + }).command, + ).toEqual({ command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-hover", + command: { command: "hover", args: { browserId: "browser-1", ref: "@e4" } }, + }).command, + ).toEqual({ command: "hover", args: { browserId: "browser-1", ref: "@e4" } }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-drag", + command: { + command: "drag", + args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" }, + }, + }).command, + ).toEqual({ + command: "drag", + args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" }, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-select", + ok: true, + result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" }, + }, + }).payload, + ).toEqual({ + requestId: "req-select", + ok: true, + result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" }, + }); + }); + + test("parses browser log commands and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-logs", + command: { command: "logs", args: { browserId: "browser-1" } }, + }).command, + ).toEqual({ command: "logs", args: { browserId: "browser-1", maxEntries: 50 } }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-logs", + ok: true, + result: { + command: "logs", + browserId: "browser-1", + console: [{ level: "info", message: "ready", timestamp: 10 }], + network: [ + { + url: "https://example.com/app.js", + type: "script", + startTime: 1, + duration: 2, + }, + ], + }, + }, + }).payload, + ).toEqual({ + requestId: "req-logs", + ok: true, + result: { + command: "logs", + browserId: "browser-1", + console: [{ level: "info", message: "ready", timestamp: 10 }], + network: [ + { + url: "https://example.com/app.js", + type: "script", + startTime: 1, + duration: 2, + }, + ], + }, + }); + }); + + test("parses browser storage commands and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-storage", + command: { command: "storage", args: { browserId: "browser-1" } }, + }).command, + ).toEqual({ command: "storage", args: { browserId: "browser-1" } }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-storage", + ok: true, + result: { + command: "storage", + browserId: "browser-1", + url: "https://example.com", + cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }, + }, + }).payload, + ).toEqual({ + requestId: "req-storage", + ok: true, + result: { + command: "storage", + browserId: "browser-1", + url: "https://example.com", + cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }, + }); + }); + + test("parses browser environment commands and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-environment", + command: { + command: "environment", + args: { + browserId: "browser-1", + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + }).command, + ).toEqual({ + command: "environment", + args: { + browserId: "browser-1", + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-environment", + ok: true, + result: { + command: "environment", + browserId: "browser-1", + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + }).payload, + ).toEqual({ + requestId: "req-environment", + ok: true, + result: { + command: "environment", + browserId: "browser-1", + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }); + }); + + test("parses browser full-page screenshot and PDF commands and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-full-page", + command: { command: "full_page_screenshot", args: { browserId: "browser-1" } }, + }).command, + ).toEqual({ command: "full_page_screenshot", args: { browserId: "browser-1" } }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-pdf", + command: { command: "pdf", args: { browserId: "browser-1" } }, + }).command, + ).toEqual({ command: "pdf", args: { browserId: "browser-1", printBackground: true } }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-full-page", + ok: true, + result: { + command: "full_page_screenshot", + browserId: "browser-1", + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 390, + height: 1200, + }, + }, + }).payload.result, + ).toEqual({ + command: "full_page_screenshot", + browserId: "browser-1", + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 390, + height: 1200, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-pdf", + ok: true, + result: { + command: "pdf", + browserId: "browser-1", + mimeType: "application/pdf", + dataBase64: "JVBERi0xLjQ=", + }, + }, + }).payload.result, + ).toEqual({ + command: "pdf", + browserId: "browser-1", + mimeType: "application/pdf", + dataBase64: "JVBERi0xLjQ=", + }); + }); + + test("parses browser download and upload commands and responses", () => { + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-download", + command: { + command: "download", + args: { browserId: "browser-1", url: "https://example.com/file.txt" }, + }, + }).command, + ).toEqual({ + command: "download", + args: { browserId: "browser-1", url: "https://example.com/file.txt" }, + }); + + expect( + BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-upload", + command: { + command: "upload", + args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] }, + }, + }).command, + ).toEqual({ + command: "upload", + args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] }, + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-download", + ok: true, + result: { + command: "download", + browserId: "browser-1", + url: "https://example.com/file.txt", + filePath: "/tmp/file.txt", + totalBytes: 5, + state: "completed", + }, + }, + }).payload.result, + ).toEqual({ + command: "download", + browserId: "browser-1", + url: "https://example.com/file.txt", + filePath: "/tmp/file.txt", + totalBytes: 5, + state: "completed", + }); + + expect( + BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-upload", + ok: true, + result: { + command: "upload", + browserId: "browser-1", + ref: "@e1", + filePaths: ["/tmp/file.txt"], + }, + }, + }).payload.result, + ).toEqual({ + command: "upload", + browserId: "browser-1", + ref: "@e1", + filePaths: ["/tmp/file.txt"], + }); + }); + + test("parses stable model-actionable error responses", () => { + const parsed = BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: false, + error: { + code: "browser_no_desktop", + message: "No desktop browser automation client is connected.", + }, + }, + }); + + expect(parsed.payload).toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_no_desktop", + message: "No desktop browser automation client is connected.", + retryable: false, + }, + }); + }); +}); diff --git a/packages/protocol/src/browser-automation/rpc-schemas.ts b/packages/protocol/src/browser-automation/rpc-schemas.ts new file mode 100644 index 000000000..4d741e8e6 --- /dev/null +++ b/packages/protocol/src/browser-automation/rpc-schemas.ts @@ -0,0 +1,599 @@ +import { z } from "zod"; + +export const BrowserAutomationErrorCodeSchema = z.enum([ + "browser_disabled", + "browser_no_desktop", + "browser_no_tab", + "browser_tab_not_found", + "browser_tab_closed", + "browser_timeout", + "browser_denied", + "browser_unsupported", + "browser_stale_ref", + "browser_unknown_error", +]); + +const BrowserAutomationTabTargetSchema = z.object({ + workspaceId: z.string().min(1).optional(), + browserId: z.string().min(1).optional(), +}); + +const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/); +const BrowserAutomationHttpUrlSchema = z + .string() + .url() + .refine((value) => { + const protocol = new URL(value).protocol; + return protocol === "http:" || protocol === "https:"; + }, "URL must use http or https"); + +export const BrowserAutomationListTabsCommandSchema = z.object({ + command: z.literal("list_tabs"), + args: z + .object({ + workspaceId: z.string().min(1).optional(), + }) + .default({}), +}); + +export const BrowserAutomationNewTabCommandSchema = z.object({ + command: z.literal("new_tab"), + args: z + .object({ + workspaceId: z.string().min(1).optional(), + url: BrowserAutomationHttpUrlSchema.optional(), + }) + .default({}), +}); + +export const BrowserAutomationPageInfoCommandSchema = z.object({ + command: z.literal("page_info"), + args: BrowserAutomationTabTargetSchema.default({}), +}); + +export const BrowserAutomationSnapshotCommandSchema = z.object({ + command: z.literal("snapshot"), + args: BrowserAutomationTabTargetSchema.default({}), +}); + +export const BrowserAutomationClickCommandSchema = z.object({ + command: z.literal("click"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema, + }), +}); + +export const BrowserAutomationFillCommandSchema = z.object({ + command: z.literal("fill"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema, + value: z.string(), + }), +}); + +export const BrowserAutomationWaitCommandSchema = z.object({ + command: z.literal("wait"), + args: BrowserAutomationTabTargetSchema.extend({ + text: z.string().min(1).optional(), + url: z.string().min(1).optional(), + timeoutMs: z.number().int().positive().max(30_000).optional(), + }), +}); + +export const BrowserAutomationTypeCommandSchema = z.object({ + command: z.literal("type"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema.optional(), + text: z.string(), + }), +}); + +export const BrowserAutomationKeypressCommandSchema = z.object({ + command: z.literal("keypress"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema.optional(), + key: z.string().min(1), + }), +}); + +export const BrowserAutomationNavigateCommandSchema = z.object({ + command: z.literal("navigate"), + args: BrowserAutomationTabTargetSchema.extend({ + url: BrowserAutomationHttpUrlSchema, + }), +}); + +export const BrowserAutomationBackCommandSchema = z.object({ + command: z.literal("back"), + args: BrowserAutomationTabTargetSchema.default({}), +}); + +export const BrowserAutomationForwardCommandSchema = z.object({ + command: z.literal("forward"), + args: BrowserAutomationTabTargetSchema.default({}), +}); + +export const BrowserAutomationReloadCommandSchema = z.object({ + command: z.literal("reload"), + args: BrowserAutomationTabTargetSchema.default({}), +}); + +export const BrowserAutomationScreenshotCommandSchema = z.object({ + command: z.literal("screenshot"), + args: BrowserAutomationTabTargetSchema.default({}), +}); + +export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({ + command: z.literal("full_page_screenshot"), + args: BrowserAutomationTabTargetSchema.default({}), +}); + +export const BrowserAutomationPdfCommandSchema = z.object({ + command: z.literal("pdf"), + args: BrowserAutomationTabTargetSchema.extend({ + landscape: z.boolean().optional(), + printBackground: z.boolean().default(true), + }).default({ printBackground: true }), +}); + +export const BrowserAutomationDownloadCommandSchema = z.object({ + command: z.literal("download"), + args: BrowserAutomationTabTargetSchema.extend({ + url: BrowserAutomationHttpUrlSchema, + fileName: z.string().min(1).optional(), + }), +}); + +export const BrowserAutomationUploadCommandSchema = z.object({ + command: z.literal("upload"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema, + filePaths: z.array(z.string().min(1)).min(1), + }), +}); + +export const BrowserAutomationFocusCommandSchema = z.object({ + command: z.literal("focus"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema, + }), +}); + +export const BrowserAutomationClearCommandSchema = z.object({ + command: z.literal("clear"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema, + }), +}); + +export const BrowserAutomationCheckCommandSchema = z.object({ + command: z.literal("check"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema, + checked: z.boolean().default(true), + }), +}); + +export const BrowserAutomationSelectCommandSchema = z.object({ + command: z.literal("select"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema, + value: z.string(), + }), +}); + +export const BrowserAutomationHoverCommandSchema = z.object({ + command: z.literal("hover"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema, + }), +}); + +export const BrowserAutomationDragCommandSchema = z.object({ + command: z.literal("drag"), + args: BrowserAutomationTabTargetSchema.extend({ + sourceRef: BrowserAutomationRefSchema, + targetRef: BrowserAutomationRefSchema, + }), +}); + +export const BrowserAutomationLogsCommandSchema = z.object({ + command: z.literal("logs"), + args: BrowserAutomationTabTargetSchema.extend({ + maxEntries: z.number().int().positive().max(200).default(50), + }).default({ maxEntries: 50 }), +}); + +export const BrowserAutomationStorageCommandSchema = z.object({ + command: z.literal("storage"), + args: BrowserAutomationTabTargetSchema.default({}), +}); + +const BrowserAutomationViewportInputSchema = z.object({ + width: z.number().int().positive(), + height: z.number().int().positive(), + deviceScaleFactor: z.number().positive().optional(), +}); + +const BrowserAutomationGeolocationInputSchema = z.object({ + latitude: z.number().min(-90).max(90), + longitude: z.number().min(-180).max(180), + accuracy: z.number().positive().optional(), +}); + +export const BrowserAutomationEnvironmentCommandSchema = z.object({ + command: z.literal("environment"), + args: BrowserAutomationTabTargetSchema.extend({ + viewport: BrowserAutomationViewportInputSchema.optional(), + geolocation: BrowserAutomationGeolocationInputSchema.optional(), + }).default({}), +}); + +export const BrowserAutomationSetBackgroundCommandSchema = z.object({ + command: z.literal("set_background"), + args: BrowserAutomationTabTargetSchema.extend({ + color: z.string().min(1), + }), +}); + +export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [ + BrowserAutomationListTabsCommandSchema, + BrowserAutomationNewTabCommandSchema, + BrowserAutomationPageInfoCommandSchema, + BrowserAutomationSnapshotCommandSchema, + BrowserAutomationClickCommandSchema, + BrowserAutomationFillCommandSchema, + BrowserAutomationWaitCommandSchema, + BrowserAutomationTypeCommandSchema, + BrowserAutomationKeypressCommandSchema, + BrowserAutomationNavigateCommandSchema, + BrowserAutomationBackCommandSchema, + BrowserAutomationForwardCommandSchema, + BrowserAutomationReloadCommandSchema, + BrowserAutomationScreenshotCommandSchema, + BrowserAutomationFullPageScreenshotCommandSchema, + BrowserAutomationPdfCommandSchema, + BrowserAutomationDownloadCommandSchema, + BrowserAutomationUploadCommandSchema, + BrowserAutomationFocusCommandSchema, + BrowserAutomationClearCommandSchema, + BrowserAutomationCheckCommandSchema, + BrowserAutomationSelectCommandSchema, + BrowserAutomationHoverCommandSchema, + BrowserAutomationDragCommandSchema, + BrowserAutomationLogsCommandSchema, + BrowserAutomationStorageCommandSchema, + BrowserAutomationEnvironmentCommandSchema, + BrowserAutomationSetBackgroundCommandSchema, +]); + +export const BrowserAutomationTabInfoSchema = z.object({ + browserId: z.string().min(1), + workspaceId: z.string().min(1).optional(), + url: z.string(), + title: z.string(), + isActive: z.boolean().default(false), + isLoading: z.boolean().default(false), + canGoBack: z.boolean().optional(), + canGoForward: z.boolean().optional(), +}); + +export const BrowserAutomationListTabsResultSchema = z.object({ + command: z.literal("list_tabs"), + tabs: z.array(BrowserAutomationTabInfoSchema), +}); + +export const BrowserAutomationNewTabResultSchema = z.object({ + command: z.literal("new_tab"), + browserId: z.string().min(1), + workspaceId: z.string().min(1), + url: z.string().min(1), +}); + +export const BrowserAutomationPageInfoResultSchema = z.object({ + command: z.literal("page_info"), + tab: BrowserAutomationTabInfoSchema, +}); + +export const BrowserAutomationSnapshotElementSchema = z.object({ + ref: z.string().regex(/^@e\d+$/), + role: z.string(), + tagName: z.string(), + text: z.string(), + selector: z.string(), + attributes: z.record(z.string(), z.string()).default({}), +}); + +export const BrowserAutomationSnapshotResultSchema = z.object({ + command: z.literal("snapshot"), + browserId: z.string().min(1), + workspaceId: z.string().min(1).optional(), + url: z.string(), + title: z.string(), + elements: z.array(BrowserAutomationSnapshotElementSchema), +}); + +export const BrowserAutomationClickResultSchema = z.object({ + command: z.literal("click"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema, +}); + +export const BrowserAutomationFillResultSchema = z.object({ + command: z.literal("fill"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema, +}); + +export const BrowserAutomationWaitResultSchema = z.object({ + command: z.literal("wait"), + browserId: z.string().min(1), + matched: z.enum(["text", "url"]), +}); + +export const BrowserAutomationTypeResultSchema = z.object({ + command: z.literal("type"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema.optional(), +}); + +export const BrowserAutomationKeypressResultSchema = z.object({ + command: z.literal("keypress"), + browserId: z.string().min(1), + key: z.string().min(1), + ref: BrowserAutomationRefSchema.optional(), +}); + +export const BrowserAutomationNavigateResultSchema = z.object({ + command: z.literal("navigate"), + browserId: z.string().min(1), + url: z.string().min(1), +}); + +export const BrowserAutomationBackResultSchema = z.object({ + command: z.literal("back"), + browserId: z.string().min(1), +}); + +export const BrowserAutomationForwardResultSchema = z.object({ + command: z.literal("forward"), + browserId: z.string().min(1), +}); + +export const BrowserAutomationReloadResultSchema = z.object({ + command: z.literal("reload"), + browserId: z.string().min(1), +}); + +export const BrowserAutomationScreenshotResultSchema = z.object({ + command: z.literal("screenshot"), + browserId: z.string().min(1), + mimeType: z.literal("image/png"), + dataBase64: z.string().min(1), + width: z.number().int().nonnegative(), + height: z.number().int().nonnegative(), +}); + +export const BrowserAutomationFullPageScreenshotResultSchema = z.object({ + command: z.literal("full_page_screenshot"), + browserId: z.string().min(1), + mimeType: z.literal("image/png"), + dataBase64: z.string().min(1), + width: z.number().int().nonnegative(), + height: z.number().int().nonnegative(), +}); + +export const BrowserAutomationPdfResultSchema = z.object({ + command: z.literal("pdf"), + browserId: z.string().min(1), + mimeType: z.literal("application/pdf"), + dataBase64: z.string().min(1), +}); + +export const BrowserAutomationDownloadResultSchema = z.object({ + command: z.literal("download"), + browserId: z.string().min(1), + url: z.string().min(1), + filePath: z.string().min(1), + totalBytes: z.number().int().nonnegative().optional(), + state: z.string().min(1), +}); + +export const BrowserAutomationUploadResultSchema = z.object({ + command: z.literal("upload"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema, + filePaths: z.array(z.string().min(1)).min(1), +}); + +export const BrowserAutomationFocusResultSchema = z.object({ + command: z.literal("focus"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema, +}); + +export const BrowserAutomationClearResultSchema = z.object({ + command: z.literal("clear"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema, +}); + +export const BrowserAutomationCheckResultSchema = z.object({ + command: z.literal("check"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema, + checked: z.boolean(), +}); + +export const BrowserAutomationSelectResultSchema = z.object({ + command: z.literal("select"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema, + value: z.string(), +}); + +export const BrowserAutomationHoverResultSchema = z.object({ + command: z.literal("hover"), + browserId: z.string().min(1), + ref: BrowserAutomationRefSchema, +}); + +export const BrowserAutomationDragResultSchema = z.object({ + command: z.literal("drag"), + browserId: z.string().min(1), + sourceRef: BrowserAutomationRefSchema, + targetRef: BrowserAutomationRefSchema, +}); + +export const BrowserAutomationConsoleLogEntrySchema = z.object({ + level: z.string(), + message: z.string(), + source: z.string().optional(), + line: z.number().int().optional(), + timestamp: z.number(), +}); + +export const BrowserAutomationNetworkLogEntrySchema = z.object({ + url: z.string(), + method: z.string().optional(), + status: z.number().int().optional(), + type: z.string().optional(), + startTime: z.number(), + duration: z.number(), + transferSize: z.number().optional(), +}); + +export const BrowserAutomationLogsResultSchema = z.object({ + command: z.literal("logs"), + browserId: z.string().min(1), + console: z.array(BrowserAutomationConsoleLogEntrySchema), + network: z.array(BrowserAutomationNetworkLogEntrySchema), +}); + +export const BrowserAutomationCookieEntrySchema = z.object({ + name: z.string(), + value: z.string(), + domain: z.string().optional(), + path: z.string().optional(), + secure: z.boolean().optional(), + httpOnly: z.boolean().optional(), + expirationDate: z.number().optional(), +}); + +export const BrowserAutomationStorageEntrySchema = z.object({ + key: z.string(), + value: z.string(), +}); + +export const BrowserAutomationStorageResultSchema = z.object({ + command: z.literal("storage"), + browserId: z.string().min(1), + url: z.string(), + cookies: z.array(BrowserAutomationCookieEntrySchema), + localStorage: z.array(BrowserAutomationStorageEntrySchema), + sessionStorage: z.array(BrowserAutomationStorageEntrySchema), +}); + +export const BrowserAutomationViewportResultSchema = z.object({ + width: z.number().int().nonnegative(), + height: z.number().int().nonnegative(), + deviceScaleFactor: z.number().positive(), +}); + +export const BrowserAutomationGeolocationResultSchema = z.object({ + latitude: z.number(), + longitude: z.number(), + accuracy: z.number(), +}); + +export const BrowserAutomationEnvironmentResultSchema = z.object({ + command: z.literal("environment"), + browserId: z.string().min(1), + viewport: BrowserAutomationViewportResultSchema, + geolocation: BrowserAutomationGeolocationResultSchema.optional(), +}); + +export const BrowserAutomationSetBackgroundResultSchema = z.object({ + command: z.literal("set_background"), + browserId: z.string().min(1), + color: z.string().min(1), +}); + +export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [ + BrowserAutomationListTabsResultSchema, + BrowserAutomationNewTabResultSchema, + BrowserAutomationPageInfoResultSchema, + BrowserAutomationSnapshotResultSchema, + BrowserAutomationClickResultSchema, + BrowserAutomationFillResultSchema, + BrowserAutomationWaitResultSchema, + BrowserAutomationTypeResultSchema, + BrowserAutomationKeypressResultSchema, + BrowserAutomationNavigateResultSchema, + BrowserAutomationBackResultSchema, + BrowserAutomationForwardResultSchema, + BrowserAutomationReloadResultSchema, + BrowserAutomationScreenshotResultSchema, + BrowserAutomationFullPageScreenshotResultSchema, + BrowserAutomationPdfResultSchema, + BrowserAutomationDownloadResultSchema, + BrowserAutomationUploadResultSchema, + BrowserAutomationFocusResultSchema, + BrowserAutomationClearResultSchema, + BrowserAutomationCheckResultSchema, + BrowserAutomationSelectResultSchema, + BrowserAutomationHoverResultSchema, + BrowserAutomationDragResultSchema, + BrowserAutomationLogsResultSchema, + BrowserAutomationStorageResultSchema, + BrowserAutomationEnvironmentResultSchema, + BrowserAutomationSetBackgroundResultSchema, +]); + +export const BrowserAutomationErrorSchema = z.object({ + code: BrowserAutomationErrorCodeSchema, + message: z.string().min(1), + retryable: z.boolean().default(false), +}); + +export const BrowserAutomationExecuteRequestSchema = z.object({ + type: z.literal("browser.automation.execute.request"), + requestId: z.string().min(1), + agentId: z.string().min(1).optional(), + cwd: z.string().min(1).optional(), + workspaceId: z.string().min(1).optional(), + browserId: z.string().min(1).optional(), + command: BrowserAutomationCommandSchema, +}); + +export const BrowserAutomationExecuteResponseSchema = z.object({ + type: z.literal("browser.automation.execute.response"), + payload: z.discriminatedUnion("ok", [ + z.object({ + requestId: z.string().min(1), + ok: z.literal(true), + result: BrowserAutomationResultSchema, + }), + z.object({ + requestId: z.string().min(1), + ok: z.literal(false), + error: BrowserAutomationErrorSchema, + }), + ]), +}); + +export type BrowserAutomationErrorCode = z.infer; +export type BrowserAutomationCommand = z.infer; +export type BrowserAutomationResult = z.infer; +export type BrowserAutomationConsoleLogEntry = z.infer< + typeof BrowserAutomationConsoleLogEntrySchema +>; +export type BrowserAutomationNetworkLogEntry = z.infer< + typeof BrowserAutomationNetworkLogEntrySchema +>; +export type BrowserAutomationCookieEntry = z.infer; +export type BrowserAutomationStorageEntry = z.infer; +export type BrowserAutomationExecuteRequest = z.infer; +export type BrowserAutomationExecuteResponse = z.infer< + typeof BrowserAutomationExecuteResponseSchema +>; diff --git a/packages/protocol/src/client-capabilities.ts b/packages/protocol/src/client-capabilities.ts index 1e82e5bab..3ec353439 100644 --- a/packages/protocol/src/client-capabilities.ts +++ b/packages/protocol/src/client-capabilities.ts @@ -11,6 +11,7 @@ export const CLIENT_CAPS = { // Old clients use a strict TerminalState schema and would reject the extra fields. // Drop the gate (always send the flags) when floor >= v0.1.88. terminalReflowableSnapshot: "terminal_reflowable_snapshot", + desktopBrowserAutomation: "desktop_browser_automation", } as const; export type ClientCapability = (typeof CLIENT_CAPS)[keyof typeof CLIENT_CAPS]; diff --git a/packages/protocol/src/messages.browser-automation.test.ts b/packages/protocol/src/messages.browser-automation.test.ts new file mode 100644 index 000000000..891565a0d --- /dev/null +++ b/packages/protocol/src/messages.browser-automation.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "vitest"; + +import { CLIENT_CAPS } from "./client-capabilities.js"; +import { + MutableDaemonConfigPatchSchema, + MutableDaemonConfigSchema, + SessionInboundMessageSchema, + SessionOutboundMessageSchema, + WSHelloMessageSchema, +} from "./messages.js"; + +describe("browser automation protocol integration", () => { + test("desktop automation capability parses in hello without narrowing old clients", () => { + expect( + WSHelloMessageSchema.parse({ + type: "hello", + clientId: "client-1", + clientType: "mobile", + protocolVersion: 1, + capabilities: { + [CLIENT_CAPS.desktopBrowserAutomation]: true, + }, + }).capabilities, + ).toMatchObject({ + [CLIENT_CAPS.desktopBrowserAutomation]: true, + }); + + expect( + WSHelloMessageSchema.parse({ + type: "hello", + clientId: "old-client", + clientType: "mobile", + protocolVersion: 1, + }).capabilities, + ).toBeUndefined(); + }); + + test("daemon to desktop execute request is an outbound session message", () => { + const parsed = SessionOutboundMessageSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-1", + command: { command: "page_info", args: { browserId: "browser-1" } }, + }); + + expect(parsed.type).toBe("browser.automation.execute.request"); + }); + + test("desktop to daemon execute response is an inbound session message", () => { + const parsed = SessionInboundMessageSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }, + }); + + expect(parsed.type).toBe("browser.automation.execute.response"); + }); + + test("mutable daemon config defaults browser tools off and accepts opt-in patches", () => { + expect( + MutableDaemonConfigSchema.parse({ + mcp: { injectIntoAgents: false }, + }).browserTools, + ).toEqual({ enabled: false }); + + expect( + MutableDaemonConfigPatchSchema.parse({ + browserTools: { enabled: true }, + }).browserTools, + ).toEqual({ enabled: true }); + }); +}); diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 7eb0edaa1..498c2caf7 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -53,6 +53,10 @@ import { LoopLogsResponseSchema, LoopStopResponseSchema, } from "@getpaseo/protocol/loop/rpc-schemas"; +import { + BrowserAutomationExecuteRequestSchema, + BrowserAutomationExecuteResponseSchema, +} from "./browser-automation/rpc-schemas.js"; import { PaseoConfigRawSchema, PaseoLifecycleCommandRawSchema, @@ -129,6 +133,11 @@ export const TerminalProfileSchema = z export type TerminalProfile = z.infer; +const MutableBrowserToolsConfigSchema = z + .object({ + enabled: z.boolean().default(false), + }) + .passthrough(); export const MutableDaemonConfigSchema = z .object({ mcp: z @@ -136,6 +145,7 @@ export const MutableDaemonConfigSchema = z injectIntoAgents: z.boolean(), }) .passthrough(), + browserTools: MutableBrowserToolsConfigSchema.default({ enabled: false }), providers: z.record(z.string(), MutableDaemonProviderConfigSchema).default({}), metadataGeneration: MutableMetadataGenerationConfigSchema.default({ providers: [] }), autoArchiveAfterMerge: z.boolean().default(false), @@ -148,6 +158,7 @@ export const MutableDaemonConfigSchema = z export const MutableDaemonConfigPatchSchema = z .object({ mcp: MutableDaemonConfigSchema.shape.mcp.partial().optional(), + browserTools: MutableBrowserToolsConfigSchema.partial().optional(), providers: z .record(z.string(), MutableDaemonProviderConfigSchema.partial().passthrough()) .optional(), @@ -2027,6 +2038,7 @@ export const CaptureTerminalRequestSchema = z.object({ }); export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ + BrowserAutomationExecuteResponseSchema, VoiceAudioChunkMessageSchema, AbortRequestMessageSchema, AudioPlayedMessageSchema, @@ -4153,6 +4165,7 @@ export const DaemonUpdateProgressMessageSchema = z.object({ export type DaemonUpdateProgressMessage = z.infer; export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ + BrowserAutomationExecuteRequestSchema, ActivityLogMessageSchema, AssistantChunkMessageSchema, AudioOutputMessageSchema, @@ -4658,6 +4671,7 @@ export const WSHelloMessageSchema = z.object({ [CLIENT_CAPS.reasoningMergeEnum]: z.boolean().optional(), [CLIENT_CAPS.customModeIcons]: z.boolean().optional(), [CLIENT_CAPS.terminalReflowableSnapshot]: z.boolean().optional(), + [CLIENT_CAPS.desktopBrowserAutomation]: z.boolean().optional(), }) .passthrough() .optional(), diff --git a/packages/relay/src/e2e.test.ts b/packages/relay/src/e2e.test.ts index 00f800f4a..663abf2e4 100644 --- a/packages/relay/src/e2e.test.ts +++ b/packages/relay/src/e2e.test.ts @@ -4,6 +4,8 @@ import net from "node:net"; import { spawn, type ChildProcess } from "node:child_process"; import { createRequire } from "node:module"; import { Buffer } from "node:buffer"; +import { dirname, resolve as resolvePath } from "node:path"; +import { fileURLToPath } from "node:url"; import { generateKeyPair, exportPublicKey, @@ -16,6 +18,7 @@ import { const nodeMajor = Number((process.versions.node ?? "0").split(".")[0] ?? "0"); const shouldRunRelayE2e = process.env.FORCE_RELAY_E2E === "1" || nodeMajor < 25; const wranglerCliPath = createRequire(import.meta.url).resolve("wrangler/bin/wrangler.js"); +const relayPackageRoot = resolvePath(dirname(fileURLToPath(import.meta.url)), ".."); const STARTUP_HOOK_TIMEOUT_MS = 90_000; const SHUTDOWN_TIMEOUT_MS = 10_000; @@ -61,7 +64,7 @@ function spawnRelayDevServer(port: number): ChildProcess { "--show-interactive-dev-session=false", ], { - cwd: process.cwd(), + cwd: relayPackageRoot, env: { ...process.env }, stdio: ["ignore", "pipe", "pipe"], detached: false, diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index a3e4aa27f..71f2bac33 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -578,6 +578,91 @@ function createPaseoWorktreeForMcpTest(options: { }; } +describe("browser MCP tools", () => { + const logger = createTestLogger(); + + it("keeps browser tools registered when browser tools are disabled", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + spies.agentManager.getAgent.mockReturnValue({ id: "agent-1", cwd: REPO_CWD }); + const execute = vi.fn().mockResolvedValue({ + requestId: "req-browser-disabled", + ok: false, + error: { + code: "browser_disabled", + message: "Browser tools are disabled.", + retryable: false, + }, + }); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + providerSnapshotManager: createOpenCodeManager().manager, + browserToolsBroker: { execute } as never, + callerAgentId: "agent-1", + logger, + }); + const tool = registeredTool(server, "browser_list_tabs"); + + const response = await tool.handler({}); + + expect(lookupTool(server, "browser_page_info")).not.toBeUndefined(); + expect(execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: REPO_CWD, + workspaceId: REPO_CWD, + command: { command: "list_tabs", args: { workspaceId: REPO_CWD } }, + }); + expect(response.structuredContent).toEqual({ + ok: false, + error: { + code: "browser_disabled", + message: "Browser tools are disabled.", + retryable: false, + }, + context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD }, + }); + }); + + it("wires browser tools through the browser tools broker", async () => { + const { agentManager, agentStorage, spies } = createTestDeps(); + spies.agentManager.getAgent.mockReturnValue({ id: "agent-1", cwd: REPO_CWD }); + const execute = vi.fn().mockResolvedValue({ + requestId: "req-browser-tabs", + ok: true, + result: { command: "list_tabs", tabs: [] }, + }); + const server = await createAgentMcpServer({ + agentManager, + agentStorage, + providerSnapshotManager: createOpenCodeManager().manager, + browserToolsBroker: { execute } as never, + callerAgentId: "agent-1", + logger, + }); + const tool = registeredTool(server, "browser_list_tabs"); + + const response = await tool.handler({}); + + expect(execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: REPO_CWD, + workspaceId: REPO_CWD, + command: { command: "list_tabs", args: { workspaceId: REPO_CWD } }, + }); + expect(response.content).toEqual([ + { + type: "text", + text: "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab.", + }, + ]); + expect(response.structuredContent).toEqual({ + ok: true, + result: { command: "list_tabs", tabs: [] }, + context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD }, + }); + }); +}); + describe("terminal MCP tools", () => { const logger = createTestLogger(); diff --git a/packages/server/src/server/agent/providers/claude/tool-call-mapper.test.ts b/packages/server/src/server/agent/providers/claude/tool-call-mapper.test.ts index 531405285..0522d0cd5 100644 --- a/packages/server/src/server/agent/providers/claude/tool-call-mapper.test.ts +++ b/packages/server/src/server/agent/providers/claude/tool-call-mapper.test.ts @@ -280,7 +280,7 @@ describe("claude tool-call mapper", () => { name: "Grep", input: { pattern: '\\\\\\"cli\\\\\\""', - path: "/Users/moboudra/dev/paseo/packages/desktop/src", + path: "/workspaces/paseo/packages/desktop/src", output_mode: "content", "-n": true, }, diff --git a/packages/server/src/server/agent/providers/opencode/tool-call-mapper.test.ts b/packages/server/src/server/agent/providers/opencode/tool-call-mapper.test.ts index f0750f737..fd6725f3f 100644 --- a/packages/server/src/server/agent/providers/opencode/tool-call-mapper.test.ts +++ b/packages/server/src/server/agent/providers/opencode/tool-call-mapper.test.ts @@ -188,9 +188,9 @@ describe("opencode tool-call mapper", () => { toolName: "read", callId: "opencode-read-xml", status: "completed", - input: { filePath: "/Users/moboudra/dev/paseo/docs/release.md" }, + input: { filePath: "/workspaces/paseo/docs/release.md" }, output: [ - "/Users/moboudra/dev/paseo/docs/release.md", + "/workspaces/paseo/docs/release.md", "file", "", "1: # Release", @@ -203,7 +203,7 @@ describe("opencode tool-call mapper", () => { expect(item.detail).toEqual({ type: "read", - filePath: "/Users/moboudra/dev/paseo/docs/release.md", + filePath: "/workspaces/paseo/docs/release.md", content: [ "1: # Release", "2:", @@ -282,8 +282,7 @@ describe("opencode tool-call mapper", () => { callId: "opencode-write-success-text", status: "completed", input: { - filePath: - "/Users/moboudra/dev/paseo/.dev/paseo-home/worktrees/1luy0po7/cold-ladybug/dummy.txt", + filePath: "/workspaces/paseo/worktrees/cold-ladybug/dummy.txt", content: "hello world\n", }, output: "Wrote file successfully.", @@ -292,8 +291,7 @@ describe("opencode tool-call mapper", () => { expect(item.detail).toEqual({ type: "write", - filePath: - "/Users/moboudra/dev/paseo/.dev/paseo-home/worktrees/1luy0po7/cold-ladybug/dummy.txt", + filePath: "/workspaces/paseo/worktrees/cold-ladybug/dummy.txt", content: "hello world\n", }); }); @@ -305,9 +303,9 @@ describe("opencode tool-call mapper", () => { callId: "opencode-edit-camel", status: "completed", input: { - filePath: "/Users/moboudra/dev/paseo/packages/website/src/data/agent-pages.ts", - oldString: 'metaTitle: "Junie agent Mobile and Desktop App, Open Source"', - newString: 'metaTitle: "Junie Agent Mobile and Desktop App, Open Source"', + filePath: "/workspaces/paseo/packages/website/src/data/agent-pages.ts", + oldString: 'metaTitle: "Agent page"', + newString: 'metaTitle: "Updated agent page"', }, output: "Edit applied successfully.", }), @@ -315,9 +313,9 @@ describe("opencode tool-call mapper", () => { expect(item.detail).toEqual({ type: "edit", - filePath: "/Users/moboudra/dev/paseo/packages/website/src/data/agent-pages.ts", - oldString: 'metaTitle: "Junie agent Mobile and Desktop App, Open Source"', - newString: 'metaTitle: "Junie Agent Mobile and Desktop App, Open Source"', + filePath: "/workspaces/paseo/packages/website/src/data/agent-pages.ts", + oldString: 'metaTitle: "Agent page"', + newString: 'metaTitle: "Updated agent page"', }); }); diff --git a/packages/server/src/server/agent/tools/paseo-tools.ts b/packages/server/src/server/agent/tools/paseo-tools.ts index 498d8ada3..75c2eb23f 100644 --- a/packages/server/src/server/agent/tools/paseo-tools.ts +++ b/packages/server/src/server/agent/tools/paseo-tools.ts @@ -73,6 +73,8 @@ import { type CreatePaseoWorktreeCommandInput, listPaseoWorktreesCommand, } from "../../worktree/commands.js"; +import { registerBrowserTools } from "../../browser-tools/tools.js"; +import type { BrowserToolsBroker } from "../../browser-tools/broker.js"; import type { PaseoToolCatalog, PaseoToolConfig, @@ -102,6 +104,7 @@ export interface PaseoToolHostDependencies { createPaseoWorktree?: CreatePaseoWorktreeWorkflowFn; // Mints a fresh directory workspace for a cwd and returns its id. ensureWorkspaceForCreate?: (cwd: string) => Promise; + browserToolsBroker?: BrowserToolsBroker | null; paseoHome?: string; worktreesRoot?: string; /** @@ -1002,6 +1005,15 @@ export function createPaseoToolCatalog(options: PaseoToolHostDependencies): Pase return toCatalog(); } + if (options.browserToolsBroker) { + registerBrowserTools({ + registerTool, + broker: options.browserToolsBroker, + callerAgentId, + resolveCallerAgent, + }); + } + registerTool( "create_agent", { diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index f704c3992..0534af4db 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -118,7 +118,9 @@ import { FileBackedChatService } from "./chat/chat-service.js"; import { CheckoutDiffManager } from "./checkout-diff-manager.js"; import { LoopService } from "./loop-service.js"; import { ScheduleService } from "./schedule/service.js"; -import { DaemonConfigStore } from "./daemon-config-store.js"; +import { DaemonConfigStore, type MutableDaemonConfig } from "./daemon-config-store.js"; +import { BrowserToolsBroker } from "./browser-tools/broker.js"; +import { DaemonConfigBrowserToolsPolicy } from "./browser-tools/policy.js"; import { WorkspaceGitServiceImpl } from "./workspace-git-service.js"; import { resolveWorkspaceIdForPath } from "./resolve-workspace-id-for-path.js"; import { @@ -330,6 +332,7 @@ export interface PaseoDaemonConfig { trustedProxies?: true | string[]; mcpEnabled?: boolean; mcpInjectIntoAgents?: boolean; + browserToolsEnabled?: boolean; autoArchiveAfterMerge?: boolean; enableTerminalAgentHooks?: boolean; appendSystemPrompt?: string; @@ -383,6 +386,7 @@ export interface PaseoDaemon { terminalManager: TerminalManager; serviceProxy: ServiceProxySubsystem; scriptRuntimeStore: WorkspaceScriptRuntimeStore; + browserToolsBroker: BrowserToolsBroker; start(): Promise; stop(): Promise; getListenTarget(): ListenTarget | null; @@ -429,6 +433,39 @@ function resolveExpressTrustProxySetting(config: PaseoDaemonConfig): true | stri return config.trustedProxies ?? ["loopback"]; } +function createInitialMutableDaemonConfig(config: PaseoDaemonConfig): MutableDaemonConfig { + const providers: MutableDaemonConfig["providers"] = Object.fromEntries( + Object.entries(config.providerOverrides ?? {}).map(([providerId, override]) => { + const providerConfig: MutableDaemonConfig["providers"][string] = {}; + if (override.enabled !== undefined) { + providerConfig.enabled = override.enabled; + } + if (override.additionalModels) { + providerConfig.additionalModels = override.additionalModels; + } + return [providerId, providerConfig]; + }), + ); + + const initialConfig: MutableDaemonConfig = { + mcp: { injectIntoAgents: config.mcpInjectIntoAgents ?? true }, + browserTools: { enabled: config.browserToolsEnabled ?? false }, + providers, + metadataGeneration: { + providers: config.metadataGeneration?.providers ?? [], + }, + autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false, + enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false, + appendSystemPrompt: config.appendSystemPrompt ?? "", + }; + + if (config.terminalProfiles !== undefined) { + initialConfig.terminalProfiles = config.terminalProfiles; + } + + return initialConfig; +} + export async function createPaseoDaemon( config: PaseoDaemonConfig, rootLogger: Logger, @@ -439,29 +476,13 @@ export async function createPaseoDaemon( const daemonVersion = resolveDaemonVersion(import.meta.url); const daemonConfigStore = new DaemonConfigStore( config.paseoHome, - { - mcp: { injectIntoAgents: config.mcpInjectIntoAgents ?? true }, - providers: Object.fromEntries( - Object.entries(config.providerOverrides ?? {}).map(([providerId, override]) => [ - providerId, - { - ...(override.enabled !== undefined ? { enabled: override.enabled } : {}), - ...(override.additionalModels ? { additionalModels: override.additionalModels } : {}), - }, - ]), - ), - metadataGeneration: { - providers: config.metadataGeneration?.providers ?? [], - }, - autoArchiveAfterMerge: config.autoArchiveAfterMerge ?? false, - enableTerminalAgentHooks: config.enableTerminalAgentHooks ?? false, - appendSystemPrompt: config.appendSystemPrompt ?? "", - ...(config.terminalProfiles !== undefined - ? { terminalProfiles: config.terminalProfiles } - : {}), - }, + createInitialMutableDaemonConfig(config), logger, ); + const browserToolsPolicy = new DaemonConfigBrowserToolsPolicy(daemonConfigStore); + const browserToolsBroker = new BrowserToolsBroker({ + policy: browserToolsPolicy, + }); const serverId = getOrCreateServerId(config.paseoHome, { logger }); const daemonKeyPair = await loadOrCreateDaemonKeyPair(config.paseoHome, logger); @@ -959,6 +980,7 @@ export async function createPaseoDaemon( clearWorkspaceArchiving: clearWorkspaceArchivingExternal, ensureWorkspaceForCreate: ensureWorkspaceForCreateExternal, createPaseoWorktree: createPaseoWorktreeForTools, + browserToolsBroker, paseoHome: config.paseoHome, worktreesRoot: config.worktreesRoot, callerAgentId: runtime.callerAgentId, @@ -1228,6 +1250,7 @@ export async function createPaseoDaemon( }, }, serviceProxyPublicBaseUrl, + browserToolsBroker, ); if (relayEnabled) { @@ -1327,6 +1350,7 @@ export async function createPaseoDaemon( terminalManager, serviceProxy, scriptRuntimeStore, + browserToolsBroker, start, stop, getListenTarget: () => boundListenTarget, diff --git a/packages/server/src/server/browser-tools/broker.test.ts b/packages/server/src/server/browser-tools/broker.test.ts new file mode 100644 index 000000000..5ba99affb --- /dev/null +++ b/packages/server/src/server/browser-tools/broker.test.ts @@ -0,0 +1,330 @@ +import { afterEach, describe, expect, test, vi } from "vitest"; +import type { + BrowserAutomationCommand, + BrowserAutomationExecuteRequest, + BrowserAutomationExecuteResponse, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js"; +import { StaticBrowserToolsPolicy } from "./policy.js"; + +class FakeDesktopClient implements BrowserToolsDesktopClient { + public readonly receivedRequests: BrowserAutomationExecuteRequest[] = []; + + public constructor(public readonly id: string) {} + + public sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void { + this.receivedRequests.push(request); + } + + public resolveLatestWith( + broker: BrowserToolsBroker, + responsePayload: BrowserAutomationExecuteResponse["payload"], + ): boolean { + return broker.receiveResponse({ + type: "browser.automation.execute.response", + payload: responsePayload, + }); + } +} + +class FailingDesktopClient implements BrowserToolsDesktopClient { + public readonly id = "desktop-1"; + + public sendBrowserAutomationRequest(): void { + throw new Error("websocket send failed"); + } +} + +function createBroker(options: { enabled: boolean; timeoutMs?: number }): BrowserToolsBroker { + return new BrowserToolsBroker({ + policy: new StaticBrowserToolsPolicy(options.enabled), + defaultTimeoutMs: options.timeoutMs ?? 100, + createRequestId: () => "req-1", + }); +} + +function pageInfoCommand(): BrowserAutomationCommand { + return { command: "page_info", args: { browserId: "browser-1" } }; +} + +describe("BrowserToolsBroker", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + test("disabled returns browser_disabled", async () => { + const broker = createBroker({ enabled: false }); + + await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_disabled", + message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.", + retryable: false, + }, + }); + }); + + test("no capable desktop returns browser_no_desktop", async () => { + const broker = createBroker({ enabled: true }); + + await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_no_desktop", + message: "No desktop browser automation client is connected.", + retryable: true, + }, + }); + }); + + test("invalid browser requests return structured failures without contacting desktop", async () => { + const broker = createBroker({ enabled: true }); + const client = new FakeDesktopClient("desktop-1"); + broker.registerClient(client); + + await expect( + broker.execute({ + command: { + command: "new_tab", + args: { url: "ftp://example.com" }, + } as unknown as BrowserAutomationCommand, + }), + ).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_unknown_error", + message: "Browser automation request is invalid: URL must use http or https.", + retryable: false, + }, + }); + expect(client.receivedRequests).toEqual([]); + expect(broker.getPendingRequestCount()).toBe(0); + }); + + test("capable fake desktop receives request and returns response", async () => { + const broker = createBroker({ enabled: true }); + const client = new FakeDesktopClient("desktop-1"); + broker.registerClient(client); + + const resultPromise = broker.execute({ + command: { command: "list_tabs", args: { workspaceId: "workspace-1" } }, + workspaceId: "workspace-1", + }); + + expect(client.receivedRequests).toEqual([ + { + type: "browser.automation.execute.request", + requestId: "req-1", + workspaceId: "workspace-1", + command: { command: "list_tabs", args: { workspaceId: "workspace-1" } }, + }, + ]); + expect(broker.getPendingRequestCount()).toBe(1); + + expect( + client.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + }, + ], + }, + }), + ).toBe(true); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-1", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + isActive: false, + isLoading: false, + }, + ], + }, + }); + expect(broker.getPendingRequestCount()).toBe(0); + }); + + test("desktop receives snapshot requests", async () => { + const broker = createBroker({ enabled: true }); + const client = new FakeDesktopClient("desktop-1"); + broker.registerClient(client); + + const resultPromise = broker.execute({ + command: { command: "snapshot", args: { workspaceId: "workspace-1" } }, + workspaceId: "workspace-1", + }); + + expect(client.receivedRequests).toEqual([ + { + type: "browser.automation.execute.request", + requestId: "req-1", + workspaceId: "workspace-1", + command: { command: "snapshot", args: { workspaceId: "workspace-1" } }, + }, + ]); + + client.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "snapshot", + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + elements: [], + }, + }); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-1", + ok: true, + result: { + command: "snapshot", + browserId: "browser-1", + workspaceId: "workspace-1", + url: "https://example.com", + title: "Example", + elements: [], + }, + }); + }); + + test("timeout resolves browser_timeout and clears pending state", async () => { + vi.useFakeTimers(); + const broker = createBroker({ enabled: true, timeoutMs: 50 }); + broker.registerClient(new FakeDesktopClient("desktop-1")); + + const resultPromise = broker.execute({ command: pageInfoCommand() }); + expect(broker.getPendingRequestCount()).toBe(1); + + await vi.advanceTimersByTimeAsync(50); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_timeout", + message: "Browser automation timed out after 50ms.", + retryable: true, + }, + }); + expect(broker.getPendingRequestCount()).toBe(0); + }); + + test("disconnect resolves retryable failure and clears pending request", async () => { + const broker = createBroker({ enabled: true }); + const client = new FakeDesktopClient("desktop-1"); + const unregister = broker.registerClient(client); + + const resultPromise = broker.execute({ command: pageInfoCommand() }); + expect(broker.getPendingRequestCount()).toBe(1); + + unregister(); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_no_desktop", + message: "The desktop browser automation client disconnected before responding.", + retryable: true, + }, + }); + expect(broker.getPendingRequestCount()).toBe(0); + }); + + test("desktop send failure resolves structured failure and clears pending request", async () => { + const broker = createBroker({ enabled: true }); + broker.registerClient(new FailingDesktopClient()); + + await expect(broker.execute({ command: pageInfoCommand() })).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_unknown_error", + message: "Browser automation request failed to send: websocket send failed", + retryable: false, + }, + }); + expect(broker.getPendingRequestCount()).toBe(0); + }); + + test("explicit browser failure response propagates typed error", async () => { + const broker = createBroker({ enabled: true }); + const client = new FakeDesktopClient("desktop-1"); + broker.registerClient(client); + + const resultPromise = broker.execute({ command: pageInfoCommand() }); + + client.resolveLatestWith(broker, { + requestId: "req-1", + ok: false, + error: { + code: "browser_tab_not_found", + message: "Browser tab browser-1 was not found.", + retryable: false, + }, + }); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_tab_not_found", + message: "Browser tab browser-1 was not found.", + retryable: false, + }, + }); + expect(broker.getPendingRequestCount()).toBe(0); + }); + + test("invalid browser response resolves a structured failure and clears pending state", async () => { + const broker = createBroker({ enabled: true }); + const client = new FakeDesktopClient("desktop-1"); + broker.registerClient(client); + + const resultPromise = broker.execute({ command: pageInfoCommand() }); + expect(broker.getPendingRequestCount()).toBe(1); + + expect( + broker.receiveResponse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-1", + ok: true, + result: { command: "future_command" }, + }, + } as unknown as BrowserAutomationExecuteResponse), + ).toBe(true); + + await expect(resultPromise).resolves.toMatchObject({ + requestId: "req-1", + ok: false, + error: { + code: "browser_unknown_error", + retryable: false, + }, + }); + expect(broker.getPendingRequestCount()).toBe(0); + }); +}); diff --git a/packages/server/src/server/browser-tools/broker.ts b/packages/server/src/server/browser-tools/broker.ts new file mode 100644 index 000000000..cc0d7a0a7 --- /dev/null +++ b/packages/server/src/server/browser-tools/broker.ts @@ -0,0 +1,284 @@ +import { randomUUID } from "node:crypto"; +import { + BrowserAutomationExecuteRequestSchema, + BrowserAutomationExecuteResponseSchema, + type BrowserAutomationCommand, + type BrowserAutomationExecuteRequest, + type BrowserAutomationExecuteResponse, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import { browserToolsFailure, type BrowserToolsResponsePayload } from "./errors.js"; +import type { BrowserToolsPolicy } from "./policy.js"; + +export interface BrowserToolsDesktopClient { + id: string; + sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void | Promise; +} + +export interface BrowserToolsExecuteInput { + command: BrowserAutomationCommand; + agentId?: string; + cwd?: string; + workspaceId?: string; + browserId?: string; + requestId?: string; + timeoutMs?: number; +} + +interface PendingBrowserToolsRequest { + clientId: string; + timeout: ReturnType; + resolve: (payload: BrowserToolsResponsePayload) => void; +} + +export interface BrowserToolsBrokerOptions { + policy: BrowserToolsPolicy; + defaultTimeoutMs?: number; + createRequestId?: () => string; +} + +const DEFAULT_BROWSER_TOOLS_TIMEOUT_MS = 15_000; + +export class BrowserToolsBroker { + private readonly policy: BrowserToolsPolicy; + private readonly defaultTimeoutMs: number; + private readonly createRequestId: () => string; + private readonly clients = new Map(); + private readonly pending = new Map(); + + public constructor(options: BrowserToolsBrokerOptions) { + this.policy = options.policy; + this.defaultTimeoutMs = options.defaultTimeoutMs ?? DEFAULT_BROWSER_TOOLS_TIMEOUT_MS; + this.createRequestId = options.createRequestId ?? (() => `browser_${randomUUID()}`); + } + + public registerClient(client: BrowserToolsDesktopClient): () => void { + this.clients.set(client.id, client); + return () => this.unregisterClient(client.id); + } + + public unregisterClient(clientId: string): void { + const deleted = this.clients.delete(clientId); + if (!deleted) { + return; + } + + for (const [requestId, pending] of this.pending) { + if (pending.clientId !== clientId) { + continue; + } + this.pending.delete(requestId); + clearTimeout(pending.timeout); + pending.resolve( + browserToolsFailure({ + requestId, + code: "browser_no_desktop", + message: "The desktop browser automation client disconnected before responding.", + retryable: true, + }), + ); + } + } + + public getPendingRequestCount(): number { + return this.pending.size; + } + + public getRegisteredClientCount(): number { + return this.clients.size; + } + + public async execute(input: BrowserToolsExecuteInput): Promise { + const requestId = input.requestId ?? this.createRequestId(); + + if (!this.policy.isEnabled()) { + return browserToolsFailure({ + requestId, + code: "browser_disabled", + message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.", + }); + } + + const client = this.selectClient(); + if (!client) { + return browserToolsFailure({ + requestId, + code: "browser_no_desktop", + message: "No desktop browser automation client is connected.", + retryable: true, + }); + } + + const request = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId, + ...(input.agentId ? { agentId: input.agentId } : {}), + ...(input.cwd ? { cwd: input.cwd } : {}), + ...(input.workspaceId ? { workspaceId: input.workspaceId } : {}), + ...(input.browserId ? { browserId: input.browserId } : {}), + command: input.command, + }); + + if (!request.success) { + return browserToolsFailure({ + requestId, + code: "browser_unknown_error", + message: formatBrowserAutomationValidationError(request.error.issues[0]?.message), + }); + } + + return this.sendRequest({ + client, + request: request.data, + timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs, + }); + } + + public receiveResponse(response: BrowserAutomationExecuteResponse): boolean { + const parsed = BrowserAutomationExecuteResponseSchema.safeParse(response); + if (!parsed.success) { + const requestId = getBrowserAutomationResponseRequestId(response); + if (!requestId) { + return false; + } + + const pending = this.pending.get(requestId); + if (!pending) { + return false; + } + + this.pending.delete(requestId); + clearTimeout(pending.timeout); + pending.resolve( + browserToolsFailure({ + requestId, + code: "browser_unknown_error", + message: formatBrowserAutomationResponseValidationError(parsed.error.issues[0]?.message), + }), + ); + return true; + } + + const pending = this.pending.get(parsed.data.payload.requestId); + if (!pending) { + return false; + } + + this.pending.delete(parsed.data.payload.requestId); + clearTimeout(pending.timeout); + pending.resolve(parsed.data.payload); + return true; + } + + private selectClient(): BrowserToolsDesktopClient | null { + for (const client of this.clients.values()) { + return client; + } + return null; + } + + private sendRequest(params: { + client: BrowserToolsDesktopClient; + request: BrowserAutomationExecuteRequest; + timeoutMs: number; + }): Promise { + const { client, request, timeoutMs } = params; + + return new Promise((resolve) => { + const timeout = setTimeout(() => { + if (!this.pending.delete(request.requestId)) { + return; + } + resolve( + browserToolsFailure({ + requestId: request.requestId, + code: "browser_timeout", + message: `Browser automation timed out after ${timeoutMs}ms.`, + retryable: true, + }), + ); + }, timeoutMs); + + this.pending.set(request.requestId, { + clientId: client.id, + timeout, + resolve, + }); + + try { + Promise.resolve(client.sendBrowserAutomationRequest(request)).catch((error: unknown) => { + resolveSendFailure({ + requestId: request.requestId, + pending: this.pending, + timeout, + resolve, + error, + }); + }); + } catch (error) { + resolveSendFailure({ + requestId: request.requestId, + pending: this.pending, + timeout, + resolve, + error, + }); + } + }); + } +} + +function resolveSendFailure(params: { + requestId: string; + pending: Map; + timeout: ReturnType; + resolve: (payload: BrowserToolsResponsePayload) => void; + error: unknown; +}): void { + if (!params.pending.delete(params.requestId)) { + return; + } + clearTimeout(params.timeout); + params.resolve( + browserToolsFailure({ + requestId: params.requestId, + code: "browser_unknown_error", + message: formatBrowserAutomationSendError(params.error), + }), + ); +} + +function formatBrowserAutomationValidationError(message: string | undefined): string { + if (!message) { + return "Browser automation request is invalid."; + } + return `Browser automation request is invalid: ${message}.`; +} + +function formatBrowserAutomationResponseValidationError(message: string | undefined): string { + if (!message) { + return "Browser automation response is invalid."; + } + return `Browser automation response is invalid: ${message}.`; +} + +function formatBrowserAutomationSendError(error: unknown): string { + if (error instanceof Error && error.message) { + return `Browser automation request failed to send: ${error.message}`; + } + return `Browser automation request failed to send: ${String(error)}`; +} + +function getBrowserAutomationResponseRequestId(response: unknown): string | null { + if (!isRecord(response)) { + return null; + } + const payload = response.payload; + if (!isRecord(payload) || typeof payload.requestId !== "string") { + return null; + } + return payload.requestId; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/packages/server/src/server/browser-tools/errors.ts b/packages/server/src/server/browser-tools/errors.ts new file mode 100644 index 000000000..fe49edef6 --- /dev/null +++ b/packages/server/src/server/browser-tools/errors.ts @@ -0,0 +1,48 @@ +import type { + BrowserAutomationErrorCode, + BrowserAutomationExecuteResponse, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; + +export type BrowserToolsResponsePayload = BrowserAutomationExecuteResponse["payload"]; +export type BrowserToolsErrorPayload = Extract["error"]; + +export class BrowserToolsRequestError extends Error { + public readonly code: BrowserAutomationErrorCode; + public readonly retryable: boolean; + + public constructor(error: BrowserToolsErrorPayload) { + super(error.message); + this.name = "BrowserToolsRequestError"; + this.code = error.code; + this.retryable = error.retryable; + } +} + +export function browserToolsFailure(params: { + requestId: string; + code: BrowserAutomationErrorCode; + message: string; + retryable?: boolean; +}): BrowserToolsResponsePayload { + return { + requestId: params.requestId, + ok: false, + error: { + code: params.code, + message: params.message, + retryable: params.retryable ?? false, + }, + }; +} + +export function createBrowserToolsRequestError(params: { + code: BrowserAutomationErrorCode; + message: string; + retryable?: boolean; +}): BrowserToolsRequestError { + return new BrowserToolsRequestError({ + code: params.code, + message: params.message, + retryable: params.retryable ?? false, + }); +} diff --git a/packages/server/src/server/browser-tools/policy.ts b/packages/server/src/server/browser-tools/policy.ts new file mode 100644 index 000000000..aeea65590 --- /dev/null +++ b/packages/server/src/server/browser-tools/policy.ts @@ -0,0 +1,29 @@ +import type { DaemonConfigStore, MutableDaemonConfig } from "../daemon-config-store.js"; + +export interface BrowserToolsPolicy { + isEnabled(): boolean; +} + +export class StaticBrowserToolsPolicy implements BrowserToolsPolicy { + public constructor(private readonly enabled: boolean) {} + + public isEnabled(): boolean { + return this.enabled; + } +} + +export class DaemonConfigBrowserToolsPolicy implements BrowserToolsPolicy { + public constructor(private readonly configStore: Pick) {} + + public isEnabled(): boolean { + return readBrowserToolsEnabled(this.configStore.get()); + } +} + +function readBrowserToolsEnabled(config: MutableDaemonConfig): boolean { + const browserTools = config.browserTools; + if (typeof browserTools !== "object" || browserTools === null || Array.isArray(browserTools)) { + return false; + } + return browserTools.enabled === true; +} diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts new file mode 100644 index 000000000..0ed8dbc36 --- /dev/null +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -0,0 +1,850 @@ +import { describe, expect, it, vi } from "vitest"; +import type { BrowserToolsBroker } from "./broker.js"; +import type { BrowserToolsResponsePayload } from "./errors.js"; +import { registerBrowserTools, type RegisterBrowserToolsOptions } from "./tools.js"; + +interface RegisteredTool { + config: { inputSchema: Record; outputSchema: unknown }; + handler: (args: Record) => Promise<{ + content: Array<{ type: string; text?: string }>; + structuredContent?: Record; + }>; +} + +function createHarness(options?: { brokerResponse?: BrowserToolsResponsePayload }) { + const tools = new Map(); + const execute = vi.fn(async () => { + return ( + options?.brokerResponse ?? { + requestId: "req-1", + ok: true as const, + result: { + command: "list_tabs" as const, + tabs: [ + { + browserId: "browser-1", + url: "https://example.com", + title: "Example", + isActive: true, + isLoading: false, + }, + ], + }, + } + ); + }); + const registerTool = vi.fn((name, config, handler) => { + tools.set(name, { + config, + handler: handler as RegisteredTool["handler"], + }); + }) as unknown as RegisterBrowserToolsOptions["registerTool"]; + + registerBrowserTools({ + registerTool, + broker: { execute } as unknown as BrowserToolsBroker, + callerAgentId: "agent-1", + resolveCallerAgent: () => ({ id: "agent-1", cwd: "/repo" }), + }); + + return { tools, execute }; +} + +function tool(harness: ReturnType, name: string): RegisteredTool { + const registered = harness.tools.get(name); + if (!registered) { + throw new Error(`Tool not registered: ${name}`); + } + return registered; +} + +describe("registerBrowserTools", () => { + it("registers browser_list_tabs and routes through the broker with caller workspace", async () => { + const harness = createHarness(); + + const response = await tool(harness, "browser_list_tabs").handler({}); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "list_tabs", args: { workspaceId: "/repo" } }, + }); + expect(response.content).toEqual([ + { + type: "text", + text: "Found 1 Paseo browser tab. Use these browserId values exactly; do not use 'default'. You may omit browserId to use the active tab.\n- browserId=browser-1 active title=\"Example\" url=https://example.com", + }, + ]); + expect(response.structuredContent).toEqual({ + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: "browser-1", + url: "https://example.com", + title: "Example", + isActive: true, + isLoading: false, + }, + ], + }, + context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" }, + }); + }); + + it("registers browser_new_tab and returns the created tab handle", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-new-tab", + ok: true, + result: { + command: "new_tab", + browserId: "browser-new", + workspaceId: "/repo", + url: "https://example.com", + }, + }, + }); + + const response = await tool(harness, "browser_new_tab").handler({ url: "https://example.com" }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { + command: "new_tab", + args: { workspaceId: "/repo", url: "https://example.com" }, + }, + }); + expect(response.content).toEqual([ + { + type: "text", + text: "Created browser tab browserId=browser-new url=https://example.com. Use this browserId exactly, or omit browserId to use this active tab.", + }, + ]); + }); + + it("routes browser_page_info through the broker with explicit browserId", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-2", + ok: true, + result: { + command: "page_info", + tab: { + browserId: "browser-2", + url: "https://example.com/docs", + title: "Docs", + isActive: false, + isLoading: false, + }, + }, + }, + }); + + const response = await tool(harness, "browser_page_info").handler({ browserId: "browser-2" }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + browserId: "browser-2", + command: { + command: "page_info", + args: { workspaceId: "/repo", browserId: "browser-2" }, + }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Current page browserId=browser-2: Docs — https://example.com/docs" }, + ]); + expect(response.structuredContent).toEqual({ + ok: true, + result: { + command: "page_info", + tab: { + browserId: "browser-2", + url: "https://example.com/docs", + title: "Docs", + isActive: false, + isLoading: false, + }, + }, + context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo", browserId: "browser-2" }, + }); + }); + + it("routes browser_snapshot through the broker with caller workspace", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-snapshot", + ok: true, + result: { + command: "snapshot", + browserId: "browser-1", + workspaceId: "/repo", + url: "https://example.com/form", + title: "Fixture", + elements: [ + { + ref: "@e1", + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name" }, + }, + ], + }, + }, + }); + + const response = await tool(harness, "browser_snapshot").handler({}); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "snapshot", args: { workspaceId: "/repo" } }, + }); + expect(response.content).toEqual([{ type: "text", text: "Snapshot captured 1 element." }]); + expect(response.structuredContent).toEqual({ + ok: true, + result: { + command: "snapshot", + browserId: "browser-1", + workspaceId: "/repo", + url: "https://example.com/form", + title: "Fixture", + elements: [ + { + ref: "@e1", + role: "textbox", + tagName: "input", + text: "Name", + selector: "#name", + attributes: { id: "name" }, + }, + ], + }, + context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" }, + }); + }); + + it("routes browser_set_background through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-bg", + ok: true, + result: { command: "set_background", browserId: "browser-1", color: "red" }, + }, + }); + + const response = await tool(harness, "browser_set_background").handler({ color: "red" }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "set_background", args: { workspaceId: "/repo", color: "red" } }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Set browser page background to red." }, + ]); + }); + + it("routes browser_click through the broker with a snapshot ref", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-click", + ok: true, + result: { command: "click", browserId: "browser-1", ref: "@e2" }, + }, + }); + + const response = await tool(harness, "browser_click").handler({ ref: "@e2" }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "click", args: { workspaceId: "/repo", ref: "@e2" } }, + }); + expect(response.content).toEqual([{ type: "text", text: "Clicked browser element @e2." }]); + expect(response.structuredContent).toEqual({ + ok: true, + result: { command: "click", browserId: "browser-1", ref: "@e2" }, + context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" }, + }); + }); + + it("routes browser_fill through the broker with a snapshot ref and value", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-fill", + ok: true, + result: { command: "fill", browserId: "browser-1", ref: "@e1" }, + }, + }); + + const response = await tool(harness, "browser_fill").handler({ ref: "@e1", value: "Ada" }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "fill", args: { workspaceId: "/repo", ref: "@e1", value: "Ada" } }, + }); + expect(response.content).toEqual([{ type: "text", text: "Filled browser element @e1." }]); + expect(response.structuredContent).toEqual({ + ok: true, + result: { command: "fill", browserId: "browser-1", ref: "@e1" }, + context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" }, + }); + }); + + it("routes browser_wait through the broker with text and timeout", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-wait", + ok: true, + result: { command: "wait", browserId: "browser-1", matched: "text" }, + }, + }); + + const response = await tool(harness, "browser_wait").handler({ + text: "Ready", + timeoutMs: 1000, + }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + timeoutMs: 2000, + command: { + command: "wait", + args: { workspaceId: "/repo", text: "Ready", timeoutMs: 1000 }, + }, + }); + expect(response.content).toEqual([{ type: "text", text: "Browser wait matched text." }]); + expect(response.structuredContent).toEqual({ + ok: true, + result: { command: "wait", browserId: "browser-1", matched: "text" }, + context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" }, + }); + }); + + it("routes browser_type through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-type", + ok: true, + result: { command: "type", browserId: "browser-1", ref: "@e1" }, + }, + }); + + const response = await tool(harness, "browser_type").handler({ ref: "@e1", text: "Ada" }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "type", args: { workspaceId: "/repo", ref: "@e1", text: "Ada" } }, + }); + expect(response.content).toEqual([{ type: "text", text: "Typed into browser element @e1." }]); + }); + + it("routes browser_keypress through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-keypress", + ok: true, + result: { command: "keypress", browserId: "browser-1", ref: "@e1", key: "Enter" }, + }, + }); + + const response = await tool(harness, "browser_keypress").handler({ + ref: "@e1", + key: "Enter", + }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { + command: "keypress", + args: { workspaceId: "/repo", ref: "@e1", key: "Enter" }, + }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Pressed Enter on browser element @e1." }, + ]); + }); + + it("routes browser_navigate through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-nav", + ok: true, + result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" }, + }, + }); + + const response = await tool(harness, "browser_navigate").handler({ + url: "https://example.com/next", + }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { + command: "navigate", + args: { workspaceId: "/repo", url: "https://example.com/next" }, + }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Navigated browser to https://example.com/next." }, + ]); + }); + + it("routes browser_back through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-back", + ok: true, + result: { command: "back", browserId: "browser-1" }, + }, + }); + + const response = await tool(harness, "browser_back").handler({}); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "back", args: { workspaceId: "/repo" } }, + }); + expect(response.content).toEqual([{ type: "text", text: "Browser back complete." }]); + }); + + it("routes browser_screenshot through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-shot", + ok: true, + result: { + command: "screenshot", + browserId: "browser-1", + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 100, + height: 50, + }, + }, + }); + + const response = await tool(harness, "browser_screenshot").handler({}); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "screenshot", args: { workspaceId: "/repo" } }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Captured browser screenshot (100x50)." }, + { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }, + ]); + }); + + it("routes browser_logs through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-logs", + ok: true, + result: { + command: "logs", + browserId: "browser-1", + console: [{ level: "info", message: "ready", timestamp: 1 }], + network: [ + { + url: "https://example.com/app.js", + type: "script", + startTime: 2, + duration: 3, + }, + ], + }, + }, + }); + + const response = await tool(harness, "browser_logs").handler({ maxEntries: 25 }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "logs", args: { workspaceId: "/repo", maxEntries: 25 } }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Read 1 console log and 1 network entry." }, + ]); + }); + + it("routes browser_storage through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-storage", + ok: true, + result: { + command: "storage", + browserId: "browser-1", + url: "https://example.com", + cookies: [{ name: "theme", value: "dark" }], + localStorage: [{ key: "token", value: "abc" }], + sessionStorage: [{ key: "tab", value: "1" }], + }, + }, + }); + + const response = await tool(harness, "browser_storage").handler({}); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "storage", args: { workspaceId: "/repo" } }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Read 1 cookie, 1 localStorage entry, and 1 sessionStorage entry." }, + ]); + }); + + it("routes browser_environment through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-environment", + ok: true, + result: { + command: "environment", + browserId: "browser-1", + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + }); + + const response = await tool(harness, "browser_environment").handler({ + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { + command: "environment", + args: { + workspaceId: "/repo", + viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, + geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, + }, + }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Browser environment viewport is 390x844." }, + ]); + }); + + it("routes browser_full_page_screenshot through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-full-page", + ok: true, + result: { + command: "full_page_screenshot", + browserId: "browser-1", + mimeType: "image/png", + dataBase64: "iVBORw0KGgo=", + width: 390, + height: 1200, + }, + }, + }); + + const response = await tool(harness, "browser_full_page_screenshot").handler({}); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "full_page_screenshot", args: { workspaceId: "/repo" } }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Captured full-page browser screenshot (390x1200)." }, + { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }, + ]); + }); + + it("routes browser_pdf through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-pdf", + ok: true, + result: { + command: "pdf", + browserId: "browser-1", + mimeType: "application/pdf", + dataBase64: "JVBERg==", + }, + }, + }); + + const response = await tool(harness, "browser_pdf").handler({ landscape: true }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { + command: "pdf", + args: { workspaceId: "/repo", landscape: true, printBackground: true }, + }, + }); + expect(response.content).toEqual([{ type: "text", text: "Exported browser page PDF." }]); + }); + + it("routes browser_download through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-download", + ok: true, + result: { + command: "download", + browserId: "browser-1", + url: "https://example.com/file.txt", + filePath: "/tmp/file.txt", + totalBytes: 5, + state: "completed", + }, + }, + }); + + const response = await tool(harness, "browser_download").handler({ + url: "https://example.com/file.txt", + fileName: "file.txt", + }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { + command: "download", + args: { workspaceId: "/repo", url: "https://example.com/file.txt", fileName: "file.txt" }, + }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Downloaded browser file to /tmp/file.txt." }, + ]); + }); + + it("routes browser_upload through the broker", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-upload", + ok: true, + result: { + command: "upload", + browserId: "browser-1", + ref: "@e1", + filePaths: ["/tmp/file.txt"], + }, + }, + }); + + const response = await tool(harness, "browser_upload").handler({ + ref: "@e1", + filePaths: ["/tmp/file.txt"], + }); + + expect(harness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { + command: "upload", + args: { workspaceId: "/repo", ref: "@e1", filePaths: ["/tmp/file.txt"] }, + }, + }); + expect(response.content).toEqual([ + { type: "text", text: "Uploaded 1 file to browser element @e1." }, + ]); + }); + + it("routes browser form control tools through the broker", async () => { + const focusHarness = createHarness({ + brokerResponse: { + requestId: "req-focus", + ok: true, + result: { command: "focus", browserId: "browser-1", ref: "@e1" }, + }, + }); + const focusResponse = await tool(focusHarness, "browser_focus").handler({ ref: "@e1" }); + expect(focusHarness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "focus", args: { workspaceId: "/repo", ref: "@e1" } }, + }); + expect(focusResponse.content).toEqual([{ type: "text", text: "Focused browser element @e1." }]); + + const checkHarness = createHarness({ + brokerResponse: { + requestId: "req-check", + ok: true, + result: { command: "check", browserId: "browser-1", ref: "@e2", checked: false }, + }, + }); + const checkResponse = await tool(checkHarness, "browser_check").handler({ + ref: "@e2", + checked: false, + }); + expect(checkHarness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "check", args: { workspaceId: "/repo", ref: "@e2", checked: false } }, + }); + expect(checkResponse.content).toEqual([ + { type: "text", text: "Unchecked browser element @e2." }, + ]); + + const selectHarness = createHarness({ + brokerResponse: { + requestId: "req-select", + ok: true, + result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" }, + }, + }); + const selectResponse = await tool(selectHarness, "browser_select").handler({ + ref: "@e3", + value: "us", + }); + expect(selectHarness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "select", args: { workspaceId: "/repo", ref: "@e3", value: "us" } }, + }); + expect(selectResponse.content).toEqual([ + { type: "text", text: "Selected us in browser element @e3." }, + ]); + + const hoverHarness = createHarness({ + brokerResponse: { + requestId: "req-hover", + ok: true, + result: { command: "hover", browserId: "browser-1", ref: "@e4" }, + }, + }); + const hoverResponse = await tool(hoverHarness, "browser_hover").handler({ ref: "@e4" }); + expect(hoverHarness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { command: "hover", args: { workspaceId: "/repo", ref: "@e4" } }, + }); + expect(hoverResponse.content).toEqual([{ type: "text", text: "Hovered browser element @e4." }]); + + const dragHarness = createHarness({ + brokerResponse: { + requestId: "req-drag", + ok: true, + result: { command: "drag", browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" }, + }, + }); + const dragResponse = await tool(dragHarness, "browser_drag").handler({ + sourceRef: "@e4", + targetRef: "@e5", + }); + expect(dragHarness.execute).toHaveBeenCalledWith({ + agentId: "agent-1", + cwd: "/repo", + workspaceId: "/repo", + command: { + command: "drag", + args: { workspaceId: "/repo", sourceRef: "@e4", targetRef: "@e5" }, + }, + }); + expect(dragResponse.content).toEqual([ + { type: "text", text: "Dragged browser element @e4 to @e5." }, + ]); + }); + + it("returns model-actionable disabled errors with structured content", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-disabled", + ok: false, + error: { + code: "browser_disabled", + message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.", + retryable: false, + }, + }, + }); + + const response = await tool(harness, "browser_list_tabs").handler({}); + + expect(response.content).toEqual([ + { + type: "text", + text: "Browser tools are disabled. Enable desktop browser tools on the host, then try again.", + }, + ]); + expect(response.structuredContent).toEqual({ + ok: false, + error: { + code: "browser_disabled", + message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.", + retryable: false, + }, + context: { agentId: "agent-1", cwd: "/repo", workspaceId: "/repo" }, + }); + }); + + it("preserves typed broker errors", async () => { + const harness = createHarness({ + brokerResponse: { + requestId: "req-timeout", + ok: false, + error: { + code: "browser_timeout", + message: "Browser automation timed out after 15000ms.", + retryable: true, + }, + }, + }); + + const response = await tool(harness, "browser_page_info").handler({}); + + expect(response.content).toEqual([ + { + type: "text", + text: "The browser did not respond before the timeout. Try again or check the desktop app.", + }, + ]); + expect(response.structuredContent?.error).toEqual({ + code: "browser_timeout", + message: "Browser automation timed out after 15000ms.", + retryable: true, + }); + }); +}); diff --git a/packages/server/src/server/browser-tools/tools.ts b/packages/server/src/server/browser-tools/tools.ts new file mode 100644 index 000000000..c78b12301 --- /dev/null +++ b/packages/server/src/server/browser-tools/tools.ts @@ -0,0 +1,1125 @@ +import { z } from "zod"; +import type { BrowserToolsBroker } from "./broker.js"; +import type { BrowserToolsResponsePayload } from "./errors.js"; +import type { + PaseoToolConfig, + PaseoToolExecutionContext, + PaseoToolResult, +} from "../agent/tools/types.js"; + +interface CallerAgentContext { + id: string; + cwd: string; +} + +export interface RegisterBrowserToolsOptions { + registerTool: ( + name: string, + config: PaseoToolConfig, + handler: ( + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Tool inputs are validated by the catalog before execution. + input: any, + context: PaseoToolExecutionContext, + ) => Promise, + ) => void; + broker: Pick; + callerAgentId?: string; + resolveCallerAgent: () => CallerAgentContext | null; +} + +const BrowserToolOutputSchema = z.object({ + ok: z.boolean(), + result: z.unknown().optional(), + error: z + .object({ + code: z.string(), + message: z.string(), + retryable: z.boolean(), + }) + .optional(), + context: z + .object({ + agentId: z.string().optional(), + cwd: z.string().optional(), + workspaceId: z.string().optional(), + browserId: z.string().optional(), + }) + .optional(), +}); + +const BrowserRefInputSchema = z.string().regex(/^@e\d+$/); + +export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { + options.registerTool( + "browser_list_tabs", + { + title: "List browser tabs", + description: + "List open Paseo desktop browser tabs for this agent's workspace context. Use the returned browserId values exactly; do not use 'default'. If a tool omits browserId, it targets the active workspace tab.", + inputSchema: {}, + outputSchema: BrowserToolOutputSchema, + }, + async () => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + command: { + command: "list_tabs", + args: context.workspaceId ? { workspaceId: context.workspaceId } : {}, + }, + }); + return browserToolResult({ payload, context }); + }, + ); + + options.registerTool( + "browser_new_tab", + { + title: "Create browser tab", + description: + "Create and focus a new Paseo desktop browser tab in this agent's workspace. Optionally pass an http(s) URL to open immediately. Returns the new browserId; use that exact ID for later calls or omit browserId to use the active tab.", + inputSchema: { + url: z.string().url().optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ url }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + command: { + command: "new_tab", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(url ? { url } : {}), + }, + }, + }); + return browserToolResult({ payload, context }); + }, + ); + + options.registerTool( + "browser_page_info", + { + title: "Get browser page info", + description: + "Get page info for a Paseo desktop browser tab. Omit browserId to use the active workspace tab. If you pass browserId, use a real ID returned by browser_list_tabs or browser_new_tab; never pass 'default'.", + inputSchema: { + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "page_info", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_snapshot", + { + title: "Snapshot browser page", + description: + "Return a model-readable snapshot of the active Paseo desktop browser tab for this agent's workspace. Snapshot refs like @e1 are valid until the page changes or a new snapshot is taken.", + inputSchema: { + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "snapshot", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_click", + { + title: "Click browser element", + description: + "Click an element ref from the latest browser_snapshot for this agent's workspace browser tab.", + inputSchema: { + ref: BrowserRefInputSchema, + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ ref, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "click", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ref, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_fill", + { + title: "Fill browser element", + description: + "Fill an input-like element ref from the latest browser_snapshot for this agent's workspace browser tab.", + inputSchema: { + ref: BrowserRefInputSchema, + value: z.string(), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ ref, value, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "fill", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ref, + value, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_wait", + { + title: "Wait for browser condition", + description: + "Wait until the active Paseo desktop browser tab contains text or reaches a URL fragment.", + inputSchema: { + text: z.string().min(1).optional(), + url: z.string().min(1).optional(), + timeoutMs: z.number().int().positive().max(30_000).optional(), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ text, url, timeoutMs, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ...(timeoutMs ? { timeoutMs: timeoutMs + 1_000 } : {}), + command: { + command: "wait", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ...(text ? { text } : {}), + ...(url ? { url } : {}), + ...(timeoutMs ? { timeoutMs } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_type", + { + title: "Type into browser", + description: + "Type text into an element ref from the latest browser_snapshot, or into the currently focused browser element when ref is omitted.", + inputSchema: { + text: z.string(), + ref: BrowserRefInputSchema.optional(), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ text, ref, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "type", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ...(ref ? { ref } : {}), + text, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_keypress", + { + title: "Press browser key", + description: + "Dispatch a keypress to an element ref from the latest browser_snapshot, or to the currently focused browser element when ref is omitted.", + inputSchema: { + key: z.string().min(1), + ref: BrowserRefInputSchema.optional(), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ key, ref, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "keypress", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ...(ref ? { ref } : {}), + key, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_navigate", + { + title: "Navigate browser", + description: "Navigate the active Paseo desktop browser tab to a URL.", + inputSchema: { url: z.string().min(1), browserId: z.string().min(1).optional() }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ url, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "navigate", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + url, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + for (const name of ["browser_back", "browser_forward", "browser_reload"] as const) { + const command = name.replace("browser_", "") as "back" | "forward" | "reload"; + options.registerTool( + name, + { + title: `Browser ${command}`, + description: `${command} the active Paseo desktop browser tab.`, + inputSchema: { browserId: z.string().min(1).optional() }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command, + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + } + + options.registerTool( + "browser_screenshot", + { + title: "Capture browser screenshot", + description: "Capture a PNG screenshot of the active Paseo desktop browser tab.", + inputSchema: { browserId: z.string().min(1).optional() }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "screenshot", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_full_page_screenshot", + { + title: "Capture full-page browser screenshot", + description: "Capture a full-page PNG screenshot of a Paseo desktop browser tab.", + inputSchema: { browserId: z.string().min(1).optional() }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "full_page_screenshot", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_pdf", + { + title: "Export browser page PDF", + description: "Export the current Paseo desktop browser tab as a PDF.", + inputSchema: { + browserId: z.string().min(1).optional(), + landscape: z.boolean().optional(), + printBackground: z.boolean().default(true), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId, landscape, printBackground }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "pdf", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ...(landscape !== undefined ? { landscape } : {}), + printBackground: printBackground ?? true, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_download", + { + title: "Download file in browser", + description: "Download a URL through the Paseo desktop browser session.", + inputSchema: { + url: z.string().min(1), + fileName: z.string().min(1).optional(), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ url, fileName, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "download", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + url, + ...(fileName ? { fileName } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_upload", + { + title: "Upload files in browser", + description: + "Set workspace files on a file input ref from the latest browser_snapshot. Paths must resolve inside the agent workspace.", + inputSchema: { + ref: BrowserRefInputSchema, + filePaths: z.array(z.string().min(1)).min(1), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ ref, filePaths, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "upload", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ref, + filePaths, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + for (const toolConfig of [ + { + name: "browser_focus", + command: "focus", + title: "Focus browser element", + description: "Focus an element ref from the latest browser_snapshot.", + }, + { + name: "browser_clear", + command: "clear", + title: "Clear browser element", + description: "Clear an input-like element ref from the latest browser_snapshot.", + }, + { + name: "browser_hover", + command: "hover", + title: "Hover browser element", + description: "Hover an element ref from the latest browser_snapshot.", + }, + ] as const) { + options.registerTool( + toolConfig.name, + { + title: toolConfig.title, + description: toolConfig.description, + inputSchema: { ref: BrowserRefInputSchema, browserId: z.string().min(1).optional() }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ ref, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: toolConfig.command, + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ref, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + } + + options.registerTool( + "browser_check", + { + title: "Check browser control", + description: "Set a checkbox or radio ref from the latest browser_snapshot.", + inputSchema: { + ref: BrowserRefInputSchema, + checked: z.boolean().default(true), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ ref, checked, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "check", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ref, + checked, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_select", + { + title: "Select browser option", + description: "Set a select element ref from the latest browser_snapshot to a value.", + inputSchema: { + ref: BrowserRefInputSchema, + value: z.string(), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ ref, value, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "select", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ref, + value, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_drag", + { + title: "Drag browser element", + description: + "Drag a source element ref onto a target element ref from the latest browser_snapshot.", + inputSchema: { + sourceRef: BrowserRefInputSchema, + targetRef: BrowserRefInputSchema, + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ sourceRef, targetRef, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "drag", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + sourceRef, + targetRef, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_logs", + { + title: "Read browser logs", + description: + "Read recent console messages and browser performance network entries for a Paseo desktop browser tab.", + inputSchema: { + maxEntries: z.number().int().positive().max(200).optional(), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ maxEntries, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "logs", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + maxEntries: maxEntries ?? 50, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_storage", + { + title: "Read browser storage", + description: + "Read cookies plus localStorage and sessionStorage for a Paseo desktop browser tab.", + inputSchema: { browserId: z.string().min(1).optional() }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "storage", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_environment", + { + title: "Set/read browser environment", + description: "Set or read viewport and geolocation for a Paseo desktop browser tab.", + inputSchema: { + viewport: z + .object({ + width: z.number().int().positive(), + height: z.number().int().positive(), + deviceScaleFactor: z.number().positive().optional(), + }) + .optional(), + geolocation: z + .object({ + latitude: z.number().min(-90).max(90), + longitude: z.number().min(-180).max(180), + accuracy: z.number().positive().optional(), + }) + .optional(), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ viewport, geolocation, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "environment", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + ...(viewport ? { viewport } : {}), + ...(geolocation ? { geolocation } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_set_background", + { + title: "Set browser background", + description: + "Set the current page background color in the active Paseo desktop browser tab. Omit browserId to use the active workspace tab; pass a returned browserId only when targeting a specific tab.", + inputSchema: { + color: z.string().min(1), + browserId: z.string().min(1).optional(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ color, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + command: { + command: "set_background", + args: { + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserId ? { browserId } : {}), + color, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); +} + +function resolveBrowserToolContext(options: RegisterBrowserToolsOptions): { + agentId?: string; + cwd?: string; + workspaceId?: string; +} { + const callerAgent = options.resolveCallerAgent(); + const cwd = callerAgent?.cwd; + return { + ...(options.callerAgentId ? { agentId: options.callerAgentId } : {}), + ...(cwd ? { cwd, workspaceId: cwd } : {}), + }; +} + +function browserToolResult(params: { + payload: BrowserToolsResponsePayload; + context: { agentId?: string; cwd?: string; workspaceId?: string; browserId?: string }; +}): PaseoToolResult { + const { payload, context } = params; + if (payload.ok) { + return { + content: browserToolSuccessContent(payload), + structuredContent: { + ok: true, + result: payload.result, + context, + }, + }; + } + + return { + content: [{ type: "text", text: summarizeBrowserError(payload.error) }], + structuredContent: { + ok: false, + error: payload.error, + context, + }, + }; +} + +function browserToolSuccessContent( + payload: Extract, +): PaseoToolResult["content"] { + const textContent = { type: "text" as const, text: summarizeBrowserSuccess(payload) }; + const imageContent = browserToolImageContent(payload.result); + return imageContent ? [textContent, imageContent] : [textContent]; +} + +function browserToolImageContent( + result: Extract["result"], +): PaseoToolResult["content"][number] | null { + if (result.command !== "screenshot" && result.command !== "full_page_screenshot") { + return null; + } + + return { + type: "image", + data: result.dataBase64, + mimeType: result.mimeType, + }; +} + +function summarizeBrowserSuccess( + payload: Extract, +): string { + const controlSummary = summarizeBrowserControlSuccess(payload.result); + if (controlSummary) { + return controlSummary; + } + + const refActionSummary = summarizeBrowserRefActionSuccess(payload.result); + if (refActionSummary) { + return refActionSummary; + } + + const diagnosticsSummary = summarizeBrowserDiagnosticsSuccess(payload.result); + if (diagnosticsSummary) { + return diagnosticsSummary; + } + + const storageSummary = summarizeBrowserStorageSuccess(payload.result); + if (storageSummary) { + return storageSummary; + } + + const environmentSummary = summarizeBrowserEnvironmentSuccess(payload.result); + if (environmentSummary) { + return environmentSummary; + } + + const keyboardSummary = summarizeBrowserKeyboardSuccess(payload.result); + if (keyboardSummary) { + return keyboardSummary; + } + + const navigationSummary = summarizeBrowserNavigationSuccess(payload.result); + if (navigationSummary) { + return navigationSummary; + } + + const mediaSummary = summarizeBrowserMediaSuccess(payload.result); + if (mediaSummary) { + return mediaSummary; + } + + if (payload.result.command === "list_tabs") { + const count = payload.result.tabs.length; + if (count === 0) { + return "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab."; + } + const tabLines = payload.result.tabs.map((tab) => { + const active = tab.isActive ? " active" : ""; + return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url}`; + }); + return [ + `Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values exactly; do not use 'default'. You may omit browserId to use the active tab.`, + ...tabLines, + ].join("\n"); + } + + if (payload.result.command === "new_tab") { + return `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId exactly, or omit browserId to use this active tab.`; + } + + if (payload.result.command === "snapshot") { + const count = payload.result.elements.length; + return `Snapshot captured ${count} element${count === 1 ? "" : "s"}.`; + } + + if (payload.result.command === "wait") { + return `Browser wait matched ${payload.result.matched}.`; + } + + if (payload.result.command === "page_info") { + return `Current page browserId=${payload.result.tab.browserId}: ${payload.result.tab.title || "Untitled"} — ${payload.result.tab.url}`; + } + + return `Browser ${payload.result.command} complete.`; +} + +function summarizeBrowserMediaSuccess( + result: Extract["result"], +): string | null { + if (result.command === "screenshot") { + return `Captured browser screenshot (${result.width}x${result.height}).`; + } + if (result.command === "full_page_screenshot") { + return `Captured full-page browser screenshot (${result.width}x${result.height}).`; + } + if (result.command === "pdf") { + return "Exported browser page PDF."; + } + if (result.command === "download") { + return `Downloaded browser file to ${result.filePath}.`; + } + if (result.command === "upload") { + const count = result.filePaths.length; + return `Uploaded ${count} file${count === 1 ? "" : "s"} to browser element ${result.ref}.`; + } + return null; +} + +function summarizeBrowserKeyboardSuccess( + result: Extract["result"], +): string | null { + if (result.command === "type") { + return result.ref + ? `Typed into browser element ${result.ref}.` + : "Typed into the focused browser element."; + } + + if (result.command === "keypress") { + return result.ref + ? `Pressed ${result.key} on browser element ${result.ref}.` + : `Pressed ${result.key} in the browser.`; + } + + return null; +} + +function summarizeBrowserNavigationSuccess( + result: Extract["result"], +): string | null { + if (result.command === "navigate") { + return `Navigated browser to ${result.url}.`; + } + + if (result.command === "back" || result.command === "forward" || result.command === "reload") { + return `Browser ${result.command} complete.`; + } + + return null; +} + +function summarizeBrowserDiagnosticsSuccess( + result: Extract["result"], +): string | null { + if (result.command !== "logs") { + return null; + } + const consoleCount = result.console.length; + const networkCount = result.network.length; + return `Read ${consoleCount} console log${consoleCount === 1 ? "" : "s"} and ${networkCount} network entr${networkCount === 1 ? "y" : "ies"}.`; +} + +function summarizeBrowserStorageSuccess( + result: Extract["result"], +): string | null { + if (result.command !== "storage") { + return null; + } + return `Read ${result.cookies.length} cookie${result.cookies.length === 1 ? "" : "s"}, ${result.localStorage.length} localStorage entr${result.localStorage.length === 1 ? "y" : "ies"}, and ${result.sessionStorage.length} sessionStorage entr${result.sessionStorage.length === 1 ? "y" : "ies"}.`; +} + +function summarizeBrowserEnvironmentSuccess( + result: Extract["result"], +): string | null { + if (result.command !== "environment") { + return null; + } + return `Browser environment viewport is ${result.viewport.width}x${result.viewport.height}.`; +} + +function summarizeBrowserRefActionSuccess( + result: Extract["result"], +): string | null { + if (result.command === "click") { + return `Clicked browser element ${result.ref}.`; + } + + if (result.command === "fill") { + return `Filled browser element ${result.ref}.`; + } + + return null; +} + +function summarizeBrowserControlSuccess( + result: Extract["result"], +): string | null { + if (result.command === "focus") { + return `Focused browser element ${result.ref}.`; + } + + if (result.command === "clear") { + return `Cleared browser element ${result.ref}.`; + } + + if (result.command === "check") { + return `${result.checked ? "Checked" : "Unchecked"} browser element ${result.ref}.`; + } + + if (result.command === "select") { + return `Selected ${result.value} in browser element ${result.ref}.`; + } + + if (result.command === "hover") { + return `Hovered browser element ${result.ref}.`; + } + + if (result.command === "drag") { + return `Dragged browser element ${result.sourceRef} to ${result.targetRef}.`; + } + + if (result.command === "set_background") { + return `Set browser page background to ${result.color}.`; + } + + return null; +} + +function summarizeBrowserError( + error: Extract["error"], +): string { + switch (error.code) { + case "browser_disabled": + return "Browser tools are disabled. Enable desktop browser tools on the host, then try again."; + case "browser_no_desktop": + return "No desktop browser automation client is connected. Open the Paseo desktop app and try again."; + case "browser_no_tab": + return "No active browser tab is available. Call browser_new_tab to create one, then omit browserId for the active tab or use the returned browserId."; + case "browser_timeout": + return "The browser did not respond before the timeout. Try again or check the desktop app."; + case "browser_unsupported": + return "This desktop build does not support that browser automation request yet."; + case "browser_stale_ref": + return "That browser element reference is stale. Take a new browser snapshot and try again."; + default: + return error.message; + } +} diff --git a/packages/server/src/server/config-browser-tools.test.ts b/packages/server/src/server/config-browser-tools.test.ts new file mode 100644 index 000000000..d313893a5 --- /dev/null +++ b/packages/server/src/server/config-browser-tools.test.ts @@ -0,0 +1,38 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; + +import { loadConfig } from "./config.js"; + +const roots: string[] = []; + +async function createPaseoHome(config: unknown): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), "paseo-config-browser-tools-")); + roots.push(root); + const paseoHome = path.join(root, ".paseo"); + await mkdir(paseoHome, { recursive: true }); + await writeFile(path.join(paseoHome, "config.json"), JSON.stringify(config, null, 2)); + return paseoHome; +} + +describe("daemon browser tools config", () => { + afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); + }); + + test("defaults browser tools off when config is absent", async () => { + const home = await createPaseoHome({ version: 1 }); + + expect(loadConfig(home, { env: {} }).browserToolsEnabled).toBe(false); + }); + + test("loads browser tools opt-in from persisted daemon config", async () => { + const home = await createPaseoHome({ + version: 1, + daemon: { browserTools: { enabled: true } }, + }); + + expect(loadConfig(home, { env: {} }).browserToolsEnabled).toBe(true); + }); +}); diff --git a/packages/server/src/server/config.ts b/packages/server/src/server/config.ts index f1e4b4bd8..be9e68efc 100644 --- a/packages/server/src/server/config.ts +++ b/packages/server/src/server/config.ts @@ -399,6 +399,10 @@ function resolveAppendSystemPrompt(persisted: ReturnType): boolean { + return persisted.daemon?.browserTools?.enabled ?? false; +} + function resolveStaticLoadConfigSettings( env: NodeJS.ProcessEnv, cli: CliConfigOverrides | undefined, @@ -408,6 +412,7 @@ function resolveStaticLoadConfigSettings( mcpEnabled: cli?.mcpEnabled ?? persisted.daemon?.mcp?.enabled ?? true, mcpInjectIntoAgents: cli?.mcpInjectIntoAgents ?? persisted.daemon?.mcp?.injectIntoAgents ?? false, + browserToolsEnabled: resolveBrowserToolsEnabled(persisted), autoArchiveAfterMerge: persisted.daemon?.autoArchiveAfterMerge ?? false, appendSystemPrompt: resolveAppendSystemPrompt(persisted), terminalProfiles: persisted.daemon?.terminalProfiles, @@ -435,6 +440,7 @@ export function loadConfig( const { mcpEnabled, mcpInjectIntoAgents, + browserToolsEnabled, autoArchiveAfterMerge, appendSystemPrompt, terminalProfiles, @@ -472,6 +478,7 @@ export function loadConfig( trustedProxies, mcpEnabled, mcpInjectIntoAgents, + browserToolsEnabled, autoArchiveAfterMerge, enableTerminalAgentHooks: persisted.daemon?.enableTerminalAgentHooks ?? false, appendSystemPrompt, diff --git a/packages/server/src/server/daemon-config-store.test.ts b/packages/server/src/server/daemon-config-store.test.ts index cd7f90b65..9d741da84 100644 --- a/packages/server/src/server/daemon-config-store.test.ts +++ b/packages/server/src/server/daemon-config-store.test.ts @@ -94,6 +94,7 @@ describe("DaemonConfigStore", () => { paseoHome, { mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, providers: {}, metadataGeneration: { providers: [] }, autoArchiveAfterMerge: false, @@ -126,6 +127,7 @@ describe("DaemonConfigStore", () => { paseoHome, { mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, providers: {}, metadataGeneration: { providers: [] }, autoArchiveAfterMerge: false, @@ -143,6 +145,29 @@ describe("DaemonConfigStore", () => { expect(persisted.daemon?.appendSystemPrompt).toBe("Prefer terse replies."); }); + test("patch persists browser tools opt-in into config.json", () => { + const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-")); + tempDirs.push(paseoHome); + + const store = new DaemonConfigStore( + paseoHome, + { + mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, + providers: {}, + metadataGeneration: { providers: [] }, + autoArchiveAfterMerge: false, + appendSystemPrompt: "", + }, + undefined, + ); + + store.patch({ browserTools: { enabled: true } }); + + const persisted = loadPersistedConfig(paseoHome); + expect(persisted.daemon?.browserTools).toEqual({ enabled: true }); + }); + test("patch persists provider additional models into config.json", () => { const paseoHome = mkdtempSync(path.join(tmpdir(), "paseo-daemon-config-store-")); tempDirs.push(paseoHome); @@ -151,6 +176,7 @@ describe("DaemonConfigStore", () => { paseoHome, { mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, providers: {}, metadataGeneration: { providers: [] }, autoArchiveAfterMerge: false, @@ -192,6 +218,7 @@ describe("DaemonConfigStore", () => { paseoHome, { mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, providers: {}, metadataGeneration: { providers: [] }, autoArchiveAfterMerge: false, @@ -240,6 +267,7 @@ describe("DaemonConfigStore", () => { paseoHome, { mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, providers: {}, metadataGeneration: { providers: [] }, autoArchiveAfterMerge: false, @@ -292,6 +320,7 @@ describe("DaemonConfigStore", () => { paseoHome, { mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, providers: {}, autoArchiveAfterMerge: false, enableTerminalAgentHooks: false, @@ -315,6 +344,7 @@ describe("DaemonConfigStore", () => { paseoHome, { mcp: { injectIntoAgents: false }, + browserTools: { enabled: false }, providers: {}, autoArchiveAfterMerge: false, enableTerminalAgentHooks: false, diff --git a/packages/server/src/server/daemon-config-store.ts b/packages/server/src/server/daemon-config-store.ts index 24ef05b19..1876322e4 100644 --- a/packages/server/src/server/daemon-config-store.ts +++ b/packages/server/src/server/daemon-config-store.ts @@ -172,6 +172,7 @@ function mergeMutableConfigIntoPersistedConfig(params: { mutable: MutableDaemonConfig; }): PersistedConfig { const { persisted, mutable } = params; + const browserToolsEnabled = readBrowserToolsEnabled(mutable); const metadataGenerationProviders = readMetadataGenerationProviders(mutable); const providerOverrides = applyMutableProviderConfigToOverrides( persisted.agents?.providers as Record | undefined, @@ -208,6 +209,10 @@ function mergeMutableConfigIntoPersistedConfig(params: { ...persisted.daemon?.mcp, injectIntoAgents: mutable.mcp.injectIntoAgents, }, + browserTools: { + ...persisted.daemon?.browserTools, + enabled: browserToolsEnabled, + }, autoArchiveAfterMerge: mutable.autoArchiveAfterMerge, enableTerminalAgentHooks: mutable.enableTerminalAgentHooks, appendSystemPrompt: mutable.appendSystemPrompt, @@ -219,6 +224,14 @@ function mergeMutableConfigIntoPersistedConfig(params: { } as PersistedConfig; } +function readBrowserToolsEnabled(mutable: MutableDaemonConfig): boolean { + const browserTools = mutable.browserTools; + if (!isRecord(browserTools)) { + return false; + } + return browserTools["enabled"] === true; +} + function readMetadataGenerationProviders( mutable: MutableDaemonConfig, ): Array<{ provider: string; model?: string; thinkingOptionId?: string }> { diff --git a/packages/server/src/server/persisted-config.test.ts b/packages/server/src/server/persisted-config.test.ts index fed19c468..da846342d 100644 --- a/packages/server/src/server/persisted-config.test.ts +++ b/packages/server/src/server/persisted-config.test.ts @@ -46,6 +46,18 @@ describe("PersistedConfigSchema daemon append system prompt config", () => { }); }); +describe("PersistedConfigSchema daemon browser tools config", () => { + test("accepts optional browser tools opt-in", () => { + const parsed = PersistedConfigSchema.parse({ + daemon: { + browserTools: { enabled: true }, + }, + }); + + expect(parsed.daemon?.browserTools?.enabled).toBe(true); + }); +}); + describe("PersistedConfigSchema daemon relay config", () => { test("accepts optional relay TLS setting", () => { const parsed = PersistedConfigSchema.parse({ diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index 6b9bc3483..5bc4f3cc9 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -240,6 +240,12 @@ export const PersistedConfigSchema = z }) .passthrough() .optional(), + browserTools: z + .object({ + enabled: z.boolean().optional(), + }) + .passthrough() + .optional(), autoArchiveAfterMerge: z.boolean().optional(), enableTerminalAgentHooks: z.boolean().optional(), appendSystemPrompt: z.string().optional(), diff --git a/packages/server/src/server/session.workspaces.test.ts b/packages/server/src/server/session.workspaces.test.ts index b50b76be4..8c911fcab 100644 --- a/packages/server/src/server/session.workspaces.test.ts +++ b/packages/server/src/server/session.workspaces.test.ts @@ -3816,7 +3816,7 @@ test("open_project_request does not match a new child directory to an existing p const session = createSessionForWorkspaceTests(); const projects = new Map>(); const workspaces = new Map>(); - const home = path.resolve("/Users/moboudra"); + const home = path.resolve("/home/developer"); const worktree = path.join(home, ".paseo", "worktrees", "project-config-lifecycle-textarea"); projects.set( @@ -3825,7 +3825,7 @@ test("open_project_request does not match a new child directory to an existing p projectId: home, rootPath: home, kind: "non_git", - displayName: "moboudra", + displayName: "developer", createdAt: "2026-04-24T09:00:00.000Z", updatedAt: "2026-04-24T09:00:00.000Z", }), @@ -3837,7 +3837,7 @@ test("open_project_request does not match a new child directory to an existing p projectId: home, cwd: home, kind: "directory", - displayName: "moboudra", + displayName: "developer", createdAt: "2026-04-24T09:00:00.000Z", updatedAt: "2026-04-24T09:00:00.000Z", }), @@ -3882,7 +3882,7 @@ test("open_project_request does not unarchive an archived parent workspace for a const session = createSessionForWorkspaceTests(); const projects = new Map>(); const workspaces = new Map>(); - const home = path.resolve("/Users/moboudra"); + const home = path.resolve("/home/developer"); const worktree = path.join(home, ".paseo", "worktrees", "project-config-lifecycle-textarea"); const archivedAt = "2026-04-24T08:00:00.000Z"; @@ -3950,9 +3950,9 @@ test("open_project_request reclassifies an archived directory workspace when git const session = createSessionForWorkspaceTests(); const projects = new Map>(); const workspaces = new Map>(); - const repoRoot = path.resolve("/Users/moboudra/dev/paseo"); + const repoRoot = path.resolve("/home/developer/dev/paseo"); const cwd = path.join( - path.resolve("/Users/moboudra"), + path.resolve("/home/developer"), ".paseo", "worktrees", "orchestrate", @@ -4049,9 +4049,9 @@ test("open_project_request reclassifies an active directory workspace when git m const session = createSessionForWorkspaceTests(); const projects = new Map>(); const workspaces = new Map>(); - const repoRoot = path.resolve("/Users/moboudra/dev/paseo"); + const repoRoot = path.resolve("/home/developer/dev/paseo"); const cwd = path.join( - path.resolve("/Users/moboudra"), + path.resolve("/home/developer"), ".paseo", "worktrees", "orchestrate", @@ -4167,9 +4167,9 @@ test("open_project_request groups a plain git worktree under an existing repo pr const session = createSessionForWorkspaceTests(); const projects = new Map>(); const workspaces = new Map>(); - const repoRoot = path.resolve("/Users/moboudra/dev/paseo"); + const repoRoot = path.resolve("/home/developer/dev/paseo"); const cwd = path.join( - path.resolve("/Users/moboudra"), + path.resolve("/home/developer"), ".paseo", "worktrees", "orchestrate", diff --git a/packages/server/src/server/test-utils/daemon-client.ts b/packages/server/src/server/test-utils/daemon-client.ts index e54470735..ac18c041d 100644 --- a/packages/server/src/server/test-utils/daemon-client.ts +++ b/packages/server/src/server/test-utils/daemon-client.ts @@ -12,7 +12,7 @@ import { export type DaemonClientConfig = Omit< SharedDaemonClientConfig, "webSocketFactory" | "transportFactory" | "clientId" ->; +> & { clientId?: string }; export type CreateAgentOptions = CreateAgentRequestOptions; export { type SendMessageOptions, type DaemonEvent, type DaemonEventHandler }; @@ -25,7 +25,7 @@ function nextTestClientId(): string { export class DaemonClient extends SharedDaemonClient { constructor(config: DaemonClientConfig) { - const clientId = nextTestClientId(); + const clientId = config.clientId ?? nextTestClientId(); super({ ...config, clientId, diff --git a/packages/server/src/server/websocket-server.browser-tools.test.ts b/packages/server/src/server/websocket-server.browser-tools.test.ts new file mode 100644 index 000000000..90c3eebad --- /dev/null +++ b/packages/server/src/server/websocket-server.browser-tools.test.ts @@ -0,0 +1,406 @@ +import { createServer, type Server as HTTPServer } from "node:http"; +import type { AddressInfo } from "node:net"; + +import type { + BrowserAutomationExecuteRequest, + BrowserAutomationExecuteResponse, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; +import type pino from "pino"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { AgentManager } from "./agent/agent-manager.js"; +import type { AgentStorage } from "./agent/agent-storage.js"; +import { BrowserToolsBroker } from "./browser-tools/broker.js"; +import { StaticBrowserToolsPolicy } from "./browser-tools/policy.js"; +import type { CheckoutDiffManager } from "./checkout-diff-manager.js"; +import type { FileBackedChatService } from "./chat/chat-service.js"; +import type { DaemonConfigStore } from "./daemon-config-store.js"; +import type { DownloadTokenStore } from "./file-download/token-store.js"; +import type { LoopService } from "./loop-service.js"; +import type { ScheduleService } from "./schedule/service.js"; +import { createStub } from "./test-utils/class-mocks.js"; +import { DaemonClient } from "./test-utils/daemon-client.js"; +import { createProviderSnapshotManagerStub } from "./test-utils/session-stubs.js"; +import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; + +interface BrowserToolsDaemonHarness { + broker: BrowserToolsBroker; + connectDesktopBrowserClient( + options?: ConnectDesktopBrowserClientOptions, + ): Promise; + stop(): Promise; +} + +interface ConnectDesktopBrowserClientOptions { + clientId?: string; + capabilities?: Partial>; +} + +interface DesktopBrowserClientHandle { + clientId: string; + nextBrowserRequest(): Promise; + respondToBrowserRequest(response: BrowserAutomationExecuteResponse): void; + disconnect(): Promise; +} + +interface QueuedBrowserRequests { + next(): Promise; + push(request: BrowserAutomationExecuteRequest): void; + close(): void; +} + +const harnesses: BrowserToolsDaemonHarness[] = []; + +afterEach(async () => { + await Promise.all(harnesses.splice(0).map((harness) => harness.stop())); +}); + +describe("WebSocketServer browser tools wiring", () => { + it("registers capable clients and dispatches broker requests over the real WebSocket path", async () => { + const harness = await startBrowserToolsDaemonHarness(); + const desktop = await harness.connectDesktopBrowserClient(); + + const resultPromise = harness.broker.execute({ + command: { command: "list_tabs", args: {} }, + }); + const request = await desktop.nextBrowserRequest(); + + expect(request).toMatchObject({ + type: "browser.automation.execute.request", + requestId: "req-1", + command: { command: "list_tabs", args: {} }, + }); + + desktop.respondToBrowserRequest({ + type: "browser.automation.execute.response", + payload: { + requestId: request.requestId, + ok: true, + result: { command: "list_tabs", tabs: [] }, + }, + }); + + await expect(resultPromise).resolves.toEqual({ + requestId: request.requestId, + ok: true, + result: { command: "list_tabs", tabs: [] }, + }); + }); + + it("unregisters capable clients on disconnect and clears pending browser commands", async () => { + const harness = await startBrowserToolsDaemonHarness(); + const desktop = await harness.connectDesktopBrowserClient(); + + const pendingResult = harness.broker.execute({ + command: { command: "list_tabs", args: {} }, + }); + const pendingExpectation = expect(pendingResult).resolves.toMatchObject({ + ok: false, + error: { code: "browser_no_desktop", retryable: true }, + }); + await desktop.nextBrowserRequest(); + + expect(harness.broker.getPendingRequestCount()).toBe(1); + + await desktop.disconnect(); + + expect(harness.broker.getRegisteredClientCount()).toBe(0); + expect(harness.broker.getPendingRequestCount()).toBe(0); + await pendingExpectation; + + await expect( + harness.broker.execute({ command: { command: "list_tabs", args: {} } }), + ).resolves.toMatchObject({ + ok: false, + error: { code: "browser_no_desktop" }, + }); + }); + + it("keeps browser automation registered when a desktop browser client resumes", async () => { + const harness = await startBrowserToolsDaemonHarness(); + const clientId = "desktop-client-1"; + await harness.connectDesktopBrowserClient({ + clientId, + capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true }, + }); + + const resumedDesktop = await harness.connectDesktopBrowserClient({ + clientId, + capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true }, + }); + + const resultPromise = harness.broker.execute({ + command: { command: "click", args: { ref: "@e1" } }, + }); + const request = await resumedDesktop.nextBrowserRequest(); + resumedDesktop.respondToBrowserRequest({ + type: "browser.automation.execute.response", + payload: { + requestId: request.requestId, + ok: true, + result: { command: "click", browserId: "browser-1", ref: "@e1" }, + }, + }); + + await expect(resultPromise).resolves.toMatchObject({ + ok: true, + result: { command: "click", browserId: "browser-1", ref: "@e1" }, + }); + }); +}); + +async function startBrowserToolsDaemonHarness(): Promise { + const httpServer = createServer(); + const broker = createBroker(); + const wsServer = createVoiceAssistantWebSocketServer({ httpServer, broker }); + const clients = new Set(); + + await listen(httpServer); + const url = `ws://127.0.0.1:${getPort(httpServer)}/ws`; + + const harness: BrowserToolsDaemonHarness = { + broker, + async connectDesktopBrowserClient(options = {}) { + const clientId = options.clientId; + const client = new DaemonClient({ + url, + ...(clientId ? { clientId } : {}), + clientType: "browser", + connectTimeoutMs: 500, + reconnect: { enabled: false }, + capabilities: options.capabilities ?? { [CLIENT_CAPS.desktopBrowserAutomation]: true }, + }); + clients.add(client); + + const requests = createBrowserRequestQueue(); + client.on("browser.automation.execute.request", (request) => { + requests.push(request); + }); + + await client.connect(); + + return { + clientId: clientId ?? "", + nextBrowserRequest: () => requests.next(), + respondToBrowserRequest: (response) => + client.sendBrowserAutomationExecuteResponse(response), + async disconnect() { + requests.close(); + clients.delete(client); + await client.close(); + await waitFor(() => broker.getRegisteredClientCount() === 0); + }, + }; + }, + async stop() { + await Promise.all(Array.from(clients, (client) => client.close())); + clients.clear(); + await wsServer.close(); + await closeHttpServer(httpServer); + }, + }; + + harnesses.push(harness); + return harness; +} + +function createBroker(): BrowserToolsBroker { + return new BrowserToolsBroker({ + policy: new StaticBrowserToolsPolicy(true), + defaultTimeoutMs: 500, + createRequestId: createRequestIdSequence(), + }); +} + +function createRequestIdSequence(): () => string { + let index = 0; + return () => { + index += 1; + return `req-${index}`; + }; +} + +function createVoiceAssistantWebSocketServer(params: { + httpServer: HTTPServer; + broker: BrowserToolsBroker; +}): VoiceAssistantWebSocketServer { + const { httpServer, broker } = params; + const agentManager = { + setAgentAttentionCallback: vi.fn(), + subscribe: vi.fn(() => () => {}), + getMetricsSnapshot: vi.fn(() => ({ + total: 0, + byLifecycle: {}, + withActiveForegroundTurn: 0, + timelineStats: { totalItems: 0, maxItemsPerAgent: 0 }, + })), + }; + const daemonConfigStore = { + onChange: vi.fn(() => () => {}), + }; + + return new VoiceAssistantWebSocketServer( + httpServer, + createStub(createLogger()), + "srv-test", + createStub(agentManager), + createStub({}), + createStub({}), + "/tmp/paseo-browser-tools-websocket-test", + createStub(daemonConfigStore), + null, + { allowedOrigins: new Set(["*"]) }, + undefined, + undefined, + undefined, + undefined, + "1.2.3-test", + undefined, + undefined, + undefined, + createStub({}), + createStub({}), + createStub({}), + createStub({ + subscribe: vi.fn(), + scheduleRefreshForCwd: vi.fn(), + getMetrics: vi.fn(() => ({ + checkoutDiffTargetCount: 0, + checkoutDiffSubscriptionCount: 0, + checkoutDiffWatcherCount: 0, + checkoutDiffFallbackRefreshTargetCount: 0, + })), + dispose: vi.fn(), + }), + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + createProviderSnapshotManagerStub().manager, + undefined, + undefined, + broker, + ); +} + +function createBrowserRequestQueue(): QueuedBrowserRequests { + const requests: BrowserAutomationExecuteRequest[] = []; + const waiters: Array<{ + resolve: (request: BrowserAutomationExecuteRequest) => void; + reject: (error: Error) => void; + }> = []; + let closed = false; + + return { + next() { + const request = requests.shift(); + if (request) { + return Promise.resolve(request); + } + if (closed) { + return Promise.reject(new Error("Desktop browser client disconnected")); + } + return new Promise((resolve, reject) => { + let timeout: ReturnType; + const waiter = { + resolve: (value: BrowserAutomationExecuteRequest) => { + clearTimeout(timeout); + resolve(value); + }, + reject: (error: Error) => { + clearTimeout(timeout); + reject(error); + }, + }; + timeout = setTimeout(() => { + const waiterIndex = waiters.indexOf(waiter); + if (waiterIndex !== -1) { + waiters.splice(waiterIndex, 1); + } + reject(new Error("Timed out waiting for browser automation request")); + }, 500); + waiters.push(waiter); + }); + }, + push(request) { + const waiter = waiters.shift(); + if (waiter) { + waiter.resolve(request); + return; + } + requests.push(request); + }, + close() { + closed = true; + for (const waiter of waiters.splice(0)) { + waiter.reject(new Error("Desktop browser client disconnected")); + } + }, + }; +} + +function createLogger() { + const logger = { + child: vi.fn(() => logger), + trace: vi.fn(), + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }; + return logger; +} + +function listen(server: HTTPServer): Promise { + return new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + resolve(); + }); + }); +} + +function getPort(server: HTTPServer): number { + const address = server.address(); + if (!isAddressInfo(address)) { + throw new Error("HTTP test server did not bind to a TCP port"); + } + return address.port; +} + +function isAddressInfo(address: string | AddressInfo | null): address is AddressInfo { + return typeof address === "object" && address !== null && typeof address.port === "number"; +} + +function closeHttpServer(server: HTTPServer): Promise { + if (!server.listening) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }); +} + +async function waitFor(predicate: () => boolean): Promise { + const startedAt = Date.now(); + while (!predicate()) { + if (Date.now() - startedAt > 500) { + throw new Error("Timed out waiting for browser tools WebSocket state"); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } +} diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 5387b63ea..4f048637d 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -69,6 +69,9 @@ import { CLIENT_SHUTDOWN_RPC_REASON, normalizeClientRestartRpcReason, } from "./lifecycle-reasons.js"; +import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; +import type { BrowserAutomationExecuteResponse } from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import type { BrowserToolsBroker } from "./browser-tools/broker.js"; const WS_CLOSE_DAEMON_AUTH_FAILED = 4401; @@ -304,6 +307,12 @@ function bufferFromWsData(data: Buffer | ArrayBuffer | Buffer[] | string): Buffe return Buffer.from(data); } +function hasDesktopBrowserAutomationCapability( + capabilities: Record | null, +): boolean { + return capabilities?.[CLIENT_CAPS.desktopBrowserAutomation] === true; +} + interface WebSocketLike { readyState: number; bufferedAmount?: number; @@ -323,6 +332,10 @@ interface SessionConnection { externalDisconnectCleanupTimeout: ReturnType | null; } +interface BrowserToolsRegistration { + unregister: () => void; +} + const SLOW_REQUEST_THRESHOLD_MS = 500; const EXTERNAL_SESSION_DISCONNECT_GRACE_MS = 90_000; const HELLO_TIMEOUT_MS = 15_000; @@ -438,6 +451,8 @@ export class VoiceAssistantWebSocketServer { private unsubscribeDaemonConfigChange: (() => void) | null = null; private readonly providerUsageService: ProviderUsageService; private unsubscribeTerminalActivity: (() => void) | null = null; + private readonly browserToolsBroker: BrowserToolsBroker | null; + private readonly browserToolsRegistrations = new Map(); constructor( server: HTTPServer, @@ -491,6 +506,7 @@ export class VoiceAssistantWebSocketServer { }; }, serviceProxyPublicBaseUrl?: string | null, + browserToolsBroker?: BrowserToolsBroker | null, ) { this.logger = logger.child({ module: "websocket-server" }); this.serverId = serverId; @@ -499,6 +515,7 @@ export class VoiceAssistantWebSocketServer { } this.daemonVersion = daemonVersion.trim(); this.daemonRuntimeConfig = daemonRuntimeConfig; + this.browserToolsBroker = browserToolsBroker ?? null; this.agentManager = agentManager; this.agentStorage = agentStorage; this.projectRegistry = projectRegistry ?? createNoopProjectRegistry(); @@ -842,6 +859,9 @@ export class VoiceAssistantWebSocketServer { this.sessions.clear(); this.socketIdentities.clear(); this.externalSessionsByKey.clear(); + for (const clientId of this.browserToolsRegistrations.keys()) { + this.unregisterBrowserToolsClient(clientId); + } this.wss.close(); } @@ -1124,10 +1144,12 @@ export class VoiceAssistantWebSocketServer { ) { existing.clientCapabilities = newClientCapabilities; existing.session.updateClientCapabilities(newClientCapabilities); + this.syncBrowserToolsClientRegistration(existing); } existing.sockets.add(ws); this.sessions.set(ws, existing); pending.identity.sessionId = existing.session.getSessionId(); + this.syncBrowserToolsClientRegistration(existing); this.sendToClient(ws, this.createServerInfoMessage()); pending.connectionLogger.info( { @@ -1152,6 +1174,7 @@ export class VoiceAssistantWebSocketServer { this.sessions.set(ws, connection); this.externalSessionsByKey.set(clientId, connection); pending.identity.sessionId = connection.session.getSessionId(); + this.syncBrowserToolsClientRegistration(connection); this.sendToClient(ws, this.createServerInfoMessage()); connection.connectionLogger.info( { @@ -1314,6 +1337,7 @@ export class VoiceAssistantWebSocketServer { this.socketIdentities.delete(ws); if (connection.sockets.size === 0) { + this.unregisterBrowserToolsClient(connection.clientId); this.incrementRuntimeCounter("sessionDisconnectedWaitingReconnect"); if (connection.externalDisconnectCleanupTimeout) { clearTimeout(connection.externalDisconnectCleanupTimeout); @@ -1375,6 +1399,7 @@ export class VoiceAssistantWebSocketServer { if (existing === connection) { this.externalSessionsByKey.delete(connection.clientId); } + this.unregisterBrowserToolsClient(connection.clientId); connection.connectionLogger.trace( { clientId: connection.clientId, totalSessions: this.sessions.size }, @@ -1383,6 +1408,39 @@ export class VoiceAssistantWebSocketServer { await connection.session.cleanup(); } + private syncBrowserToolsClientRegistration(connection: SessionConnection): void { + if (!this.browserToolsBroker) { + return; + } + if (!hasDesktopBrowserAutomationCapability(connection.clientCapabilities)) { + this.unregisterBrowserToolsClient(connection.clientId); + return; + } + const existing = this.browserToolsRegistrations.get(connection.clientId); + if (existing) { + return; + } + + const unregister = this.browserToolsBroker.registerClient({ + id: connection.clientId, + sendBrowserAutomationRequest: (request) => { + this.sendToConnection(connection, wrapSessionMessage(request)); + }, + }); + this.browserToolsRegistrations.set(connection.clientId, { + unregister, + }); + } + + private unregisterBrowserToolsClient(clientId: string): void { + const registration = this.browserToolsRegistrations.get(clientId); + if (!registration) { + return; + } + this.browserToolsRegistrations.delete(clientId); + registration.unregister(); + } + private handleInvalidInboundMessage(args: { ws: WebSocketLike; parsed: unknown; @@ -1625,6 +1683,11 @@ export class VoiceAssistantWebSocketServer { "ws_control_rpc_received", ); } + if (message.message.type === "browser.automation.execute.response") { + this.browserToolsBroker?.receiveResponse(message.message as BrowserAutomationExecuteResponse); + return; + } + const startMs = performance.now(); await activeConnection.session.handleMessage(message.message); const durationMs = performance.now() - startMs; diff --git a/paseo.json b/paseo.json index b98a48a69..50ebd4399 100644 --- a/paseo.json +++ b/paseo.json @@ -12,19 +12,22 @@ "scripts": { "daemon": { "type": "service", - "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_SKIP_DEV_SERVER_BUILD=1 PASEO_LISTEN=0.0.0.0:$PASEO_PORT ./scripts/dev-daemon.sh" + "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"${PASEO_WORKTREE_PATH:-$PWD}\" PASEO_HOME=\"${PASEO_WORKTREE_PATH:-$PWD}/.dev/paseo-home\" PASEO_SKIP_DEV_SERVER_BUILD=1 PASEO_LISTEN=0.0.0.0:${PASEO_PORT:-6768} ./scripts/dev-daemon.sh" }, "app": { "type": "service", - "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT} EXPO_PORT=$PASEO_PORT ./scripts/dev-app.sh" + "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"${PASEO_WORKTREE_PATH:-$PWD}\" PASEO_HOME=\"${PASEO_WORKTREE_PATH:-$PWD}/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT:-6768} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT:-6768} EXPO_PORT=${PASEO_PORT:-} ./scripts/dev-app.sh" }, "desktop": { "type": "service", - "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT} EXPO_PORT=$PASEO_PORT npm run dev --workspace=@getpaseo/desktop" + "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"${PASEO_WORKTREE_PATH:-$PWD}\" PASEO_HOME=\"${PASEO_WORKTREE_PATH:-$PWD}/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT:-6768} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT:-6768} EXPO_PORT=${PASEO_PORT:-} npm run dev --workspace=@getpaseo/desktop" }, "ios-simulator": { "type": "service", "command": "cross-env PASEO_DEV_MANAGED_HOME=1 PASEO_DEV_ROOT=\"$PWD\" PASEO_HOME=\"$PWD/.dev/paseo-home\" PASEO_LISTEN=0.0.0.0:${PASEO_SERVICE_DAEMON_PORT} PASEO_DEV_DAEMON_ENDPOINT=localhost:${PASEO_SERVICE_DAEMON_PORT} ./scripts/paseo-ios-simulator-service.sh" + }, + "typecheck": { + "command": "npm run typecheck" } } }