diff --git a/docs/architecture.md b/docs/architecture.md index fed447e4b..744220be4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -138,7 +138,7 @@ Electron wrapper for macOS, Linux, and Windows. > **Window-state v1 limitation:** only the _first_ window of a session restores and persists saved geometry (size/position/maximized). Windows opened via ⌘⇧N / second-instance / "Open in new window" open at the default size, OS-cascaded, and do not persist — this avoids every window stacking on the same restored bounds and fighting over the single window-state store. Lifting this needs per-window state keys. > -> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus and agent automation targets are intentionally separate: the workspace-active browser follows the user's focused tab, while the agent-active browser is the default target for browser MCP commands. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up. +> **In-app browser panes are not yet per-window.** Browser webviews are tracked by one process-global registry that keeps a single current `WebContents` per browser id. Human focus still records the workspace-active browser for UI state and `list_tabs` reporting, but agent automation targets only explicit browser ids returned by `browser_new_tab` or `browser_list_tabs`. The webview registration queue (`pendingBrowserWebviewIds` in `main.ts`) is still process-global. With browser panes open in two windows, a menu Reload can target the other window's webview, and near-simultaneous webview attach across windows can register under the wrong browser id. Multi-window v1 ships windows; making the browser-webview subsystem window-scoped is a follow-up. ### `packages/website` — Marketing site diff --git a/packages/app/src/browser-automation/handler.test.ts b/packages/app/src/browser-automation/handler.test.ts index 2e72a4726..718894b7c 100644 --- a/packages/app/src/browser-automation/handler.test.ts +++ b/packages/app/src/browser-automation/handler.test.ts @@ -73,7 +73,7 @@ function browserNewTabRequest(): BrowserAutomationExecuteRequest { workspaceId: "wks_workspace_a", command: { command: "new_tab", - args: { workspaceId: "wks_workspace_a", url: "https://example.com" }, + args: { url: "https://example.com" }, }, }; } @@ -128,7 +128,6 @@ describe("mountBrowserAutomationHandler", () => { test("creates a focused workspace browser tab for browser_new_tab", async () => { const client = new FakeDaemonClient(); const setWorkspaceActiveBrowser = vi.fn(async () => undefined); - const setAgentActiveBrowser = vi.fn(async () => undefined); const registerWorkspaceBrowser = vi.fn(async () => undefined); const ensureResidentBrowserWebview = vi.fn(); const executeAutomationCommand = vi.fn(async () => currentListTabsPayload()); @@ -150,7 +149,6 @@ describe("mountBrowserAutomationHandler", () => { executeAutomationCommand, registerWorkspaceBrowser, setWorkspaceActiveBrowser, - setAgentActiveBrowser, }, }) satisfies DesktopHostBridge, ensureResidentBrowserWebview, @@ -180,10 +178,6 @@ describe("mountBrowserAutomationHandler", () => { browserId, workspaceId: "wks_workspace_a", }); - expect(setAgentActiveBrowser).toHaveBeenCalledWith({ - agentId: "agent-1", - browserId, - }); expect(setWorkspaceActiveBrowser).toHaveBeenCalledWith({ browserId, workspaceId: "wks_workspace_a", diff --git a/packages/app/src/browser-automation/handler.ts b/packages/app/src/browser-automation/handler.ts index b122d846d..a2ea259e9 100644 --- a/packages/app/src/browser-automation/handler.ts +++ b/packages/app/src/browser-automation/handler.ts @@ -111,8 +111,6 @@ async function handleBrowserAutomationRequest(params: { return; } - await rememberAgentBrowserTarget({ request, browserHost }); - if (!executeAutomationCommand) { client.sendBrowserAutomationExecuteResponse({ type: "browser.automation.execute.response", @@ -159,11 +157,11 @@ async function openBrowserTabForRequest(params: { BrowserAutomationExecuteRequest["command"], { command: "new_tab" } >; - const workspaceId = request.workspaceId ?? command.args.workspaceId; + const workspaceId = request.workspaceId; if (!serverId || !workspaceId) { return browserAutomationFailure({ requestId: request.requestId, - code: "browser_no_tab", + code: "browser_unsupported", message: "Cannot create a browser tab without a workspace context.", }); } @@ -174,7 +172,7 @@ async function openBrowserTabForRequest(params: { if (!workspaceKey) { return browserAutomationFailure({ requestId: request.requestId, - code: "browser_no_tab", + code: "browser_unsupported", message: "Cannot create a browser tab without a workspace context.", }); } @@ -185,9 +183,6 @@ async function openBrowserTabForRequest(params: { await browserHost?.registerWorkspaceBrowser?.({ browserId, workspaceId }); await browserHost?.setWorkspaceActiveBrowser?.({ browserId, workspaceId }); - if (request.agentId) { - await browserHost?.setAgentActiveBrowser?.({ agentId: request.agentId, browserId }); - } if (browserHost?.executeAutomationCommand) { ensureResidentBrowserWebview({ browserId, url: normalizedUrl }); @@ -236,7 +231,7 @@ async function waitForBrowserRegistration(params: { agentId: params.request.agentId, cwd: params.request.cwd, workspaceId: params.workspaceId, - command: { command: "list_tabs", args: { workspaceId: params.workspaceId } }, + command: { command: "list_tabs", args: {} }, }); if (payload.ok && payload.result.command === "list_tabs") { if (payload.result.tabs.some((tab) => tab.browserId === params.browserId)) { @@ -252,28 +247,6 @@ function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -async function rememberAgentBrowserTarget(input: { - request: BrowserAutomationExecuteRequest; - browserHost: DesktopHostBridge["browser"] | undefined; -}): Promise { - if (!input.request.agentId) { - return; - } - const browserId = readRequestBrowserId(input.request); - if (!browserId) { - return; - } - await input.browserHost?.setAgentActiveBrowser?.({ agentId: input.request.agentId, browserId }); -} - -function readRequestBrowserId(request: BrowserAutomationExecuteRequest): string | null { - if (request.browserId) { - return request.browserId; - } - const args = request.command.args as { browserId?: unknown }; - return typeof args.browserId === "string" && args.browserId.length > 0 ? args.browserId : null; -} - function normalizeBridgePayload( requestId: string, payload: BrowserAutomationResponsePayload, diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index bcb5d4839..2380fcc81 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -132,7 +132,6 @@ export interface DesktopBrowserBridge { workspaceId: string; browserId: string | null; }) => Promise; - setAgentActiveBrowser?: (input: { agentId: string; browserId: string | null }) => Promise; openDevTools?: (browserId: string) => Promise; clearPartition?: (browserId: string) => Promise; executeAutomationCommand?: ( diff --git a/packages/app/src/stores/browser-store/index.ts b/packages/app/src/stores/browser-store/index.ts index c16a6f4a0..d7ff86b06 100644 --- a/packages/app/src/stores/browser-store/index.ts +++ b/packages/app/src/stores/browser-store/index.ts @@ -1,4 +1,5 @@ import AsyncStorage from "@react-native-async-storage/async-storage"; +import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { create } from "zustand"; import { createJSONStorage, persist } from "zustand/middleware"; import { @@ -22,10 +23,14 @@ interface BrowserStoreState extends BrowserIndexState { } function createBrowserId(): string { + let browserId: string; if (typeof globalThis.crypto?.randomUUID === "function") { - return globalThis.crypto.randomUUID(); + browserId = globalThis.crypto.randomUUID(); + } else { + const randomSuffix = Math.random().toString(16).slice(2) || "0"; + browserId = `${Date.now()}-${randomSuffix}`; } - return `${Date.now()}-${Math.random().toString(16).slice(2)}`; + return BrowserAutomationBrowserIdSchema.parse(browserId); } export const useBrowserStore = create()( diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 3e5994561..94660a772 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -14,9 +14,7 @@ import { listRegisteredPaseoBrowserIds, listRegisteredPaseoBrowserIdsForWorkspace, getPaseoBrowserWebContents, - getWorkspaceActivePaseoBrowserWebContents, getWorkspaceActivePaseoBrowserId, - getAgentActivePaseoBrowserId, getPaseoBrowserWorkspaceId, } from "../browser-webviews/index.js"; @@ -149,12 +147,7 @@ function createRegistry(): BrowserRegistry { return contents ? adaptWebContents(contents) : null; }, getBrowserWorkspaceId: getPaseoBrowserWorkspaceId, - getWorkspaceActiveTabContents(workspaceId: string): TabContents | null { - const contents = getWorkspaceActivePaseoBrowserWebContents(workspaceId); - return contents ? adaptWebContents(contents) : null; - }, getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId, - getAgentActiveBrowserId: getAgentActivePaseoBrowserId, }; } diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index 8a902ba6d..0b2acae09 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -1,2222 +1,451 @@ -import { resolve as resolvePath } from "node:path"; - -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, test } from "vitest"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; -import type { TabContents, BrowserRegistry, TabImage } from "./service.js"; +import type { BrowserRegistry, TabContents, TabImage } from "./service.js"; import { executeAutomationCommand } from "./service.js"; -function fakeTab(overrides: Partial & { id: number }): TabContents { - return { - getURL: () => "https://example.com", - getTitle: () => "Example", - canGoBack: () => false, - canGoForward: () => false, - isLoading: () => false, - isDestroyed: () => false, - executeJavaScript: async () => "[]", - loadURL: async () => {}, - goBack: () => {}, - goForward: () => {}, - reload: () => {}, - capturePage: async () => ({ - toPNG: () => new Uint8Array([137, 80, 78, 71]), - getSize: () => ({ width: 10, height: 5 }), - }), - invalidate: () => {}, - isBackgroundThrottlingAllowed: () => true, - setBackgroundThrottling: () => {}, - ...overrides, - }; -} +const BROWSER_A = "11111111-1111-4111-8111-111111111111"; +const BROWSER_B = "22222222-2222-4222-8222-222222222222"; -function createRegistry(overrides: Partial = {}): BrowserRegistry { - return { - listRegisteredBrowserIds: () => [], - listRegisteredBrowserIdsForWorkspace: () => [], - getTabContents: () => null, - getBrowserWorkspaceId: () => null, - getWorkspaceActiveTabContents: () => null, - getWorkspaceActiveBrowserId: () => null, - getAgentActiveBrowserId: () => null, - ...overrides, - }; -} - -function hasScriptWith(scripts: string[], first: string, second: string): boolean { - for (const script of scripts) { - if (script.includes(first) && script.includes(second)) { - return true; - } +class FakeImage implements TabImage { + public toPNG(): Uint8Array { + return new Uint8Array([137, 80, 78, 71]); + } + + public getSize(): { width: number; height: number } { + return { width: 10, height: 5 }; } - return false; } -const TAB_A = fakeTab({ id: 1, getURL: () => "https://a.com", getTitle: () => "Tab A" }); -const TAB_B = fakeTab({ id: 2, getURL: () => "https://b.com", getTitle: () => "Tab B" }); +class FakeTab implements TabContents { + public readonly loadedUrls: string[] = []; + public readonly scripts: string[] = []; + public readonly actions: string[] = []; + public readonly capturedViewports: Array<{ stayHidden?: boolean }> = []; + public destroyed = false; + public bodyText = ""; + public snapshotElements: unknown[] = []; + + public constructor( + public readonly id: number, + private readonly url: string, + private readonly title: string, + ) {} + + public getURL(): string { + return this.loadedUrls.at(-1) ?? this.url; + } + + public getTitle(): string { + return this.title; + } + + public canGoBack(): boolean { + return true; + } + + public canGoForward(): boolean { + return false; + } + + public isLoading(): boolean { + return false; + } + + public isDestroyed(): boolean { + return this.destroyed; + } + + public async executeJavaScript(code: string): Promise { + this.scripts.push(code); + if (code.includes("document.body.innerText")) { + return this.bodyText; + } + if (code.includes("querySelectorAll")) { + return JSON.stringify(this.snapshotElements); + } + return true; + } + + public async loadURL(url: string): Promise { + this.loadedUrls.push(url); + } + + public goBack(): void { + this.actions.push("back"); + } + + public goForward(): void { + this.actions.push("forward"); + } + + public reload(): void { + this.actions.push("reload"); + } + + public async capturePage(options?: { stayHidden?: boolean }): Promise { + this.capturedViewports.push(options ?? {}); + return new FakeImage(); + } + + public invalidate(): void { + this.actions.push("invalidate"); + } + + public isBackgroundThrottlingAllowed(): boolean { + return true; + } + + public setBackgroundThrottling(allowed: boolean): void { + this.actions.push(`background:${allowed}`); + } +} + +class FakeRegistry implements BrowserRegistry { + private readonly tabs = new Map(); + + public activeBrowserId: string | null = null; + + public register(browserId: string, workspaceId: string, tab: FakeTab): void { + this.tabs.set(browserId, { workspaceId, tab }); + } + + public listRegisteredBrowserIds(): string[] { + return Array.from(this.tabs.keys()); + } + + public listRegisteredBrowserIdsForWorkspace(workspaceId: string): string[] { + return Array.from(this.tabs.entries()) + .filter((entry) => entry[1].workspaceId === workspaceId) + .map((entry) => entry[0]); + } + + public getTabContents(browserId: string): TabContents | null { + return this.tabs.get(browserId)?.tab ?? null; + } + + public getBrowserWorkspaceId(browserId: string): string | null { + return this.tabs.get(browserId)?.workspaceId ?? null; + } + + public getWorkspaceActiveBrowserId(): string | null { + return this.activeBrowserId; + } +} + +function pageRequest(command: { command: "page_info"; args: { browserId: string } }) { + return { + type: "browser.automation.execute.request" as const, + requestId: "req-page", + workspaceId: "workspace-a", + command, + }; +} describe("executeAutomationCommand", () => { - describe("list_tabs", () => { - it("returns registered tabs with url/title/active data", () => { - const registry = createRegistry({ - listRegisteredBrowserIds: () => ["a", "b"], - getTabContents: (id) => { - if (id === "a") return TAB_A; - if (id === "b") return TAB_B; - return null; - }, - getBrowserWorkspaceId: (id) => (id === "a" || id === "b" ? "workspace-a" : null), - getWorkspaceActiveBrowserId: () => "a", - }); + test("list tabs reports workspace ownership and active tab information", () => { + const tabA = new FakeTab(1, "https://a.test", "A"); + const tabB = new FakeTab(2, "https://b.test", "B"); + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-a", tabA); + registry.register(BROWSER_B, "workspace-b", tabB); + registry.activeBrowserId = BROWSER_A; - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r1", - command: { command: "list_tabs", args: {} }, - }, - registry, - ); + const result = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-list", + workspaceId: "workspace-a", + command: { command: "list_tabs", args: {} }, + }, + registry, + ); - expect(result).toEqual({ - requestId: "r1", - ok: true, - result: { - command: "list_tabs", - tabs: [ - { - browserId: "a", - workspaceId: "workspace-a", - url: "https://a.com", - title: "Tab A", - isActive: false, - isLoading: false, - canGoBack: false, - canGoForward: false, - }, - { - browserId: "b", - workspaceId: "workspace-a", - url: "https://b.com", - title: "Tab B", - isActive: false, - isLoading: false, - canGoBack: false, - canGoForward: false, - }, - ], - }, - }); - }); - - it("skips destroyed tabs", () => { - const destroyedTab = fakeTab({ id: 3, isDestroyed: () => true }); - const registry = createRegistry({ - listRegisteredBrowserIds: () => ["a", "dead"], - getTabContents: (id) => { - if (id === "a") return TAB_A; - if (id === "dead") return destroyedTab; - return null; - }, - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r2", - command: { command: "list_tabs", args: {} }, - }, - registry, - ); - - expect(result.ok).toBe(true); - if (!result.ok) throw new Error("expected success"); - expect(result.result.command).toBe("list_tabs"); - expect(result.result.tabs).toHaveLength(1); - expect(result.result.tabs[0]?.browserId).toBe("a"); - }); - - it("returns empty list when no tabs registered", () => { - const registry = createRegistry(); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r3", - command: { command: "list_tabs", args: {} }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r3", - ok: true, - result: { command: "list_tabs", tabs: [] }, - }); - }); - - it("lists only tabs owned by the requested workspace", () => { - const registry = createRegistry({ - listRegisteredBrowserIdsForWorkspace: (workspaceId) => { - if (workspaceId === "workspace-a") return ["a"]; - if (workspaceId === "workspace-b") return ["b"]; - return []; - }, - getTabContents: (id) => { - if (id === "a") return TAB_A; - if (id === "b") return TAB_B; - return null; - }, - getBrowserWorkspaceId: (id) => { - if (id === "a") return "workspace-a"; - if (id === "b") return "workspace-b"; - return null; - }, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : "b"), - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-workspace-list", - workspaceId: "workspace-a", - command: { command: "list_tabs", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(result.ok).toBe(true); - if (!result.ok) throw new Error("expected success"); - expect(result.result.command).toBe("list_tabs"); - expect(result.result.tabs).toHaveLength(1); - expect(result.result.tabs[0]?.browserId).toBe("a"); - expect(result.result.tabs[0]?.workspaceId).toBe("workspace-a"); - }); - }); - - describe("page_info", () => { - it("uses explicit browserId", () => { - const registry = createRegistry({ - getTabContents: (id) => (id === "b" ? TAB_B : null), - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r4", - browserId: "b", - command: { command: "page_info", args: { browserId: "b" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r4", - ok: true, - result: { - command: "page_info", - tab: { - browserId: "b", - url: "https://b.com", - title: "Tab B", - isActive: false, - isLoading: false, - canGoBack: false, - canGoForward: false, - }, - }, - }); - }); - - it("uses the top-level browserId when command args omit it", () => { - const registry = createRegistry({ - getTabContents: (id) => (id === "b" ? TAB_B : null), - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-top-level-browser-id", - browserId: "b", - command: { command: "page_info", args: {} }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-top-level-browser-id", - ok: true, - result: { - command: "page_info", - tab: { - browserId: "b", - url: "https://b.com", - title: "Tab B", - isActive: false, - isLoading: false, - canGoBack: false, - canGoForward: false, - }, - }, - }); - }); - - it("uses active workspace browser when browserId omitted", () => { - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? TAB_A : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r5", - workspaceId: "workspace-a", - command: { command: "page_info", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r5", - ok: true, - result: { - command: "page_info", - tab: { - browserId: "a", + expect(result).toEqual({ + requestId: "req-list", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: BROWSER_A, workspaceId: "workspace-a", - url: "https://a.com", - title: "Tab A", + url: "https://a.test", + title: "A", isActive: true, isLoading: false, - canGoBack: false, + canGoBack: true, canGoForward: false, }, - }, - }); - }); - - it("uses the agent active browser before the human-focused workspace browser", () => { - const registry = createRegistry({ - getTabContents: (id) => { - if (id === "agent-browser") return TAB_B; - if (id === "human-browser") return TAB_A; - return null; - }, - getBrowserWorkspaceId: (id) => - id === "agent-browser" || id === "human-browser" ? "workspace-a" : null, - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? TAB_A : null, - getWorkspaceActiveBrowserId: (workspaceId) => - workspaceId === "workspace-a" ? "human-browser" : null, - getAgentActiveBrowserId: (agentId) => (agentId === "agent-1" ? "agent-browser" : null), - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-agent-active", - agentId: "agent-1", - workspaceId: "workspace-a", - command: { command: "page_info", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-agent-active", - ok: true, - result: { - command: "page_info", - tab: { - browserId: "agent-browser", - workspaceId: "workspace-a", - url: "https://b.com", - title: "Tab B", - isActive: false, - isLoading: false, - canGoBack: false, - canGoForward: false, - }, - }, - }); - }); - - it("returns canGoBack/canGoForward when available", () => { - const tabWithNav = fakeTab({ - id: 10, - canGoBack: () => true, - canGoForward: () => true, - }); - const registry = createRegistry({ - getTabContents: () => tabWithNav, - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-nav", - browserId: "x", - command: { command: "page_info", args: { browserId: "x" } }, - }, - registry, - ); - - expect(result.ok).toBe(true); - if (!result.ok) throw new Error("expected success"); - expect(result.result.command).toBe("page_info"); - expect(result.result.tab.canGoBack).toBe(true); - expect(result.result.tab.canGoForward).toBe(true); - }); - - it("returns browser_no_tab when no active workspace browser", () => { - const registry = createRegistry({ - getWorkspaceActiveTabContents: () => null, - getWorkspaceActiveBrowserId: () => null, - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r6", - command: { command: "page_info", args: {} }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r6", - ok: false, - error: { - code: "browser_no_tab", - message: "No active browser tab in workspace", - retryable: false, - }, - }); - }); - - it("returns browser_tab_not_found for missing explicit browserId", () => { - const registry = createRegistry({ getTabContents: () => null }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r7", - browserId: "missing", - command: { command: "page_info", args: { browserId: "missing" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r7", - ok: false, - error: { - code: "browser_tab_not_found", - message: "No browser tab found for ID: missing", - retryable: false, - }, - }); - }); - - it("returns browser_tab_not_found when explicit browserId belongs to another workspace", () => { - const registry = createRegistry({ - getTabContents: (id) => (id === "b" ? TAB_B : null), - getBrowserWorkspaceId: (id) => (id === "b" ? "workspace-b" : null), - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-cross-workspace-browser-id", - workspaceId: "workspace-a", - browserId: "b", - command: { - command: "page_info", - args: { workspaceId: "workspace-a", browserId: "b" }, - }, - }, - registry, - ); - - expect(result.ok).toBe(false); - if (result.ok) throw new Error("expected failure"); - expect(result.error.code).toBe("browser_tab_not_found"); - }); - - it("returns browser_tab_closed for destroyed tab", () => { - const destroyedTab = fakeTab({ id: 99, isDestroyed: () => true }); - const registry = createRegistry({ - getTabContents: () => destroyedTab, - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r8", - browserId: "dead", - command: { command: "page_info", args: { browserId: "dead" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r8", - ok: false, - error: { - code: "browser_tab_closed", - message: "Browser tab dead has been closed", - retryable: false, - }, - }); - }); - - it("returns browser_tab_closed for destroyed active workspace tab", () => { - const destroyedTab = fakeTab({ id: 99, isDestroyed: () => true }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? destroyedTab : null, - getWorkspaceActiveBrowserId: (workspaceId) => - workspaceId === "workspace-a" ? "dead" : null, - }); - - const result = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-active-dead", - workspaceId: "workspace-a", - command: { command: "page_info", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(result.ok).toBe(false); - if (result.ok) throw new Error("expected failure"); - expect(result.error.code).toBe("browser_tab_closed"); - }); - }); - - describe("set_background", () => { - it("sets the active tab page background color", async () => { - const scripts: string[] = []; - const tab = fakeTab({ - id: 99, - executeJavaScript: async (script) => { - scripts.push(script); - return true; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: () => tab, - getWorkspaceActiveBrowserId: () => "browser-1", - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-bg", - workspaceId: "workspace-a", - command: { - command: "set_background", - args: { workspaceId: "workspace-a", color: "red" }, - }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-bg", - ok: true, - result: { command: "set_background", browserId: "browser-1", color: "red" }, - }); - expect(hasScriptWith(scripts, "document.body.style.background", "red")).toBe(true); - }); - }); - - describe("snapshot", () => { - it("returns snapshot refs for the active workspace browser", async () => { - const tab = fakeTab({ - id: 5, - getURL: () => "https://example.com/form", - getTitle: () => "Fixture", - executeJavaScript: async () => - JSON.stringify([ - { - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, - }, - { - role: "button", - tagName: "button", - text: "Greet", - selector: "button", - attributes: {}, - }, - ]), - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot", - workspaceId: "workspace-a", - command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-snapshot", - ok: true, - result: { - command: "snapshot", - browserId: "a", - workspaceId: "workspace-a", - url: "https://example.com/form", - title: "Fixture", - elements: [ - { - ref: "@e1", - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, - }, - { - ref: "@e2", - role: "button", - tagName: "button", - text: "Greet", - selector: "button", - attributes: {}, - }, - ], - }, - }); - }); - }); - - describe("click and fill", () => { - it("fills and clicks refs from the latest snapshot", async () => { - const executedScripts: string[] = []; - const tab = fakeTab({ - id: 6, - getURL: () => "https://example.com/form", - getTitle: () => "Fixture", - executeJavaScript: async (script) => { - executedScripts.push(script); - if (script.includes("CANDIDATE_SELECTOR")) { - return JSON.stringify([ - { - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, - }, - { - role: "button", - tagName: "button", - text: "Greet", - selector: "#greet", - attributes: { id: "greet" }, - }, - ]); - } - return true; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot-for-actions", - workspaceId: "workspace-a", - command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - const fillResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-fill", - workspaceId: "workspace-a", - command: { - command: "fill", - args: { workspaceId: "workspace-a", ref: "@e1", value: "Ada" }, - }, - }, - registry, - ); - const clickResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-click", - workspaceId: "workspace-a", - command: { command: "click", args: { workspaceId: "workspace-a", ref: "@e2" } }, - }, - registry, - ); - - expect(fillResult).toEqual({ - requestId: "r-fill", - ok: true, - result: { command: "fill", browserId: "a", ref: "@e1" }, - }); - expect(clickResult).toEqual({ - requestId: "r-click", - ok: true, - result: { command: "click", browserId: "a", ref: "@e2" }, - }); - let filledName = false; - let clickedGreet = false; - for (const script of executedScripts) { - filledName ||= script.includes("#name") && script.includes("Ada"); - clickedGreet ||= script.includes("#greet") && script.includes("click"); - } - expect(filledName).toBe(true); - expect(clickedGreet).toBe(true); - }); - - it("returns browser_stale_ref when the page has navigated since the snapshot", async () => { - let currentUrl = "https://example.com/form"; - const tab = fakeTab({ - id: 7, - getURL: () => currentUrl, - executeJavaScript: async () => - JSON.stringify([ - { - role: "button", - tagName: "button", - text: "Greet", - selector: "#greet", - attributes: { id: "greet" }, - }, - ]), - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot-before-nav", - workspaceId: "workspace-a", - command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - currentUrl = "https://example.com/next"; - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-stale-click", - workspaceId: "workspace-a", - command: { command: "click", args: { workspaceId: "workspace-a", ref: "@e1" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-stale-click", - ok: false, - error: { - code: "browser_stale_ref", - message: "Browser element reference @e1 is stale. Take a new snapshot and try again.", - retryable: false, - }, - }); - }); - - it("returns browser_stale_ref when a same-URL DOM change removes the ref", async () => { - const tab = fakeTab({ - id: 26, - getURL: () => "https://example.com/form", - executeJavaScript: async (script) => { - if (script.includes("CANDIDATE_SELECTOR")) { - return JSON.stringify([ - { - role: "button", - tagName: "button", - text: "Greet", - selector: "#greet", - attributes: { id: "greet" }, - }, - ]); - } - return false; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot-before-dom-removal", - workspaceId: "workspace-a", - command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-dom-removed-click", - workspaceId: "workspace-a", - command: { command: "click", args: { workspaceId: "workspace-a", ref: "@e1" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-dom-removed-click", - ok: false, - error: { - code: "browser_stale_ref", - message: "Browser element reference @e1 is stale. Take a new snapshot and try again.", - retryable: false, - }, - }); - }); - - it("focuses, clears, checks, and selects snapshot refs", async () => { - const executedScripts: string[] = []; - const tab = fakeTab({ - id: 17, - getURL: () => "https://example.com/controls", - executeJavaScript: async (script) => { - executedScripts.push(script); - if (script.includes("CANDIDATE_SELECTOR")) { - return JSON.stringify([ - { - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, - }, - { - role: "checkbox", - tagName: "input", - text: "Subscribe", - selector: "#subscribe", - attributes: { id: "subscribe", type: "checkbox" }, - }, - { - role: "combobox", - tagName: "select", - text: "Country", - selector: "#country", - attributes: { id: "country" }, - }, - { - role: "button", - tagName: "button", - text: "Preview", - selector: "#preview", - attributes: { id: "preview" }, - }, - { - role: "generic", - tagName: "div", - text: "Drop zone", - selector: "#drop-zone", - attributes: { id: "drop-zone", tabindex: "0" }, - }, - ]); - } - return true; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot-for-controls", - workspaceId: "workspace-a", - command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - const focusResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-focus", - workspaceId: "workspace-a", - command: { command: "focus", args: { workspaceId: "workspace-a", ref: "@e1" } }, - }, - registry, - ); - const clearResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-clear", - workspaceId: "workspace-a", - command: { command: "clear", args: { workspaceId: "workspace-a", ref: "@e1" } }, - }, - registry, - ); - const checkResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-check", - workspaceId: "workspace-a", - command: { - command: "check", - args: { workspaceId: "workspace-a", ref: "@e2", checked: true }, - }, - }, - registry, - ); - const selectResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-select", - workspaceId: "workspace-a", - command: { - command: "select", - args: { workspaceId: "workspace-a", ref: "@e3", value: "us" }, - }, - }, - registry, - ); - const hoverResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-hover", - workspaceId: "workspace-a", - command: { command: "hover", args: { workspaceId: "workspace-a", ref: "@e4" } }, - }, - registry, - ); - const dragResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-drag", - workspaceId: "workspace-a", - command: { - command: "drag", - args: { workspaceId: "workspace-a", sourceRef: "@e4", targetRef: "@e5" }, - }, - }, - registry, - ); - - expect(focusResult).toEqual({ - requestId: "r-focus", - ok: true, - result: { command: "focus", browserId: "a", ref: "@e1" }, - }); - expect(clearResult).toEqual({ - requestId: "r-clear", - ok: true, - result: { command: "clear", browserId: "a", ref: "@e1" }, - }); - expect(checkResult).toEqual({ - requestId: "r-check", - ok: true, - result: { command: "check", browserId: "a", ref: "@e2", checked: true }, - }); - expect(selectResult).toEqual({ - requestId: "r-select", - ok: true, - result: { command: "select", browserId: "a", ref: "@e3", value: "us" }, - }); - expect(hoverResult).toEqual({ - requestId: "r-hover", - ok: true, - result: { command: "hover", browserId: "a", ref: "@e4" }, - }); - expect(dragResult).toEqual({ - requestId: "r-drag", - ok: true, - result: { command: "drag", browserId: "a", sourceRef: "@e4", targetRef: "@e5" }, - }); - expect(hasScriptWith(executedScripts, "#name", "focus")).toBe(true); - expect(hasScriptWith(executedScripts, "#name", "deleteContent")).toBe(true); - expect(hasScriptWith(executedScripts, "#subscribe", "checked")).toBe(true); - expect(hasScriptWith(executedScripts, "#country", "us")).toBe(true); - expect(hasScriptWith(executedScripts, "#preview", "mouseover")).toBe(true); - expect(hasScriptWith(executedScripts, "#drop-zone", "dragover")).toBe(true); - }); - }); - - describe("wait", () => { - it("waits until page text appears", async () => { - let reads = 0; - const tab = fakeTab({ - id: 8, - getURL: () => "https://example.com/wait", - executeJavaScript: async () => { - reads += 1; - return reads >= 2 ? "Loading\nReady" : "Loading"; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-wait-text", - workspaceId: "workspace-a", - command: { - command: "wait", - args: { workspaceId: "workspace-a", text: "Ready", timeoutMs: 100 }, - }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-wait-text", - ok: true, - result: { command: "wait", browserId: "a", matched: "text" }, - }); - }); - - it("returns browser_timeout when waited-for text does not appear", async () => { - const tab = fakeTab({ - id: 9, - getURL: () => "https://example.com/wait", - executeJavaScript: async () => "Loading", - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-wait-timeout", - workspaceId: "workspace-a", - command: { - command: "wait", - args: { workspaceId: "workspace-a", text: "Ready", timeoutMs: 1 }, - }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-wait-timeout", - ok: false, - error: { - code: "browser_timeout", - message: "Timed out waiting for browser text: Ready", - retryable: true, - }, - }); - }); - }); - - describe("type and keypress", () => { - it("types text into a snapshot ref and dispatches keypress", async () => { - const executedScripts: string[] = []; - const tab = fakeTab({ - id: 10, - getURL: () => "https://example.com/type", - executeJavaScript: async (script) => { - executedScripts.push(script); - if (script.includes("CANDIDATE_SELECTOR")) { - return JSON.stringify([ - { - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name" }, - }, - ]); - } - return true; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot-for-type", - workspaceId: "workspace-a", - command: { command: "snapshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - const typeResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-type", - workspaceId: "workspace-a", - command: { - command: "type", - args: { workspaceId: "workspace-a", ref: "@e1", text: "Ada" }, - }, - }, - registry, - ); - const keyResult = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-keypress", - workspaceId: "workspace-a", - command: { - command: "keypress", - args: { workspaceId: "workspace-a", ref: "@e1", key: "Enter" }, - }, - }, - registry, - ); - - expect(typeResult).toEqual({ - requestId: "r-type", - ok: true, - result: { command: "type", browserId: "a", ref: "@e1" }, - }); - expect(keyResult).toEqual({ - requestId: "r-keypress", - ok: true, - result: { command: "keypress", browserId: "a", key: "Enter", ref: "@e1" }, - }); - let typedAda = false; - let pressedEnter = false; - for (const script of executedScripts) { - typedAda ||= script.includes("#name") && script.includes("Ada"); - pressedEnter ||= script.includes("#name") && script.includes("Enter"); - } - expect(typedAda).toBe(true); - expect(pressedEnter).toBe(true); - }); - }); - - describe("logs", () => { - it("returns recent console messages and network performance entries", async () => { - const tab = fakeTab({ - id: 19, - getConsoleMessages: () => [ - { level: "info", message: "first", timestamp: 1 }, - { level: "error", message: "second", source: "fixture", line: 7, timestamp: 2 }, ], - executeJavaScript: async () => - JSON.stringify([ - { - url: "https://example.com/app.js", - type: "script", - startTime: 3, - duration: 4, - transferSize: 100, - }, - ]), - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-logs", - workspaceId: "workspace-a", - command: { command: "logs", args: { workspaceId: "workspace-a", maxEntries: 1 } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-logs", - ok: true, - result: { - command: "logs", - browserId: "a", - console: [ - { level: "error", message: "second", source: "fixture", line: 7, timestamp: 2 }, - ], - network: [ - { - url: "https://example.com/app.js", - type: "script", - startTime: 3, - duration: 4, - transferSize: 100, - }, - ], - }, - }); + }, }); }); - describe("storage", () => { - it("returns cookies, localStorage, and sessionStorage for the target tab", async () => { - const tab = fakeTab({ - id: 20, - getURL: () => "https://example.com/storage", - getCookies: async () => [ - { name: "theme", value: "dark", domain: "example.com", httpOnly: true }, + test("page info reads the explicit browser id from command args", () => { + const tab = new FakeTab(1, "https://a.test", "A"); + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-a", tab); + + const result = executeAutomationCommand( + pageRequest({ command: "page_info", args: { browserId: BROWSER_A } }), + registry, + ); + + expect(result).toEqual({ + requestId: "req-page", + ok: true, + result: { + command: "page_info", + tab: { + browserId: BROWSER_A, + workspaceId: "workspace-a", + url: "https://a.test", + title: "A", + isActive: false, + isLoading: false, + canGoBack: true, + canGoForward: false, + }, + }, + }); + }); + + test("page info returns tab not found for an id in another workspace", () => { + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-b", new FakeTab(1, "https://a.test", "A")); + + const result = executeAutomationCommand( + pageRequest({ command: "page_info", args: { browserId: BROWSER_A } }), + registry, + ); + + expect(result).toEqual({ + requestId: "req-page", + ok: false, + error: { + code: "browser_tab_not_found", + message: `No browser tab found for ID: ${BROWSER_A}`, + retryable: false, + }, + }); + }); + + test("page info returns tab closed for a destroyed explicit tab", () => { + const tab = new FakeTab(1, "https://a.test", "A"); + tab.destroyed = true; + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-a", tab); + + const result = executeAutomationCommand( + pageRequest({ command: "page_info", args: { browserId: BROWSER_A } }), + registry, + ); + + expect(result).toEqual({ + requestId: "req-page", + ok: false, + error: { + code: "browser_tab_closed", + message: `Browser tab ${BROWSER_A} has been closed`, + retryable: false, + }, + }); + }); + + test("snapshot and click use refs from the same explicit tab", async () => { + const tab = new FakeTab(1, "https://a.test/form", "Form"); + tab.snapshotElements = [ + { + role: "button", + tagName: "button", + text: "Submit", + selector: "#submit", + attributes: { id: "submit" }, + }, + ]; + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-a", tab); + const snapshotEngine = new BrowserSnapshotEngine(); + + const snapshot = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-snapshot", + workspaceId: "workspace-a", + command: { command: "snapshot", args: { browserId: BROWSER_A } }, + }, + registry, + { snapshotEngine }, + ); + const click = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-click", + workspaceId: "workspace-a", + command: { command: "click", args: { browserId: BROWSER_A, ref: "@e1" } }, + }, + registry, + { snapshotEngine }, + ); + + expect(snapshot).toEqual({ + requestId: "req-snapshot", + ok: true, + result: { + command: "snapshot", + browserId: BROWSER_A, + workspaceId: "workspace-a", + url: "https://a.test/form", + title: "Form", + elements: [ + { + ref: "@e1", + role: "button", + tagName: "button", + text: "Submit", + selector: "#submit", + attributes: { id: "submit" }, + }, ], - executeJavaScript: async () => - JSON.stringify({ - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }), - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-storage", - workspaceId: "workspace-a", - command: { command: "storage", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-storage", - ok: true, - result: { - command: "storage", - browserId: "a", - url: "https://example.com/storage", - cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }, - }); + }, + }); + expect(click).toEqual({ + requestId: "req-click", + ok: true, + result: { command: "click", browserId: BROWSER_A, ref: "@e1" }, }); }); - describe("environment", () => { - it("sets viewport and geolocation for the target tab", async () => { - const debugCommands: Array<{ command: string; params?: Record }> = []; - const scripts: string[] = []; - const tab = fakeTab({ - id: 21, - sendDebugCommand: async (command, params) => { - debugCommands.push({ command, params }); - }, - executeJavaScript: async (script) => { - scripts.push(script); - if (script.includes("window.innerWidth")) { - return JSON.stringify({ width: 390, height: 844, deviceScaleFactor: 3 }); - } - return true; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); + test("wait resolves when the explicit tab contains the requested text", async () => { + const tab = new FakeTab(1, "https://a.test", "A"); + tab.bodyText = "Ready"; + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-a", tab); - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-environment", - workspaceId: "workspace-a", - command: { - command: "environment", - args: { - workspaceId: "workspace-a", - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-wait", + workspaceId: "workspace-a", + command: { + command: "wait", + args: { browserId: BROWSER_A, text: "Ready", timeoutMs: 100 }, }, - registry, - ); + }, + registry, + ); - expect(debugCommands).toEqual([ - { - command: "Emulation.setDeviceMetricsOverride", - params: { width: 390, height: 844, deviceScaleFactor: 3, mobile: false }, - }, - { - command: "Emulation.setGeolocationOverride", - params: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - ]); - expect(hasScriptWith(scripts, "navigator", "geolocation")).toBe(true); - expect(result).toEqual({ - requestId: "r-environment", - ok: true, - result: { - command: "environment", - browserId: "a", - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }); + expect(result).toEqual({ + requestId: "req-wait", + ok: true, + result: { command: "wait", browserId: BROWSER_A, matched: "text" }, }); }); - describe("full-page screenshot and PDF", () => { - it("captures a full-page screenshot through CDP", async () => { - const debugCommands: Array<{ command: string; params?: Record }> = []; - const tab = fakeTab({ - id: 22, - sendDebugCommand: async (command, params) => { - debugCommands.push({ command, params }); - if (command === "Page.getLayoutMetrics") { - return { contentSize: { width: 390.2, height: 1200.1 } }; - } - return { data: "iVBORw0KGgo=" }; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); + test("navigate loads the requested URL in the explicit tab", async () => { + const tab = new FakeTab(1, "https://a.test", "A"); + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-a", tab); - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-full-page", - workspaceId: "workspace-a", - command: { command: "full_page_screenshot", args: { workspaceId: "workspace-a" } }, + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-navigate", + workspaceId: "workspace-a", + command: { + command: "navigate", + args: { browserId: BROWSER_A, url: "https://example.com/next" }, }, - registry, - ); + }, + registry, + ); - expect(debugCommands).toEqual([ - { command: "Page.getLayoutMetrics", params: undefined }, - { - command: "Page.captureScreenshot", - params: { - format: "png", - captureBeyondViewport: true, - clip: { x: 0, y: 0, width: 391, height: 1201, scale: 1 }, - }, - }, - ]); - expect(result).toEqual({ - requestId: "r-full-page", - ok: true, - result: { - command: "full_page_screenshot", - browserId: "a", - mimeType: "image/png", - dataBase64: "iVBORw0KGgo=", - width: 391, - height: 1201, - }, - }); - }); - - it("returns screenshot_no_frame when full-page CDP capture never paints", async () => { - vi.useFakeTimers(); - try { - const tab = fakeTab({ - id: 22, - sendDebugCommand: async (command) => { - if (command === "Page.getLayoutMetrics") { - return { contentSize: { width: 390, height: 1200 } }; - } - return new Promise(() => {}); - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => - workspaceId === "workspace-a" ? "a" : null, - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const resultPromise = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-full-page-no-frame", - workspaceId: "workspace-a", - command: { command: "full_page_screenshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - await vi.advanceTimersByTimeAsync(5_000); - - await expect(resultPromise).resolves.toEqual({ - requestId: "r-full-page-no-frame", - ok: false, - error: { - code: "screenshot_no_frame", - message: - "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, - }, - }); - } finally { - vi.useRealTimers(); - } - }); - - it("exports the target tab as PDF", async () => { - const printOptions: Record[] = []; - const tab = fakeTab({ - id: 23, - printToPDF: async (options) => { - printOptions.push(options ?? {}); - return new Uint8Array([0x25, 0x50, 0x44, 0x46]); - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-pdf", - workspaceId: "workspace-a", - command: { - command: "pdf", - args: { workspaceId: "workspace-a", landscape: true, printBackground: false }, - }, - }, - registry, - ); - - expect(printOptions).toEqual([{ printBackground: false, landscape: true }]); - expect(result).toEqual({ - requestId: "r-pdf", - ok: true, - result: { - command: "pdf", - browserId: "a", - mimeType: "application/pdf", - dataBase64: "JVBERg==", - }, - }); + expect(result).toEqual({ + requestId: "req-navigate", + ok: true, + result: { command: "navigate", browserId: BROWSER_A, url: "https://example.com/next" }, }); + expect(tab.loadedUrls).toEqual(["https://example.com/next"]); }); - describe("download and upload", () => { - it("downloads a URL through the target tab", async () => { - const tab = fakeTab({ - id: 24, - downloadURL: async (input) => ({ - filePath: `/tmp/${input.fileName ?? "download"}`, - totalBytes: 5, - state: "completed", - }), - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); + test("navigation actions dispatch to the explicit tab", () => { + const tab = new FakeTab(1, "https://a.test", "A"); + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-a", tab); - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-download", - workspaceId: "workspace-a", - command: { - command: "download", - args: { - workspaceId: "workspace-a", - url: "https://example.com/file.txt", - fileName: "file.txt", - }, - }, - }, - registry, - ); + const back = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-back", + workspaceId: "workspace-a", + command: { command: "back", args: { browserId: BROWSER_A } }, + }, + registry, + ); + const forward = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-forward", + workspaceId: "workspace-a", + command: { command: "forward", args: { browserId: BROWSER_A } }, + }, + registry, + ); + const reload = executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-reload", + workspaceId: "workspace-a", + command: { command: "reload", args: { browserId: BROWSER_A } }, + }, + registry, + ); - expect(result).toEqual({ - requestId: "r-download", - ok: true, - result: { - command: "download", - browserId: "a", - url: "https://example.com/file.txt", - filePath: "/tmp/file.txt", - totalBytes: 5, - state: "completed", - }, - }); + expect(back).toEqual({ + requestId: "req-back", + ok: true, + result: { command: "back", browserId: BROWSER_A }, }); - - it("rejects downloads for non-http URLs", async () => { - const downloadedUrls: string[] = []; - const tab = fakeTab({ - id: 27, - downloadURL: async (input) => { - downloadedUrls.push(input.url); - return { filePath: "/tmp/download", state: "completed" }; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-download-file", - workspaceId: "workspace-a", - command: { - command: "download", - args: { workspaceId: "workspace-a", url: "file:///tmp/secret.txt" }, - }, - }, - registry, - ); - - expect(downloadedUrls).toEqual([]); - expect(result).toEqual({ - requestId: "r-download-file", - ok: false, - error: { - code: "browser_denied", - message: "Browser download only supports http and https URLs.", - retryable: false, - }, - }); + expect(forward).toEqual({ + requestId: "req-forward", + ok: true, + result: { command: "forward", browserId: BROWSER_A }, }); - - it("sets workspace files on a file input ref through CDP", async () => { - const debugCommands: Array<{ command: string; params?: Record }> = []; - const workspaceRoot = "/tmp/paseo-workspace-a"; - const workspaceOpaqueId = "wks_workspace_a"; - const tab = fakeTab({ - id: 25, - executeJavaScript: async () => - JSON.stringify([ - { - role: "textbox", - tagName: "input", - text: "", - selector: "#file", - attributes: { id: "file", type: "file" }, - }, - ]), - sendDebugCommand: async (command, params) => { - debugCommands.push({ command, params }); - if (command === "DOM.getDocument") { - return { root: { nodeId: 1 } }; - } - if (command === "DOM.querySelector") { - return { nodeId: 2 }; - } - return {}; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (requestedWorkspaceId) => - requestedWorkspaceId === workspaceOpaqueId ? tab : null, - getWorkspaceActiveBrowserId: (requestedWorkspaceId) => - requestedWorkspaceId === workspaceOpaqueId ? "a" : null, - getBrowserWorkspaceId: (id) => (id === "a" ? workspaceOpaqueId : null), - }); - const snapshotEngine = new BrowserSnapshotEngine(); - const snapshot = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot-upload", - cwd: workspaceRoot, - workspaceId: workspaceOpaqueId, - command: { command: "snapshot", args: { workspaceId: workspaceOpaqueId } }, - }, - registry, - { snapshotEngine }, - ); - if (!snapshot.ok || snapshot.result.command !== "snapshot") { - throw new Error("snapshot failed"); - } - const ref = snapshot.result.elements[0]?.ref; - if (!ref) { - throw new Error("missing upload ref"); - } - const uploadPath = resolvePath(workspaceRoot, "uploads/file.txt"); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-upload", - cwd: workspaceRoot, - workspaceId: workspaceOpaqueId, - command: { - command: "upload", - args: { workspaceId: workspaceOpaqueId, ref, filePaths: ["uploads/file.txt"] }, - }, - }, - registry, - { snapshotEngine }, - ); - - expect(debugCommands.at(-1)).toEqual({ - command: "DOM.setFileInputFiles", - params: { nodeId: 2, files: [uploadPath] }, - }); - expect(result).toEqual({ - requestId: "r-upload", - ok: true, - result: { - command: "upload", - browserId: "a", - ref, - filePaths: [uploadPath], - }, - }); - }); - - it("rejects upload paths outside the workspace", async () => { - const workspaceRoot = "/tmp/paseo-workspace-a"; - const workspaceOpaqueId = "wks_workspace_a"; - const debugCommands: Array<{ command: string; params?: Record }> = []; - const tab = fakeTab({ - id: 26, - executeJavaScript: async () => - JSON.stringify([ - { - role: "textbox", - tagName: "input", - text: "", - selector: "#file", - attributes: { id: "file", type: "file" }, - }, - ]), - sendDebugCommand: async (command, params) => { - debugCommands.push({ command, params }); - if (command === "DOM.getDocument") { - return { root: { nodeId: 1 } }; - } - if (command === "DOM.querySelector") { - return { nodeId: 2 }; - } - return {}; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (requestedWorkspaceId) => - requestedWorkspaceId === workspaceOpaqueId ? tab : null, - getWorkspaceActiveBrowserId: (requestedWorkspaceId) => - requestedWorkspaceId === workspaceOpaqueId ? "a" : null, - getBrowserWorkspaceId: (id) => (id === "a" ? workspaceOpaqueId : null), - }); - const snapshotEngine = new BrowserSnapshotEngine(); - const snapshot = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot-upload-outside", - cwd: workspaceRoot, - workspaceId: workspaceOpaqueId, - command: { command: "snapshot", args: { workspaceId: workspaceOpaqueId } }, - }, - registry, - { snapshotEngine }, - ); - if (!snapshot.ok || snapshot.result.command !== "snapshot") { - throw new Error("snapshot failed"); - } - const ref = snapshot.result.elements[0]?.ref; - if (!ref) { - throw new Error("missing upload ref"); - } - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-upload-outside", - cwd: workspaceRoot, - workspaceId: workspaceOpaqueId, - command: { - command: "upload", - args: { workspaceId: workspaceOpaqueId, ref, filePaths: ["../secret.txt"] }, - }, - }, - registry, - { snapshotEngine }, - ); - - expect(debugCommands).not.toContainEqual( - expect.objectContaining({ command: "DOM.setFileInputFiles" }), - ); - expect(result).toEqual({ - requestId: "r-upload-outside", - ok: false, - error: { - code: "browser_unsupported", - message: "browser_upload only accepts files inside the agent workspace.", - retryable: false, - }, - }); - }); - - it("rejects upload when the request has no cwd", async () => { - const workspaceOpaqueId = "wks_workspace_a"; - const debugCommands: Array<{ command: string; params?: Record }> = []; - const tab = fakeTab({ - id: 28, - executeJavaScript: async () => - JSON.stringify([ - { - role: "textbox", - tagName: "input", - text: "", - selector: "#file", - attributes: { id: "file", type: "file" }, - }, - ]), - sendDebugCommand: async (command, params) => { - debugCommands.push({ command, params }); - if (command === "DOM.getDocument") { - return { root: { nodeId: 1 } }; - } - if (command === "DOM.querySelector") { - return { nodeId: 2 }; - } - return {}; - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (requestedWorkspaceId) => - requestedWorkspaceId === workspaceOpaqueId ? tab : null, - getWorkspaceActiveBrowserId: (requestedWorkspaceId) => - requestedWorkspaceId === workspaceOpaqueId ? "a" : null, - getBrowserWorkspaceId: (id) => (id === "a" ? workspaceOpaqueId : null), - }); - const snapshotEngine = new BrowserSnapshotEngine(); - const snapshot = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-snapshot-upload-missing-cwd", - workspaceId: workspaceOpaqueId, - command: { command: "snapshot", args: { workspaceId: workspaceOpaqueId } }, - }, - registry, - { snapshotEngine }, - ); - if (!snapshot.ok || snapshot.result.command !== "snapshot") { - throw new Error("snapshot failed"); - } - const ref = snapshot.result.elements[0]?.ref; - if (!ref) { - throw new Error("missing upload ref"); - } - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-upload-missing-cwd", - workspaceId: workspaceOpaqueId, - command: { - command: "upload", - args: { workspaceId: workspaceOpaqueId, ref, filePaths: ["uploads/file.txt"] }, - }, - }, - registry, - { snapshotEngine }, - ); - - expect(debugCommands).not.toContainEqual( - expect.objectContaining({ command: "DOM.setFileInputFiles" }), - ); - expect(result).toEqual({ - requestId: "r-upload-missing-cwd", - ok: false, - error: { - code: "browser_unsupported", - message: "browser_upload requires request cwd", - retryable: false, - }, - }); + expect(reload).toEqual({ + requestId: "req-reload", + ok: true, + result: { command: "reload", browserId: BROWSER_A }, }); + expect(tab.actions).toEqual(["back", "forward", "reload"]); }); - describe("navigation", () => { - it("navigates the active workspace browser to a URL", async () => { - const navigatedUrls: string[] = []; - const tab = fakeTab({ - id: 11, - loadURL: async (url) => { - navigatedUrls.push(url); - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); + test("screenshot captures the explicit tab viewport", async () => { + const tab = new FakeTab(1, "https://a.test", "A"); + const registry = new FakeRegistry(); + registry.register(BROWSER_A, "workspace-a", tab); - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-navigate", - workspaceId: "workspace-a", - command: { - command: "navigate", - args: { workspaceId: "workspace-a", url: "https://example.com/next" }, - }, - }, - registry, - ); + const result = await executeAutomationCommand( + { + type: "browser.automation.execute.request", + requestId: "req-screenshot", + workspaceId: "workspace-a", + command: { command: "screenshot", args: { browserId: BROWSER_A } }, + }, + registry, + ); - expect(result).toEqual({ - requestId: "r-navigate", - ok: true, - result: { command: "navigate", browserId: "a", url: "https://example.com/next" }, - }); - expect(navigatedUrls).toEqual(["https://example.com/next"]); - }); - - it("rejects navigation to local files and script URLs", async () => { - const navigatedUrls: string[] = []; - const tab = fakeTab({ - id: 13, - loadURL: async (url) => { - navigatedUrls.push(url); - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - for (const [requestId, url] of [ - ["r-navigate-file", "file:///tmp/secret.txt"], - ["r-navigate-js", "javascript:alert(1)"], - ] as const) { - await expect( - executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId, - workspaceId: "workspace-a", - command: { command: "navigate", args: { workspaceId: "workspace-a", url } }, - }, - registry, - ), - ).resolves.toEqual({ - requestId, - ok: false, - error: { - code: "browser_denied", - message: "Browser navigation only supports http and https URLs.", - retryable: false, - }, - }); - } - - expect(navigatedUrls).toEqual([]); - }); - - it("dispatches back, forward, and reload to the active browser", async () => { - const actions: string[] = []; - const tab = fakeTab({ - id: 12, - goBack: () => actions.push("back"), - goForward: () => actions.push("forward"), - reload: () => actions.push("reload"), - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const back = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-back", - workspaceId: "workspace-a", - command: { command: "back", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - const forward = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-forward", - workspaceId: "workspace-a", - command: { command: "forward", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - const reload = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-reload", - workspaceId: "workspace-a", - command: { command: "reload", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(back).toEqual({ - requestId: "r-back", - ok: true, - result: { command: "back", browserId: "a" }, - }); - expect(forward).toEqual({ - requestId: "r-forward", - ok: true, - result: { command: "forward", browserId: "a" }, - }); - expect(reload).toEqual({ - requestId: "r-reload", - ok: true, - result: { command: "reload", browserId: "a" }, - }); - expect(actions).toEqual(["back", "forward", "reload"]); - }); - }); - - describe("screenshot", () => { - it("captures a viewport screenshot through Electron capturePage when CDP is available", async () => { - const actions: string[] = []; - let captureOptions: unknown; - const tab = fakeTab({ - id: 13, - sendDebugCommand: async (command) => { - throw new Error(`Unexpected CDP command ${command}`); - }, - capturePage: async (options) => { - actions.push("capture"); - captureOptions = options; - return { - toPNG: () => new Uint8Array([137, 80, 78, 71, 1, 2, 3]), - getSize: () => ({ width: 640, height: 480 }), - }; - }, - invalidate: () => { - actions.push("invalidate"); - }, - setBackgroundThrottling: (allowed) => { - actions.push(`background:${allowed}`); - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-screenshot-capture-page", - workspaceId: "workspace-a", - command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-screenshot-capture-page", - ok: true, - result: { - command: "screenshot", - browserId: "a", - mimeType: "image/png", - dataBase64: "iVBORwECAw==", - width: 640, - height: 480, - }, - }); - expect(captureOptions).toEqual({ stayHidden: false }); - expect(actions).toEqual(["background:false", "invalidate", "capture", "background:true"]); - }); - - it("serializes overlapping viewport captures before restoring background throttling", async () => { - const actions: string[] = []; - const captures: Array<{ resolve: (image: TabImage) => void }> = []; - let backgroundThrottlingAllowed = true; - const image: TabImage = { - toPNG: () => new Uint8Array([137, 80, 78, 71]), - getSize: () => ({ width: 640, height: 480 }), - }; - const tab = fakeTab({ - id: 13, - capturePage: async () => { - actions.push("capture"); - return new Promise((resolve) => { - captures.push({ resolve }); - }); - }, - invalidate: () => { - actions.push("invalidate"); - }, - isBackgroundThrottlingAllowed: () => backgroundThrottlingAllowed, - setBackgroundThrottling: (allowed) => { - backgroundThrottlingAllowed = allowed; - actions.push(`background:${allowed}`); - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const first = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-screenshot-overlap-first", - workspaceId: "workspace-a", - command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - const second = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-screenshot-overlap-second", - workspaceId: "workspace-a", - command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - await Promise.resolve(); - await Promise.resolve(); - expect(captures).toHaveLength(1); - expect(backgroundThrottlingAllowed).toBe(false); - - const firstCapture = captures[0]; - if (!firstCapture) { - throw new Error("expected first capture to start"); - } - firstCapture.resolve(image); - await expect(first).resolves.toMatchObject({ requestId: "r-screenshot-overlap-first" }); - await Promise.resolve(); - await Promise.resolve(); - expect(captures).toHaveLength(2); - expect(backgroundThrottlingAllowed).toBe(false); - - const secondCapture = captures[1]; - if (!secondCapture) { - throw new Error("expected second capture to start"); - } - secondCapture.resolve(image); - await expect(second).resolves.toMatchObject({ requestId: "r-screenshot-overlap-second" }); - - expect(backgroundThrottlingAllowed).toBe(true); - expect(actions).toEqual([ - "background:false", - "invalidate", - "capture", - "background:true", - "background:false", - "invalidate", - "capture", - "background:true", - ]); - }); - - it("returns screenshot_no_frame when viewport capture never paints", async () => { - vi.useFakeTimers(); - try { - const actions: string[] = []; - let backgroundThrottlingAllowed = false; - const tab = fakeTab({ - id: 13, - sendDebugCommand: async (command) => { - throw new Error(`Unexpected CDP command ${command}`); - }, - capturePage: async () => { - actions.push("capture"); - return new Promise(() => {}); - }, - invalidate: () => { - actions.push("invalidate"); - }, - isBackgroundThrottlingAllowed: () => backgroundThrottlingAllowed, - setBackgroundThrottling: (allowed) => { - backgroundThrottlingAllowed = allowed; - actions.push(`background:${allowed}`); - }, - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => - workspaceId === "workspace-a" ? "a" : null, - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const resultPromise = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-screenshot-no-frame", - workspaceId: "workspace-a", - command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - await vi.advanceTimersByTimeAsync(5_000); - - await expect(resultPromise).resolves.toEqual({ - requestId: "r-screenshot-no-frame", - ok: false, - error: { - code: "screenshot_no_frame", - message: - "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, - }, - }); - expect(backgroundThrottlingAllowed).toBe(false); - expect(actions).toEqual(["background:false", "invalidate", "capture", "background:false"]); - } finally { - vi.useRealTimers(); - } - }); - - it("captures a PNG screenshot from the active browser", async () => { - const tab = fakeTab({ - id: 13, - capturePage: async () => ({ - toPNG: () => new Uint8Array([137, 80, 78, 71, 1, 2, 3]), - getSize: () => ({ width: 640, height: 480 }), - }), - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => (workspaceId === "workspace-a" ? "a" : null), - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const result = await executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-screenshot", - workspaceId: "workspace-a", - command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - expect(result).toEqual({ - requestId: "r-screenshot", - ok: true, - result: { - command: "screenshot", - browserId: "a", - mimeType: "image/png", - dataBase64: "iVBORwECAw==", - width: 640, - height: 480, - }, - }); - }); - - it("returns screenshot_no_frame when capturePage never paints", async () => { - vi.useFakeTimers(); - try { - const tab = fakeTab({ - id: 13, - capturePage: async () => new Promise(() => {}), - }); - const registry = createRegistry({ - getWorkspaceActiveTabContents: (workspaceId) => - workspaceId === "workspace-a" ? tab : null, - getWorkspaceActiveBrowserId: (workspaceId) => - workspaceId === "workspace-a" ? "a" : null, - getBrowserWorkspaceId: (id) => (id === "a" ? "workspace-a" : null), - }); - - const resultPromise = executeAutomationCommand( - { - type: "browser.automation.execute.request", - requestId: "r-screenshot-no-frame", - workspaceId: "workspace-a", - command: { command: "screenshot", args: { workspaceId: "workspace-a" } }, - }, - registry, - ); - - await vi.advanceTimersByTimeAsync(5_000); - - await expect(resultPromise).resolves.toEqual({ - requestId: "r-screenshot-no-frame", - ok: false, - error: { - code: "screenshot_no_frame", - message: - "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, - }, - }); - } finally { - vi.useRealTimers(); - } + expect(result).toEqual({ + requestId: "req-screenshot", + ok: true, + result: { + command: "screenshot", + browserId: BROWSER_A, + mimeType: "image/png", + dataBase64: "iVBORw==", + width: 10, + height: 5, + }, }); + expect(tab.capturedViewports).toEqual([{ stayHidden: false }]); }); }); diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index b5f660633..8e64dae5c 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -54,9 +54,7 @@ export interface BrowserRegistry { listRegisteredBrowserIdsForWorkspace(workspaceId: string): string[]; getTabContents(browserId: string): TabContents | null; getBrowserWorkspaceId(browserId: string): string | null; - getWorkspaceActiveTabContents(workspaceId: string): TabContents | null; getWorkspaceActiveBrowserId(workspaceId: string): string | null; - getAgentActiveBrowserId(agentId: string): string | null; } export type AutomationCommandPayload = BrowserAutomationExecuteResponse["payload"]; @@ -167,45 +165,18 @@ function tabInfoFromContents( } export function executeAutomationCommand( - rawRequest: BrowserAutomationExecuteRequest, + request: BrowserAutomationExecuteRequest, registry: BrowserRegistry, options?: { snapshotEngine?: BrowserSnapshotEngine }, ): AutomationCommandPayload | Promise { - const request = resolveAgentBrowserTarget(rawRequest, registry); const { requestId, command } = request; - const workspaceId = request.workspaceId ?? command.args.workspaceId; + const workspaceId = request.workspaceId; const snapshotEngine = options?.snapshotEngine ?? defaultSnapshotEngine; const handler = commandHandlers[command.command]; return handler({ request, command, requestId, workspaceId, registry, snapshotEngine }); } -function resolveAgentBrowserTarget( - request: BrowserAutomationExecuteRequest, - registry: BrowserRegistry, -): BrowserAutomationExecuteRequest { - if (request.browserId || readCommandBrowserId(request.command)) { - return request; - } - if (!request.agentId) { - return request; - } - - const agentBrowserId = registry.getAgentActiveBrowserId(request.agentId); - if (!agentBrowserId) { - return request; - } - - return { ...request, browserId: agentBrowserId }; -} - -function readCommandBrowserId(command: BrowserAutomationCommand): string | undefined { - const args = command.args as { browserId?: unknown }; - return typeof args.browserId === "string" && args.browserId.length > 0 - ? args.browserId - : undefined; -} - interface CommandHandlerContext { request: BrowserAutomationExecuteRequest; command: BrowserAutomationCommand; @@ -224,54 +195,49 @@ const commandHandlers: Record fail(requestId, "browser_unsupported", "browser_new_tab is handled by the app runtime."), - page_info: ({ request, command, requestId, workspaceId, registry }) => { + page_info: ({ command, requestId, workspaceId, registry }) => { const pageInfoCommand = command as Extract; - return executePageInfo( - requestId, - workspaceId, - pageInfoCommand.args.browserId ?? request.browserId, - registry, - ); + return executePageInfo(requestId, workspaceId, pageInfoCommand.args.browserId, registry); }, - snapshot: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + snapshot: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const snapshotCommand = command as Extract; return executeSnapshot( requestId, workspaceId, - snapshotCommand.args.browserId ?? request.browserId, + snapshotCommand.args.browserId, registry, snapshotEngine, ); }, - click: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + click: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const clickCommand = command as Extract; return executeClick( requestId, workspaceId, - clickCommand.args.browserId ?? request.browserId, + clickCommand.args.browserId, clickCommand.args.ref, registry, snapshotEngine, ); }, - fill: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + fill: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const fillCommand = command as Extract; return executeFill( requestId, workspaceId, - fillCommand.args.browserId ?? request.browserId, + fillCommand.args.browserId, fillCommand.args.ref, fillCommand.args.value, registry, snapshotEngine, ); }, - wait: ({ request, command, requestId, workspaceId, registry }) => { + wait: ({ command, requestId, workspaceId, registry }) => { const waitCommand = command as Extract; return executeWait( requestId, workspaceId, - waitCommand.args.browserId ?? request.browserId, + waitCommand.args.browserId, { text: waitCommand.args.text, url: waitCommand.args.url, @@ -280,83 +246,78 @@ const commandHandlers: Record { + type: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const typeCommand = command as Extract; return executeType( requestId, workspaceId, - typeCommand.args.browserId ?? request.browserId, + typeCommand.args.browserId, typeCommand.args.ref, typeCommand.args.text, registry, snapshotEngine, ); }, - keypress: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + keypress: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const keypressCommand = command as Extract; return executeKeypress( requestId, workspaceId, - keypressCommand.args.browserId ?? request.browserId, + keypressCommand.args.browserId, keypressCommand.args.ref, keypressCommand.args.key, registry, snapshotEngine, ); }, - navigate: ({ request, command, requestId, workspaceId, registry }) => { + navigate: ({ command, requestId, workspaceId, registry }) => { const navigateCommand = command as Extract; return executeNavigate( requestId, workspaceId, - navigateCommand.args.browserId ?? request.browserId, + navigateCommand.args.browserId, navigateCommand.args.url, registry, ); }, - back: ({ request, command, requestId, workspaceId, registry }) => { + back: ({ command, requestId, workspaceId, registry }) => { const backCommand = command as Extract; return executeNavigationAction( requestId, workspaceId, - backCommand.args.browserId ?? request.browserId, + backCommand.args.browserId, "back", registry, ); }, - forward: ({ request, command, requestId, workspaceId, registry }) => { + forward: ({ command, requestId, workspaceId, registry }) => { const forwardCommand = command as Extract; return executeNavigationAction( requestId, workspaceId, - forwardCommand.args.browserId ?? request.browserId, + forwardCommand.args.browserId, "forward", registry, ); }, - reload: ({ request, command, requestId, workspaceId, registry }) => { + reload: ({ command, requestId, workspaceId, registry }) => { const reloadCommand = command as Extract; return executeNavigationAction( requestId, workspaceId, - reloadCommand.args.browserId ?? request.browserId, + reloadCommand.args.browserId, "reload", registry, ); }, - screenshot: ({ request, command, requestId, workspaceId, registry }) => { + screenshot: ({ command, requestId, workspaceId, registry }) => { const screenshotCommand = command as Extract< BrowserAutomationCommand, { command: "screenshot" } >; - return executeScreenshot( - requestId, - workspaceId, - screenshotCommand.args.browserId ?? request.browserId, - registry, - ); + return executeScreenshot(requestId, workspaceId, screenshotCommand.args.browserId, registry); }, - full_page_screenshot: ({ request, command, requestId, workspaceId, registry }) => { + full_page_screenshot: ({ command, requestId, workspaceId, registry }) => { const screenshotCommand = command as Extract< BrowserAutomationCommand, { command: "full_page_screenshot" } @@ -364,26 +325,26 @@ const commandHandlers: Record { + pdf: ({ command, requestId, workspaceId, registry }) => { const pdfCommand = command as Extract; return executePdf( requestId, workspaceId, - pdfCommand.args.browserId ?? request.browserId, + pdfCommand.args.browserId, { landscape: pdfCommand.args.landscape, printBackground: pdfCommand.args.printBackground }, registry, ); }, - download: ({ request, command, requestId, workspaceId, registry }) => { + download: ({ command, requestId, workspaceId, registry }) => { const downloadCommand = command as Extract; return executeDownload( requestId, workspaceId, - downloadCommand.args.browserId ?? request.browserId, + downloadCommand.args.browserId, { url: downloadCommand.args.url, fileName: downloadCommand.args.fileName }, registry, ); @@ -394,101 +355,96 @@ const commandHandlers: Record { + focus: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const focusCommand = command as Extract; return executeFocus( requestId, workspaceId, - focusCommand.args.browserId ?? request.browserId, + focusCommand.args.browserId, focusCommand.args.ref, registry, snapshotEngine, ); }, - clear: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + clear: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const clearCommand = command as Extract; return executeClear( requestId, workspaceId, - clearCommand.args.browserId ?? request.browserId, + clearCommand.args.browserId, clearCommand.args.ref, registry, snapshotEngine, ); }, - check: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + check: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const checkCommand = command as Extract; return executeCheck( requestId, workspaceId, - checkCommand.args.browserId ?? request.browserId, + checkCommand.args.browserId, checkCommand.args.ref, checkCommand.args.checked, registry, snapshotEngine, ); }, - select: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + select: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const selectCommand = command as Extract; return executeSelect( requestId, workspaceId, - selectCommand.args.browserId ?? request.browserId, + selectCommand.args.browserId, selectCommand.args.ref, selectCommand.args.value, registry, snapshotEngine, ); }, - hover: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + hover: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const hoverCommand = command as Extract; return executeHover( requestId, workspaceId, - hoverCommand.args.browserId ?? request.browserId, + hoverCommand.args.browserId, hoverCommand.args.ref, registry, snapshotEngine, ); }, - drag: ({ request, command, requestId, workspaceId, registry, snapshotEngine }) => { + drag: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const dragCommand = command as Extract; return executeDrag( requestId, workspaceId, - dragCommand.args.browserId ?? request.browserId, + dragCommand.args.browserId, dragCommand.args.sourceRef, dragCommand.args.targetRef, registry, snapshotEngine, ); }, - logs: ({ request, command, requestId, workspaceId, registry }) => { + logs: ({ command, requestId, workspaceId, registry }) => { const logsCommand = command as Extract; return executeLogs( requestId, workspaceId, - logsCommand.args.browserId ?? request.browserId, + logsCommand.args.browserId, logsCommand.args.maxEntries, registry, ); }, - storage: ({ request, command, requestId, workspaceId, registry }) => { + storage: ({ command, requestId, workspaceId, registry }) => { const storageCommand = command as Extract; - return executeStorage( - requestId, - workspaceId, - storageCommand.args.browserId ?? request.browserId, - registry, - ); + return executeStorage(requestId, workspaceId, storageCommand.args.browserId, registry); }, - environment: ({ request, command, requestId, workspaceId, registry }) => { + environment: ({ command, requestId, workspaceId, registry }) => { const environmentCommand = command as Extract< BrowserAutomationCommand, { command: "environment" } @@ -496,7 +452,7 @@ const commandHandlers: Record { + set_background: ({ command, requestId, workspaceId, registry }) => { const setBackgroundCommand = command as Extract< BrowserAutomationCommand, { command: "set_background" } @@ -512,7 +468,7 @@ const commandHandlers: Record { @@ -624,7 +580,7 @@ async function executeSnapshot( async function executeClick( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string, registry: BrowserRegistry, snapshotEngine: BrowserSnapshotEngine, @@ -647,7 +603,7 @@ async function executeClick( async function executeFill( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string, value: string, registry: BrowserRegistry, @@ -672,7 +628,7 @@ async function executeFill( async function executeFocus( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string, registry: BrowserRegistry, snapshotEngine: BrowserSnapshotEngine, @@ -695,7 +651,7 @@ async function executeFocus( async function executeClear( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string, registry: BrowserRegistry, snapshotEngine: BrowserSnapshotEngine, @@ -718,7 +674,7 @@ async function executeClear( async function executeCheck( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string, checked: boolean, registry: BrowserRegistry, @@ -747,7 +703,7 @@ async function executeCheck( async function executeSelect( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string, value: string, registry: BrowserRegistry, @@ -776,7 +732,7 @@ async function executeSelect( async function executeHover( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string, registry: BrowserRegistry, snapshotEngine: BrowserSnapshotEngine, @@ -799,7 +755,7 @@ async function executeHover( async function executeDrag( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, sourceRef: string, targetRef: string, registry: BrowserRegistry, @@ -828,7 +784,7 @@ async function executeDrag( async function executeLogs( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, maxEntries: number, registry: BrowserRegistry, ): Promise { @@ -855,7 +811,7 @@ async function executeLogs( async function executeStorage( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, registry: BrowserRegistry, ): Promise { const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); @@ -882,7 +838,7 @@ async function executeStorage( async function executeEnvironment( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, environment: { viewport?: { width: number; height: number; deviceScaleFactor?: number }; geolocation?: { latitude: number; longitude: number; accuracy?: number }; @@ -941,7 +897,7 @@ function staleRefFailure(requestId: string, ref: string): FailurePayload { async function executeWait( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, condition: { text?: string; url?: string; timeoutMs?: number }, registry: BrowserRegistry, ): Promise { @@ -999,7 +955,7 @@ async function executeWait( async function executeSetBackground( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, color: string, registry: BrowserRegistry, ): Promise { @@ -1032,7 +988,7 @@ function buildSetBackgroundScript(color: string): string { async function executeType( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string | undefined, text: string, registry: BrowserRegistry, @@ -1061,7 +1017,7 @@ async function executeType( async function executeKeypress( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, ref: string | undefined, key: string, registry: BrowserRegistry, @@ -1090,7 +1046,7 @@ async function executeKeypress( async function executeNavigate( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, url: string, registry: BrowserRegistry, ): Promise { @@ -1112,7 +1068,7 @@ async function executeNavigate( function executeNavigationAction( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, action: "back" | "forward" | "reload", registry: BrowserRegistry, ): AutomationCommandPayload { @@ -1135,7 +1091,7 @@ function executeNavigationAction( async function executeScreenshot( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, registry: BrowserRegistry, ): Promise { const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); @@ -1212,7 +1168,7 @@ async function getCdpLayoutMetrics(contents: TabContents): Promise<{ async function executeFullPageScreenshot( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, registry: BrowserRegistry, ): Promise { const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); @@ -1260,7 +1216,7 @@ async function executeFullPageScreenshot( async function executePdf( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, options: { landscape?: boolean; printBackground: boolean }, registry: BrowserRegistry, ): Promise { @@ -1290,7 +1246,7 @@ async function executePdf( async function executeDownload( requestId: string, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, input: { url: string; fileName?: string }, registry: BrowserRegistry, ): Promise { @@ -1331,7 +1287,7 @@ async function executeUpload( requestId: string, cwd: string | undefined, workspaceId: string | undefined, - browserId: string | undefined, + browserId: string, input: { ref: string; filePaths: string[] }, registry: BrowserRegistry, snapshotEngine: BrowserSnapshotEngine, @@ -1573,41 +1529,22 @@ const VIEWPORT_SCRIPT = String.raw`(() => JSON.stringify({ function resolveTabTarget(input: { requestId: string; workspaceId: string | undefined; - browserId: string | undefined; + browserId: string; registry: BrowserRegistry; }): ResolvedTabTarget | FailurePayload { const { requestId, workspaceId, browserId, registry } = input; - let contents: TabContents | null; - let resolvedBrowserId: string; + if (workspaceId && registry.getBrowserWorkspaceId(browserId) !== workspaceId) { + return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`); + } - if (browserId) { - if (workspaceId && registry.getBrowserWorkspaceId(browserId) !== workspaceId) { - return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`); - } - contents = registry.getTabContents(browserId); - resolvedBrowserId = browserId; - if (!contents) { - return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`); - } - } else { - if (!workspaceId) { - return fail(requestId, "browser_no_tab", "No active browser tab in workspace"); - } - contents = registry.getWorkspaceActiveTabContents(workspaceId); - const activeId = registry.getWorkspaceActiveBrowserId(workspaceId); - if (!contents || !activeId) { - return fail(requestId, "browser_no_tab", "No active browser tab in workspace"); - } - resolvedBrowserId = activeId; + const contents = registry.getTabContents(browserId); + if (!contents) { + return fail(requestId, "browser_tab_not_found", `No browser tab found for ID: ${browserId}`); } if (contents.isDestroyed()) { - return fail( - requestId, - "browser_tab_closed", - `Browser tab ${resolvedBrowserId} has been closed`, - ); + return fail(requestId, "browser_tab_closed", `Browser tab ${browserId} has been closed`); } - return { browserId: resolvedBrowserId, contents }; + return { browserId, contents }; } diff --git a/packages/desktop/src/features/browser-webviews/index.ts b/packages/desktop/src/features/browser-webviews/index.ts index 09fc9561b..ed7ad3fdf 100644 --- a/packages/desktop/src/features/browser-webviews/index.ts +++ b/packages/desktop/src/features/browser-webviews/index.ts @@ -75,17 +75,6 @@ export function getWorkspaceActivePaseoBrowserId(workspaceId: string): string | return browserRegistry.getWorkspaceActiveBrowserId(workspaceId); } -export function setAgentActivePaseoBrowserId(input: { - agentId: string; - browserId: string | null; -}): void { - browserRegistry.setAgentActiveBrowser(input); -} - -export function getAgentActivePaseoBrowserId(agentId: string): string | null { - return browserRegistry.getAgentActiveBrowserId(agentId); -} - export function getPaseoBrowserWebContents(browserId: string): WebContents | null { const contentsId = browserRegistry.getWebContentsIdForBrowser(browserId); if (contentsId === null) { @@ -99,16 +88,6 @@ export function getPaseoBrowserWebContents(browserId: string): WebContents | nul return null; } -export function getWorkspaceActivePaseoBrowserWebContents(workspaceId: string): WebContents | null { - const activeBrowserId = getWorkspaceActivePaseoBrowserId(workspaceId); - return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null; -} - -export function getAgentActivePaseoBrowserWebContents(agentId: string): WebContents | null { - const activeBrowserId = getAgentActivePaseoBrowserId(agentId); - return activeBrowserId ? getPaseoBrowserWebContents(activeBrowserId) : null; -} - export function getMostRecentWorkspaceActivePaseoBrowserWebContents(): WebContents | null { const browserId = browserRegistry.getMostRecentWorkspaceActiveBrowserId(); return browserId ? getPaseoBrowserWebContents(browserId) : null; diff --git a/packages/desktop/src/features/browser-webviews/registry.test.ts b/packages/desktop/src/features/browser-webviews/registry.test.ts index 02bd40090..1120feba1 100644 --- a/packages/desktop/src/features/browser-webviews/registry.test.ts +++ b/packages/desktop/src/features/browser-webviews/registry.test.ts @@ -8,7 +8,6 @@ describe("PaseoBrowserWebviewRegistry", () => { registry.registerWebContents({ webContentsId: 1, browserId: "browser-a" }); registry.registerWorkspace({ browserId: "browser-a", workspaceId: "workspace-a" }); registry.setWorkspaceActiveBrowser({ workspaceId: "workspace-a", browserId: "browser-a" }); - registry.setAgentActiveBrowser({ agentId: "agent-a", browserId: "browser-a" }); registry.registerWebContents({ webContentsId: 2, browserId: "browser-a" }); expect(registry.getBrowserIdForWebContents(1)).toBeNull(); @@ -16,7 +15,6 @@ describe("PaseoBrowserWebviewRegistry", () => { expect(registry.getWebContentsIdForBrowser("browser-a")).toBe(2); expect(registry.getWorkspaceId("browser-a")).toBe("workspace-a"); expect(registry.getWorkspaceActiveBrowserId("workspace-a")).toBe("browser-a"); - expect(registry.getAgentActiveBrowserId("agent-a")).toBe("browser-a"); }); it("ignores stale destroy events after a duplicate browserId moved", () => { diff --git a/packages/desktop/src/features/browser-webviews/registry.ts b/packages/desktop/src/features/browser-webviews/registry.ts index a9ccd4966..93f921280 100644 --- a/packages/desktop/src/features/browser-webviews/registry.ts +++ b/packages/desktop/src/features/browser-webviews/registry.ts @@ -8,7 +8,6 @@ export class PaseoBrowserWebviewRegistry { private readonly webContentsIdsByBrowserId = new Map(); private readonly workspaceIdsByBrowserId = new Map(); private readonly activeBrowserIdsByWorkspaceId = new Map(); - private readonly activeBrowserIdsByAgentId = new Map(); public registerWebContents(input: { webContentsId: number; browserId: string }): void { const previousWebContentsId = this.webContentsIdsByBrowserId.get(input.browserId) ?? null; @@ -80,29 +79,11 @@ export class PaseoBrowserWebviewRegistry { return Array.from(this.activeBrowserIdsByWorkspaceId.values()).at(-1) ?? null; } - public setAgentActiveBrowser(input: { agentId: string; browserId: string | null }): void { - if (input.browserId) { - this.activeBrowserIdsByAgentId.delete(input.agentId); - this.activeBrowserIdsByAgentId.set(input.agentId, input.browserId); - return; - } - this.activeBrowserIdsByAgentId.delete(input.agentId); - } - - public getAgentActiveBrowserId(agentId: string): string | null { - return this.activeBrowserIdsByAgentId.get(agentId) ?? null; - } - private deleteActiveBrowserReferences(browserId: string): void { for (const [workspaceId, activeBrowserId] of this.activeBrowserIdsByWorkspaceId) { if (activeBrowserId === browserId) { this.activeBrowserIdsByWorkspaceId.delete(workspaceId); } } - for (const [agentId, activeBrowserId] of this.activeBrowserIdsByAgentId) { - if (activeBrowserId === browserId) { - this.activeBrowserIdsByAgentId.delete(agentId); - } - } } } diff --git a/packages/desktop/src/main.ts b/packages/desktop/src/main.ts index 1109ef564..fcad58da0 100644 --- a/packages/desktop/src/main.ts +++ b/packages/desktop/src/main.ts @@ -56,7 +56,6 @@ import { registerBrowserWebviewNavigationGuards, registerPaseoBrowserWorkspace, registerPaseoBrowserWebContents, - setAgentActivePaseoBrowserId, setWorkspaceActivePaseoBrowserId, } from "./features/browser-webviews/index.js"; import { parseOpenProjectPathFromArgv } from "./open-project-routing.js"; @@ -144,20 +143,6 @@ function readActiveBrowserInput( return { workspaceId: record.workspaceId.trim(), browserId: browserId || null }; } -function readAgentActiveBrowserInput( - input: unknown, -): { agentId: string; browserId: string | null } | null { - if (typeof input !== "object" || input === null || Array.isArray(input)) { - return null; - } - const record = input as Record; - if (typeof record.agentId !== "string" || record.agentId.trim().length === 0) { - return null; - } - const browserId = typeof record.browserId === "string" ? record.browserId.trim() : null; - return { agentId: record.agentId.trim(), browserId: browserId || null }; -} - const pendingBrowserWebviewIds: string[] = []; function isBrowserRefreshInput(input: Electron.Input): boolean { @@ -350,13 +335,6 @@ ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput: } }); -ipcMain.handle("paseo:browser:set-agent-active-browser", (_event, rawInput: unknown) => { - const input = readAgentActiveBrowserInput(rawInput); - if (input) { - setAgentActivePaseoBrowserId(input); - } -}); - ipcMain.handle("paseo:browser:open-devtools", (_event, browserId: unknown) => { if (typeof browserId !== "string" || browserId.trim().length === 0) { const result = { diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index bf63cf5ae..02fe9dc0e 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -78,8 +78,6 @@ contextBridge.exposeInMainWorld("paseoDesktop", { ipcRenderer.invoke("paseo:browser:register-workspace-browser", input), setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) => ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input), - setAgentActiveBrowser: (input: { agentId: string; browserId: string | null }) => - ipcRenderer.invoke("paseo:browser:set-agent-active-browser", input), openDevTools: (browserId: string) => ipcRenderer.invoke("paseo:browser:open-devtools", browserId), clearPartition: (browserId: string) => diff --git a/packages/protocol/src/browser-automation/rpc-schemas.test.ts b/packages/protocol/src/browser-automation/rpc-schemas.test.ts index eba3bad50..5e240c473 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.test.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.test.ts @@ -5,96 +5,215 @@ import { BrowserAutomationExecuteResponseSchema, } from "./rpc-schemas.js"; -describe("browser automation execute RPC schemas", () => { - test("rejects navigate and download commands for non-http URLs", () => { - for (const command of ["navigate", "download"] as const) { - expect(() => - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: `req-${command}`, - command: { - command, - args: { url: "file:///tmp/secret.txt" }, - }, - }), - ).toThrow(); - } - }); +const BROWSER_ID = "11111111-1111-4111-8111-111111111111"; +const FALLBACK_BROWSER_ID = "1777777777777-abcdef"; +const BROWSER_ID_MESSAGE = + "browserId must be a real id returned by browser_new_tab or browser_list_tabs"; +const WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url"; - test("parses list tabs requests with top-level correlation and typed command args", () => { +describe("browser automation execute RPC schemas", () => { + test("list tabs reads workspace from the request envelope", () => { const parsed = BrowserAutomationExecuteRequestSchema.parse({ type: "browser.automation.execute.request", - requestId: "req-1", + requestId: "req-list-tabs", workspaceId: "workspace-1", - command: { - command: "list_tabs", - args: { workspaceId: "workspace-1" }, - }, + command: { command: "list_tabs", args: {} }, }); expect(parsed).toEqual({ type: "browser.automation.execute.request", - requestId: "req-1", + requestId: "req-list-tabs", workspaceId: "workspace-1", - command: { - command: "list_tabs", - args: { workspaceId: "workspace-1" }, - }, + command: { command: "list_tabs", args: {} }, }); }); - test("parses new tab requests and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-new-tab", - workspaceId: "workspace-1", - command: { - command: "new_tab", - args: { workspaceId: "workspace-1", url: "https://example.com" }, - }, - }).command, - ).toEqual({ - command: "new_tab", - args: { workspaceId: "workspace-1", url: "https://example.com" }, + test("new tab reads workspace from the request envelope", () => { + const parsed = BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-new-tab", + workspaceId: "workspace-1", + command: { command: "new_tab", args: { url: "https://example.com" } }, }); - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-new-tab", - ok: true, - result: { - command: "new_tab", - browserId: "browser-1", - workspaceId: "workspace-1", - url: "https://example.com", - }, + expect(parsed.command).toEqual({ + command: "new_tab", + args: { url: "https://example.com" }, + }); + }); + + test("tab commands require a browser id from browser_new_tab or browser_list_tabs", () => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId: "req-snapshot", + workspaceId: "workspace-1", + command: { command: "snapshot", args: {} }, + }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] }, + }); + }); + + test("tab commands reject hallucinated browser ids", () => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId: "req-page-info", + workspaceId: "workspace-1", + command: { command: "page_info", args: { browserId: "default" } }, + }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] }, + }); + }); + + test("tab commands parse browser ids produced by the fallback generator", () => { + const parsed = BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-page-info", + workspaceId: "workspace-1", + command: { command: "page_info", args: { browserId: FALLBACK_BROWSER_ID } }, + }); + + expect(parsed.command).toEqual({ + command: "page_info", + args: { browserId: FALLBACK_BROWSER_ID }, + }); + }); + + test("requests reject browser id in the envelope", () => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId: "req-click", + workspaceId: "workspace-1", + browserId: BROWSER_ID, + command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } }, + }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: 'Unrecognized key: "browserId"' })] }, + }); + }); + + test("tab commands reject workspace id in command args", () => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId: "req-click", + workspaceId: "workspace-1", + command: { + command: "click", + args: { workspaceId: "workspace-1", browserId: BROWSER_ID, ref: "@e1" }, + }, + }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: 'Unrecognized key: "workspaceId"' })] }, + }); + }); + + test("wait rejects calls without exactly one condition", () => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId: "req-wait", + command: { command: "wait", args: { browserId: BROWSER_ID } }, + }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: WAIT_CONDITION_MESSAGE })] }, + }); + }); + + test("wait rejects calls with both text and url conditions", () => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId: "req-wait", + command: { + command: "wait", + args: { browserId: BROWSER_ID, text: "Ready", url: "/ready" }, + }, + }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: WAIT_CONDITION_MESSAGE })] }, + }); + }); + + test("wait accepts one text condition", () => { + const parsed = BrowserAutomationExecuteRequestSchema.parse({ + type: "browser.automation.execute.request", + requestId: "req-wait", + command: { + command: "wait", + args: { browserId: BROWSER_ID, text: "Ready", timeoutMs: 1000 }, + }, + }); + + expect(parsed.command).toEqual({ + command: "wait", + args: { browserId: BROWSER_ID, text: "Ready", timeoutMs: 1000 }, + }); + }); + + test("navigate rejects non-http URLs at the protocol boundary", () => { + const parsed = BrowserAutomationExecuteRequestSchema.safeParse({ + type: "browser.automation.execute.request", + requestId: "req-navigate", + command: { + command: "navigate", + args: { browserId: BROWSER_ID, url: "file:///tmp/secret.txt" }, + }, + }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: "URL must use http or https" })] }, + }); + }); + + test("new tab responses declare the generated browser id shape", () => { + const parsed = BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-new-tab", + ok: true, + result: { + command: "new_tab", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", }, - }).payload, - ).toEqual({ + }, + }); + + expect(parsed.payload).toEqual({ requestId: "req-new-tab", ok: true, result: { command: "new_tab", - browserId: "browser-1", + browserId: BROWSER_ID, workspaceId: "workspace-1", url: "https://example.com", }, }); }); - test("parses page info responses with result data under payload", () => { - const parsed = BrowserAutomationExecuteResponseSchema.parse({ + test("responses reject hallucinated browser ids", () => { + const parsed = BrowserAutomationExecuteResponseSchema.safeParse({ type: "browser.automation.execute.response", payload: { - requestId: "req-1", + requestId: "req-page-info", ok: true, result: { command: "page_info", tab: { - browserId: "browser-1", + browserId: "default", workspaceId: "workspace-1", url: "https://example.com", title: "Example", @@ -103,672 +222,9 @@ describe("browser automation execute RPC schemas", () => { }, }); - expect(parsed.payload).toEqual({ - requestId: "req-1", - ok: true, - result: { - command: "page_info", - tab: { - browserId: "browser-1", - workspaceId: "workspace-1", - url: "https://example.com", - title: "Example", - isActive: false, - isLoading: false, - }, - }, - }); - }); - - test("parses snapshot requests and ref-bearing snapshot responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-snapshot", - workspaceId: "workspace-1", - command: { - command: "snapshot", - args: { workspaceId: "workspace-1", browserId: "browser-1" }, - }, - }), - ).toEqual({ - type: "browser.automation.execute.request", - requestId: "req-snapshot", - workspaceId: "workspace-1", - command: { - command: "snapshot", - args: { workspaceId: "workspace-1", browserId: "browser-1" }, - }, - }); - - const parsed = BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-snapshot", - ok: true, - result: { - command: "snapshot", - browserId: "browser-1", - workspaceId: "workspace-1", - url: "https://example.com/form", - title: "Fixture", - elements: [ - { - ref: "@e1", - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, - }, - ], - }, - }, - }); - - expect(parsed.payload).toEqual({ - requestId: "req-snapshot", - ok: true, - result: { - command: "snapshot", - browserId: "browser-1", - workspaceId: "workspace-1", - url: "https://example.com/form", - title: "Fixture", - elements: [ - { - ref: "@e1", - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, - }, - ], - }, - }); - }); - - test("parses click and fill ref commands", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-click", - command: { - command: "click", - args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" }, - }, - }).command, - ).toEqual({ - command: "click", - args: { workspaceId: "workspace-1", browserId: "browser-1", ref: "@e1" }, - }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-fill", - command: { - command: "fill", - args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" }, - }, - }).command, - ).toEqual({ - command: "fill", - args: { workspaceId: "workspace-1", ref: "@e2", value: "Ada" }, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-fill", - ok: true, - result: { command: "fill", browserId: "browser-1", ref: "@e2" }, - }, - }).payload, - ).toEqual({ - requestId: "req-fill", - ok: true, - result: { command: "fill", browserId: "browser-1", ref: "@e2" }, - }); - }); - - test("parses wait text commands and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-wait", - command: { - command: "wait", - args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 }, - }, - }).command, - ).toEqual({ - command: "wait", - args: { workspaceId: "workspace-1", text: "Ready", timeoutMs: 1000 }, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-wait", - ok: true, - result: { command: "wait", browserId: "browser-1", matched: "text" }, - }, - }).payload, - ).toEqual({ - requestId: "req-wait", - ok: true, - result: { command: "wait", browserId: "browser-1", matched: "text" }, - }); - }); - - test("parses set background commands and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-bg", - command: { - command: "set_background", - args: { workspaceId: "workspace-1", color: "red" }, - }, - }).command, - ).toEqual({ - command: "set_background", - args: { workspaceId: "workspace-1", color: "red" }, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-bg", - ok: true, - result: { command: "set_background", browserId: "browser-1", color: "red" }, - }, - }).payload, - ).toEqual({ - requestId: "req-bg", - ok: true, - result: { command: "set_background", browserId: "browser-1", color: "red" }, - }); - }); - - test("parses type and keypress commands", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-type", - command: { command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } }, - }).command, - ).toEqual({ command: "type", args: { browserId: "browser-1", ref: "@e1", text: "Ada" } }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-keypress", - command: { command: "keypress", args: { browserId: "browser-1", key: "Enter" } }, - }).command, - ).toEqual({ command: "keypress", args: { browserId: "browser-1", key: "Enter" } }); - }); - - test("parses navigation commands", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-nav", - command: { - command: "navigate", - args: { browserId: "browser-1", url: "https://example.com/next" }, - }, - }).command, - ).toEqual({ - command: "navigate", - args: { browserId: "browser-1", url: "https://example.com/next" }, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-nav", - ok: true, - result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" }, - }, - }).payload, - ).toEqual({ - requestId: "req-nav", - ok: true, - result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" }, - }); - }); - - test("parses screenshot responses", () => { - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-shot", - ok: true, - result: { - command: "screenshot", - browserId: "browser-1", - mimeType: "image/png", - dataBase64: "iVBORw0KGgo=", - width: 100, - height: 50, - }, - }, - }).payload, - ).toEqual({ - requestId: "req-shot", - ok: true, - result: { - command: "screenshot", - browserId: "browser-1", - mimeType: "image/png", - dataBase64: "iVBORw0KGgo=", - width: 100, - height: 50, - }, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-shot-no-frame", - ok: false, - error: { - code: "screenshot_no_frame", - message: - "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, - }, - }, - }).payload, - ).toEqual({ - requestId: "req-shot-no-frame", - ok: false, - error: { - code: "screenshot_no_frame", - message: "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, - }, - }); - }); - - test("parses form control commands and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-focus", - command: { command: "focus", args: { browserId: "browser-1", ref: "@e1" } }, - }).command, - ).toEqual({ command: "focus", args: { browserId: "browser-1", ref: "@e1" } }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-clear", - command: { command: "clear", args: { browserId: "browser-1", ref: "@e1" } }, - }).command, - ).toEqual({ command: "clear", args: { browserId: "browser-1", ref: "@e1" } }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-check", - command: { command: "check", args: { browserId: "browser-1", ref: "@e2" } }, - }).command, - ).toEqual({ command: "check", args: { browserId: "browser-1", ref: "@e2", checked: true } }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-select", - command: { command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } }, - }).command, - ).toEqual({ command: "select", args: { browserId: "browser-1", ref: "@e3", value: "us" } }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-hover", - command: { command: "hover", args: { browserId: "browser-1", ref: "@e4" } }, - }).command, - ).toEqual({ command: "hover", args: { browserId: "browser-1", ref: "@e4" } }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-drag", - command: { - command: "drag", - args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" }, - }, - }).command, - ).toEqual({ - command: "drag", - args: { browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" }, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-select", - ok: true, - result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" }, - }, - }).payload, - ).toEqual({ - requestId: "req-select", - ok: true, - result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" }, - }); - }); - - test("parses browser log commands and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-logs", - command: { command: "logs", args: { browserId: "browser-1" } }, - }).command, - ).toEqual({ command: "logs", args: { browserId: "browser-1", maxEntries: 50 } }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-logs", - ok: true, - result: { - command: "logs", - browserId: "browser-1", - console: [{ level: "info", message: "ready", timestamp: 10 }], - network: [ - { - url: "https://example.com/app.js", - type: "script", - startTime: 1, - duration: 2, - }, - ], - }, - }, - }).payload, - ).toEqual({ - requestId: "req-logs", - ok: true, - result: { - command: "logs", - browserId: "browser-1", - console: [{ level: "info", message: "ready", timestamp: 10 }], - network: [ - { - url: "https://example.com/app.js", - type: "script", - startTime: 1, - duration: 2, - }, - ], - }, - }); - }); - - test("parses browser storage commands and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-storage", - command: { command: "storage", args: { browserId: "browser-1" } }, - }).command, - ).toEqual({ command: "storage", args: { browserId: "browser-1" } }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-storage", - ok: true, - result: { - command: "storage", - browserId: "browser-1", - url: "https://example.com", - cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }, - }, - }).payload, - ).toEqual({ - requestId: "req-storage", - ok: true, - result: { - command: "storage", - browserId: "browser-1", - url: "https://example.com", - cookies: [{ name: "theme", value: "dark", domain: "example.com", httpOnly: true }], - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }, - }); - }); - - test("parses browser environment commands and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-environment", - command: { - command: "environment", - args: { - browserId: "browser-1", - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - }).command, - ).toEqual({ - command: "environment", - args: { - browserId: "browser-1", - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-environment", - ok: true, - result: { - command: "environment", - browserId: "browser-1", - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - }).payload, - ).toEqual({ - requestId: "req-environment", - ok: true, - result: { - command: "environment", - browserId: "browser-1", - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }); - }); - - test("parses browser full-page screenshot and PDF commands and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-full-page", - command: { command: "full_page_screenshot", args: { browserId: "browser-1" } }, - }).command, - ).toEqual({ command: "full_page_screenshot", args: { browserId: "browser-1" } }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-pdf", - command: { command: "pdf", args: { browserId: "browser-1" } }, - }).command, - ).toEqual({ command: "pdf", args: { browserId: "browser-1", printBackground: true } }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-full-page", - ok: true, - result: { - command: "full_page_screenshot", - browserId: "browser-1", - mimeType: "image/png", - dataBase64: "iVBORw0KGgo=", - width: 390, - height: 1200, - }, - }, - }).payload.result, - ).toEqual({ - command: "full_page_screenshot", - browserId: "browser-1", - mimeType: "image/png", - dataBase64: "iVBORw0KGgo=", - width: 390, - height: 1200, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-pdf", - ok: true, - result: { - command: "pdf", - browserId: "browser-1", - mimeType: "application/pdf", - dataBase64: "JVBERi0xLjQ=", - }, - }, - }).payload.result, - ).toEqual({ - command: "pdf", - browserId: "browser-1", - mimeType: "application/pdf", - dataBase64: "JVBERi0xLjQ=", - }); - }); - - test("parses browser download and upload commands and responses", () => { - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-download", - command: { - command: "download", - args: { browserId: "browser-1", url: "https://example.com/file.txt" }, - }, - }).command, - ).toEqual({ - command: "download", - args: { browserId: "browser-1", url: "https://example.com/file.txt" }, - }); - - expect( - BrowserAutomationExecuteRequestSchema.parse({ - type: "browser.automation.execute.request", - requestId: "req-upload", - command: { - command: "upload", - args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] }, - }, - }).command, - ).toEqual({ - command: "upload", - args: { browserId: "browser-1", ref: "@e1", filePaths: ["/tmp/file.txt"] }, - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-download", - ok: true, - result: { - command: "download", - browserId: "browser-1", - url: "https://example.com/file.txt", - filePath: "/tmp/file.txt", - totalBytes: 5, - state: "completed", - }, - }, - }).payload.result, - ).toEqual({ - command: "download", - browserId: "browser-1", - url: "https://example.com/file.txt", - filePath: "/tmp/file.txt", - totalBytes: 5, - state: "completed", - }); - - expect( - BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-upload", - ok: true, - result: { - command: "upload", - browserId: "browser-1", - ref: "@e1", - filePaths: ["/tmp/file.txt"], - }, - }, - }).payload.result, - ).toEqual({ - command: "upload", - browserId: "browser-1", - ref: "@e1", - filePaths: ["/tmp/file.txt"], - }); - }); - - test("parses stable model-actionable error responses", () => { - const parsed = BrowserAutomationExecuteResponseSchema.parse({ - type: "browser.automation.execute.response", - payload: { - requestId: "req-1", - ok: false, - error: { - code: "browser_no_desktop", - message: "No desktop browser automation client is connected.", - }, - }, - }); - - expect(parsed.payload).toEqual({ - requestId: "req-1", - ok: false, - error: { - code: "browser_no_desktop", - message: "No desktop browser automation client is connected.", - retryable: false, - }, + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] }, }); }); }); diff --git a/packages/protocol/src/browser-automation/rpc-schemas.ts b/packages/protocol/src/browser-automation/rpc-schemas.ts index 62125515a..b47b70d10 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.ts @@ -3,7 +3,6 @@ import { z } from "zod"; export const BrowserAutomationErrorCodeSchema = z.enum([ "browser_disabled", "browser_no_desktop", - "browser_no_tab", "browser_tab_not_found", "browser_tab_closed", "browser_timeout", @@ -14,10 +13,23 @@ export const BrowserAutomationErrorCodeSchema = z.enum([ "browser_unknown_error", ]); -const BrowserAutomationTabTargetSchema = z.object({ - workspaceId: z.string().min(1).optional(), - browserId: z.string().min(1).optional(), -}); +const BROWSER_AUTOMATION_BROWSER_ID_PATTERN = + /^(?:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|\d{13,}-[0-9a-f]+)$/i; +const BROWSER_AUTOMATION_BROWSER_ID_MESSAGE = + "browserId must be a real id returned by browser_new_tab or browser_list_tabs"; +const BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE = + "browser_wait requires exactly one of text or url"; + +export const BrowserAutomationBrowserIdSchema = z + .string({ error: () => BROWSER_AUTOMATION_BROWSER_ID_MESSAGE }) + .min(1, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE) + .regex(BROWSER_AUTOMATION_BROWSER_ID_PATTERN, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE); + +const BrowserAutomationTabTargetSchema = z + .object({ + browserId: BrowserAutomationBrowserIdSchema, + }) + .strict(); const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/); const BrowserAutomationHttpUrlSchema = z @@ -30,31 +42,27 @@ const BrowserAutomationHttpUrlSchema = z export const BrowserAutomationListTabsCommandSchema = z.object({ command: z.literal("list_tabs"), - args: z - .object({ - workspaceId: z.string().min(1).optional(), - }) - .default({}), + args: z.object({}).strict().default({}), }); export const BrowserAutomationNewTabCommandSchema = z.object({ command: z.literal("new_tab"), args: z .object({ - workspaceId: z.string().min(1).optional(), url: BrowserAutomationHttpUrlSchema.optional(), }) + .strict() .default({}), }); export const BrowserAutomationPageInfoCommandSchema = z.object({ command: z.literal("page_info"), - args: BrowserAutomationTabTargetSchema.default({}), + args: BrowserAutomationTabTargetSchema, }); export const BrowserAutomationSnapshotCommandSchema = z.object({ command: z.literal("snapshot"), - args: BrowserAutomationTabTargetSchema.default({}), + args: BrowserAutomationTabTargetSchema, }); export const BrowserAutomationClickCommandSchema = z.object({ @@ -78,6 +86,8 @@ export const BrowserAutomationWaitCommandSchema = z.object({ text: z.string().min(1).optional(), url: z.string().min(1).optional(), timeoutMs: z.number().int().positive().max(30_000).optional(), + }).refine((args) => Number(Boolean(args.text)) + Number(Boolean(args.url)) === 1, { + message: BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE, }), }); @@ -106,27 +116,27 @@ export const BrowserAutomationNavigateCommandSchema = z.object({ export const BrowserAutomationBackCommandSchema = z.object({ command: z.literal("back"), - args: BrowserAutomationTabTargetSchema.default({}), + args: BrowserAutomationTabTargetSchema, }); export const BrowserAutomationForwardCommandSchema = z.object({ command: z.literal("forward"), - args: BrowserAutomationTabTargetSchema.default({}), + args: BrowserAutomationTabTargetSchema, }); export const BrowserAutomationReloadCommandSchema = z.object({ command: z.literal("reload"), - args: BrowserAutomationTabTargetSchema.default({}), + args: BrowserAutomationTabTargetSchema, }); export const BrowserAutomationScreenshotCommandSchema = z.object({ command: z.literal("screenshot"), - args: BrowserAutomationTabTargetSchema.default({}), + args: BrowserAutomationTabTargetSchema, }); export const BrowserAutomationFullPageScreenshotCommandSchema = z.object({ command: z.literal("full_page_screenshot"), - args: BrowserAutomationTabTargetSchema.default({}), + args: BrowserAutomationTabTargetSchema, }); export const BrowserAutomationPdfCommandSchema = z.object({ @@ -134,7 +144,7 @@ export const BrowserAutomationPdfCommandSchema = z.object({ args: BrowserAutomationTabTargetSchema.extend({ landscape: z.boolean().optional(), printBackground: z.boolean().default(true), - }).default({ printBackground: true }), + }), }); export const BrowserAutomationDownloadCommandSchema = z.object({ @@ -202,12 +212,12 @@ export const BrowserAutomationLogsCommandSchema = z.object({ command: z.literal("logs"), args: BrowserAutomationTabTargetSchema.extend({ maxEntries: z.number().int().positive().max(200).default(50), - }).default({ maxEntries: 50 }), + }), }); export const BrowserAutomationStorageCommandSchema = z.object({ command: z.literal("storage"), - args: BrowserAutomationTabTargetSchema.default({}), + args: BrowserAutomationTabTargetSchema, }); const BrowserAutomationViewportInputSchema = z.object({ @@ -227,7 +237,7 @@ export const BrowserAutomationEnvironmentCommandSchema = z.object({ args: BrowserAutomationTabTargetSchema.extend({ viewport: BrowserAutomationViewportInputSchema.optional(), geolocation: BrowserAutomationGeolocationInputSchema.optional(), - }).default({}), + }), }); export const BrowserAutomationSetBackgroundCommandSchema = z.object({ @@ -269,7 +279,7 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [ ]); export const BrowserAutomationTabInfoSchema = z.object({ - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, workspaceId: z.string().min(1).optional(), url: z.string(), title: z.string(), @@ -286,7 +296,7 @@ export const BrowserAutomationListTabsResultSchema = z.object({ export const BrowserAutomationNewTabResultSchema = z.object({ command: z.literal("new_tab"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, workspaceId: z.string().min(1), url: z.string().min(1), }); @@ -307,7 +317,7 @@ export const BrowserAutomationSnapshotElementSchema = z.object({ export const BrowserAutomationSnapshotResultSchema = z.object({ command: z.literal("snapshot"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, workspaceId: z.string().min(1).optional(), url: z.string(), title: z.string(), @@ -316,59 +326,59 @@ export const BrowserAutomationSnapshotResultSchema = z.object({ export const BrowserAutomationClickResultSchema = z.object({ command: z.literal("click"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, }); export const BrowserAutomationFillResultSchema = z.object({ command: z.literal("fill"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, }); export const BrowserAutomationWaitResultSchema = z.object({ command: z.literal("wait"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, matched: z.enum(["text", "url"]), }); export const BrowserAutomationTypeResultSchema = z.object({ command: z.literal("type"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema.optional(), }); export const BrowserAutomationKeypressResultSchema = z.object({ command: z.literal("keypress"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, key: z.string().min(1), ref: BrowserAutomationRefSchema.optional(), }); export const BrowserAutomationNavigateResultSchema = z.object({ command: z.literal("navigate"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, url: z.string().min(1), }); export const BrowserAutomationBackResultSchema = z.object({ command: z.literal("back"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, }); export const BrowserAutomationForwardResultSchema = z.object({ command: z.literal("forward"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, }); export const BrowserAutomationReloadResultSchema = z.object({ command: z.literal("reload"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, }); export const BrowserAutomationScreenshotResultSchema = z.object({ command: z.literal("screenshot"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, mimeType: z.literal("image/png"), dataBase64: z.string().min(1), width: z.number().int().nonnegative(), @@ -377,7 +387,7 @@ export const BrowserAutomationScreenshotResultSchema = z.object({ export const BrowserAutomationFullPageScreenshotResultSchema = z.object({ command: z.literal("full_page_screenshot"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, mimeType: z.literal("image/png"), dataBase64: z.string().min(1), width: z.number().int().nonnegative(), @@ -386,14 +396,14 @@ export const BrowserAutomationFullPageScreenshotResultSchema = z.object({ export const BrowserAutomationPdfResultSchema = z.object({ command: z.literal("pdf"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, mimeType: z.literal("application/pdf"), dataBase64: z.string().min(1), }); export const BrowserAutomationDownloadResultSchema = z.object({ command: z.literal("download"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, url: z.string().min(1), filePath: z.string().min(1), totalBytes: z.number().int().nonnegative().optional(), @@ -402,46 +412,46 @@ export const BrowserAutomationDownloadResultSchema = z.object({ export const BrowserAutomationUploadResultSchema = z.object({ command: z.literal("upload"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, filePaths: z.array(z.string().min(1)).min(1), }); export const BrowserAutomationFocusResultSchema = z.object({ command: z.literal("focus"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, }); export const BrowserAutomationClearResultSchema = z.object({ command: z.literal("clear"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, }); export const BrowserAutomationCheckResultSchema = z.object({ command: z.literal("check"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, checked: z.boolean(), }); export const BrowserAutomationSelectResultSchema = z.object({ command: z.literal("select"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, value: z.string(), }); export const BrowserAutomationHoverResultSchema = z.object({ command: z.literal("hover"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, }); export const BrowserAutomationDragResultSchema = z.object({ command: z.literal("drag"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, sourceRef: BrowserAutomationRefSchema, targetRef: BrowserAutomationRefSchema, }); @@ -466,7 +476,7 @@ export const BrowserAutomationNetworkLogEntrySchema = z.object({ export const BrowserAutomationLogsResultSchema = z.object({ command: z.literal("logs"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, console: z.array(BrowserAutomationConsoleLogEntrySchema), network: z.array(BrowserAutomationNetworkLogEntrySchema), }); @@ -488,7 +498,7 @@ export const BrowserAutomationStorageEntrySchema = z.object({ export const BrowserAutomationStorageResultSchema = z.object({ command: z.literal("storage"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, url: z.string(), cookies: z.array(BrowserAutomationCookieEntrySchema), localStorage: z.array(BrowserAutomationStorageEntrySchema), @@ -509,14 +519,14 @@ export const BrowserAutomationGeolocationResultSchema = z.object({ export const BrowserAutomationEnvironmentResultSchema = z.object({ command: z.literal("environment"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, viewport: BrowserAutomationViewportResultSchema, geolocation: BrowserAutomationGeolocationResultSchema.optional(), }); export const BrowserAutomationSetBackgroundResultSchema = z.object({ command: z.literal("set_background"), - browserId: z.string().min(1), + browserId: BrowserAutomationBrowserIdSchema, color: z.string().min(1), }); @@ -557,15 +567,16 @@ export const BrowserAutomationErrorSchema = z.object({ retryable: z.boolean().default(false), }); -export const BrowserAutomationExecuteRequestSchema = z.object({ - type: z.literal("browser.automation.execute.request"), - requestId: z.string().min(1), - agentId: z.string().min(1).optional(), - cwd: z.string().min(1).optional(), - workspaceId: z.string().min(1).optional(), - browserId: z.string().min(1).optional(), - command: BrowserAutomationCommandSchema, -}); +export const BrowserAutomationExecuteRequestSchema = z + .object({ + type: z.literal("browser.automation.execute.request"), + requestId: z.string().min(1), + agentId: z.string().min(1).optional(), + cwd: z.string().min(1).optional(), + workspaceId: z.string().min(1).optional(), + command: BrowserAutomationCommandSchema, + }) + .strict(); export const BrowserAutomationExecuteResponseSchema = z.object({ type: z.literal("browser.automation.execute.response"), diff --git a/packages/protocol/src/messages.browser-automation.test.ts b/packages/protocol/src/messages.browser-automation.test.ts index 891565a0d..d864f8f0a 100644 --- a/packages/protocol/src/messages.browser-automation.test.ts +++ b/packages/protocol/src/messages.browser-automation.test.ts @@ -10,6 +10,8 @@ import { } from "./messages.js"; describe("browser automation protocol integration", () => { + const browserId = "11111111-1111-4111-8111-111111111111"; + test("desktop automation capability parses in hello without narrowing old clients", () => { expect( WSHelloMessageSchema.parse({ @@ -39,7 +41,7 @@ describe("browser automation protocol integration", () => { const parsed = SessionOutboundMessageSchema.parse({ type: "browser.automation.execute.request", requestId: "req-1", - command: { command: "page_info", args: { browserId: "browser-1" } }, + command: { command: "page_info", args: { browserId } }, }); expect(parsed.type).toBe("browser.automation.execute.request"); diff --git a/packages/server/src/server/agent/mcp-server.test.ts b/packages/server/src/server/agent/mcp-server.test.ts index 9ed7f8e2e..b3578f46a 100644 --- a/packages/server/src/server/agent/mcp-server.test.ts +++ b/packages/server/src/server/agent/mcp-server.test.ts @@ -609,8 +609,7 @@ describe("browser MCP tools", () => { expect(execute).toHaveBeenCalledWith({ agentId: "agent-1", cwd: REPO_CWD, - workspaceId: REPO_CWD, - command: { command: "list_tabs", args: { workspaceId: REPO_CWD } }, + command: { command: "list_tabs", args: {} }, }); expect(response.structuredContent).toEqual({ ok: false, @@ -619,7 +618,7 @@ describe("browser MCP tools", () => { message: "Browser tools are disabled.", retryable: false, }, - context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD }, + context: { agentId: "agent-1", cwd: REPO_CWD }, }); }); @@ -646,19 +645,18 @@ describe("browser MCP tools", () => { expect(execute).toHaveBeenCalledWith({ agentId: "agent-1", cwd: REPO_CWD, - workspaceId: REPO_CWD, - command: { command: "list_tabs", args: { workspaceId: REPO_CWD } }, + command: { command: "list_tabs", args: {} }, }); expect(response.content).toEqual([ { type: "text", - text: "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab.", + text: "No Paseo browser tabs are open. Call browser_new_tab to create one.", }, ]); expect(response.structuredContent).toEqual({ ok: true, result: { command: "list_tabs", tabs: [] }, - context: { agentId: "agent-1", cwd: REPO_CWD, workspaceId: REPO_CWD }, + context: { agentId: "agent-1", cwd: REPO_CWD }, }); }); }); diff --git a/packages/server/src/server/browser-tools/broker.test.ts b/packages/server/src/server/browser-tools/broker.test.ts index 5ba99affb..ea6856baf 100644 --- a/packages/server/src/server/browser-tools/broker.test.ts +++ b/packages/server/src/server/browser-tools/broker.test.ts @@ -7,6 +7,8 @@ import type { import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js"; import { StaticBrowserToolsPolicy } from "./policy.js"; +const BROWSER_ID = "11111111-1111-4111-8111-111111111111"; + class FakeDesktopClient implements BrowserToolsDesktopClient { public readonly receivedRequests: BrowserAutomationExecuteRequest[] = []; @@ -44,7 +46,7 @@ function createBroker(options: { enabled: boolean; timeoutMs?: number }): Browse } function pageInfoCommand(): BrowserAutomationCommand { - return { command: "page_info", args: { browserId: "browser-1" } }; + return { command: "page_info", args: { browserId: BROWSER_ID } }; } describe("BrowserToolsBroker", () => { @@ -111,7 +113,7 @@ describe("BrowserToolsBroker", () => { broker.registerClient(client); const resultPromise = broker.execute({ - command: { command: "list_tabs", args: { workspaceId: "workspace-1" } }, + command: { command: "list_tabs", args: {} }, workspaceId: "workspace-1", }); @@ -120,7 +122,7 @@ describe("BrowserToolsBroker", () => { type: "browser.automation.execute.request", requestId: "req-1", workspaceId: "workspace-1", - command: { command: "list_tabs", args: { workspaceId: "workspace-1" } }, + command: { command: "list_tabs", args: {} }, }, ]); expect(broker.getPendingRequestCount()).toBe(1); @@ -133,7 +135,7 @@ describe("BrowserToolsBroker", () => { command: "list_tabs", tabs: [ { - browserId: "browser-1", + browserId: BROWSER_ID, workspaceId: "workspace-1", url: "https://example.com", title: "Example", @@ -150,7 +152,7 @@ describe("BrowserToolsBroker", () => { command: "list_tabs", tabs: [ { - browserId: "browser-1", + browserId: BROWSER_ID, workspaceId: "workspace-1", url: "https://example.com", title: "Example", @@ -169,7 +171,7 @@ describe("BrowserToolsBroker", () => { broker.registerClient(client); const resultPromise = broker.execute({ - command: { command: "snapshot", args: { workspaceId: "workspace-1" } }, + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, workspaceId: "workspace-1", }); @@ -178,7 +180,7 @@ describe("BrowserToolsBroker", () => { type: "browser.automation.execute.request", requestId: "req-1", workspaceId: "workspace-1", - command: { command: "snapshot", args: { workspaceId: "workspace-1" } }, + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, }, ]); @@ -187,7 +189,7 @@ describe("BrowserToolsBroker", () => { ok: true, result: { command: "snapshot", - browserId: "browser-1", + browserId: BROWSER_ID, workspaceId: "workspace-1", url: "https://example.com", title: "Example", @@ -200,7 +202,7 @@ describe("BrowserToolsBroker", () => { ok: true, result: { command: "snapshot", - browserId: "browser-1", + browserId: BROWSER_ID, workspaceId: "workspace-1", url: "https://example.com", title: "Example", @@ -281,7 +283,7 @@ describe("BrowserToolsBroker", () => { ok: false, error: { code: "browser_tab_not_found", - message: "Browser tab browser-1 was not found.", + message: `Browser tab ${BROWSER_ID} was not found.`, retryable: false, }, }); @@ -291,7 +293,7 @@ describe("BrowserToolsBroker", () => { ok: false, error: { code: "browser_tab_not_found", - message: "Browser tab browser-1 was not found.", + message: `Browser tab ${BROWSER_ID} was not found.`, retryable: false, }, }); diff --git a/packages/server/src/server/browser-tools/broker.ts b/packages/server/src/server/browser-tools/broker.ts index cc0d7a0a7..83ddc4a69 100644 --- a/packages/server/src/server/browser-tools/broker.ts +++ b/packages/server/src/server/browser-tools/broker.ts @@ -19,7 +19,6 @@ export interface BrowserToolsExecuteInput { agentId?: string; cwd?: string; workspaceId?: string; - browserId?: string; requestId?: string; timeoutMs?: number; } @@ -114,7 +113,6 @@ export class BrowserToolsBroker { ...(input.agentId ? { agentId: input.agentId } : {}), ...(input.cwd ? { cwd: input.cwd } : {}), ...(input.workspaceId ? { workspaceId: input.workspaceId } : {}), - ...(input.browserId ? { browserId: input.browserId } : {}), command: input.command, }); diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts index 67eb3c8d5..a65930615 100644 --- a/packages/server/src/server/browser-tools/tools.test.ts +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -1,936 +1,303 @@ -import { describe, expect, it, vi } from "vitest"; -import type { BrowserToolsBroker } from "./broker.js"; +import { describe, expect, test } from "vitest"; +import { z } from "zod"; +import type { BrowserToolsBroker, BrowserToolsExecuteInput } from "./broker.js"; import type { BrowserToolsResponsePayload } from "./errors.js"; import { registerBrowserTools, type RegisterBrowserToolsOptions } from "./tools.js"; +import type { + PaseoToolConfig, + PaseoToolExecutionContext, + PaseoToolResult, +} from "../agent/tools/types.js"; + +const BROWSER_ID = "11111111-1111-4111-8111-111111111111"; +const BROWSER_ID_MESSAGE = + "browserId must be a real id returned by browser_new_tab or browser_list_tabs"; +const WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url"; interface RegisteredTool { - config: { inputSchema: Record; outputSchema: unknown }; - handler: (args: Record) => Promise<{ - content: Array<{ type: string; text?: string }>; - structuredContent?: Record; - }>; + config: PaseoToolConfig; + handler: (args: unknown, context: PaseoToolExecutionContext) => Promise; } -function createHarness(options?: { - brokerResponse?: BrowserToolsResponsePayload; - resolveCallerAgent?: RegisterBrowserToolsOptions["resolveCallerAgent"]; - callerAgentId?: string | null; -}) { - const tools = new Map(); - const execute = vi.fn(async () => { - return ( - options?.brokerResponse ?? { - requestId: "req-1", - ok: true as const, - result: { - command: "list_tabs" as const, - tabs: [ - { - browserId: "browser-1", - url: "https://example.com", - title: "Example", - isActive: true, - isLoading: false, - }, - ], - }, - } - ); - }); - const registerTool = vi.fn((name, config, handler) => { - tools.set(name, { - config, - handler: handler as RegisteredTool["handler"], - }); - }) as unknown as RegisterBrowserToolsOptions["registerTool"]; +class FakeBrowserBroker { + public readonly calls: BrowserToolsExecuteInput[] = []; - registerBrowserTools({ - registerTool, - broker: { execute } as unknown as BrowserToolsBroker, - ...(options?.callerAgentId !== null - ? { callerAgentId: options?.callerAgentId ?? "agent-1" } - : {}), - resolveCallerAgent: - options?.resolveCallerAgent ?? - (() => ({ id: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" })), - }); + public constructor(private response: BrowserToolsResponsePayload = listTabsPayload()) {} - return { tools, execute }; -} - -function tool(harness: ReturnType, name: string): RegisteredTool { - const registered = harness.tools.get(name); - if (!registered) { - throw new Error(`Tool not registered: ${name}`); + public setResponse(response: BrowserToolsResponsePayload): void { + this.response = response; } - return registered; + + public async execute(input: BrowserToolsExecuteInput): Promise { + this.calls.push(input); + return this.response; + } +} + +class BrowserToolHarness { + public readonly broker = new FakeBrowserBroker(); + private readonly tools = new Map(); + + public constructor( + private readonly callerAgent: ReturnType = { + id: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + }, + private readonly callerAgentId: string | null = "agent-1", + ) { + registerBrowserTools({ + registerTool: (name, config, handler) => { + this.tools.set(name, { config, handler }); + }, + broker: this.broker as Pick, + ...(this.callerAgentId ? { callerAgentId: this.callerAgentId } : {}), + resolveCallerAgent: () => this.callerAgent, + }); + } + + public validate(name: string, input: unknown) { + return schemaFor(this.get(name).config.inputSchema).safeParse(input); + } + + public async execute(name: string, input: unknown): Promise { + const parsed = schemaFor(this.get(name).config.inputSchema).parse(input); + return this.get(name).handler(parsed, {}); + } + + private get(name: string): RegisteredTool { + const tool = this.tools.get(name); + if (!tool) { + throw new Error(`Tool not registered: ${name}`); + } + return tool; + } +} + +function schemaFor(inputSchema: PaseoToolConfig["inputSchema"]): z.ZodType { + if (!inputSchema) { + return z.object({}).passthrough(); + } + if (typeof (inputSchema as { safeParse?: unknown }).safeParse === "function") { + return inputSchema as z.ZodType; + } + return z.object(inputSchema as z.ZodRawShape).passthrough(); +} + +function listTabsPayload(): Extract { + return { + requestId: "req-list-tabs", + ok: true, + result: { + command: "list_tabs", + tabs: [ + { + browserId: BROWSER_ID, + url: "https://example.com", + title: "Example", + isActive: true, + isLoading: false, + }, + ], + }, + }; } describe("registerBrowserTools", () => { - it("registers browser_list_tabs and routes through the broker with caller workspace", async () => { - const harness = createHarness(); + test("list tabs sends workspace in the request envelope", async () => { + const harness = new BrowserToolHarness(); - const response = await tool(harness, "browser_list_tabs").handler({}); + const response = await harness.execute("browser_list_tabs", {}); - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "list_tabs", args: { workspaceId: "wks_workspace_a" } }, - }); + expect(harness.broker.calls).toEqual([ + { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + command: { command: "list_tabs", args: {} }, + }, + ]); expect(response.content).toEqual([ { type: "text", - text: "Found 1 Paseo browser tab. Use these browserId values exactly; do not use 'default'. You may omit browserId to use the active tab.\n- browserId=browser-1 active title=\"Example\" url=https://example.com", + text: `Found 1 Paseo browser tab. Use these browserId values for tab-scoped browser tools.\n- browserId=${BROWSER_ID} active title="Example" url=https://example.com`, }, ]); - expect(response.structuredContent).toEqual({ + }); + + test("new tab sends workspace in the request envelope", async () => { + const harness = new BrowserToolHarness(); + harness.broker.setResponse({ + requestId: "req-new-tab", ok: true, result: { - command: "list_tabs", - tabs: [ - { - browserId: "browser-1", - url: "https://example.com", - title: "Example", - isActive: true, - isLoading: false, - }, - ], - }, - context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" }, - }); - }); - - it("omits workspaceId when the caller agent has no workspaceId", async () => { - const harness = createHarness({ - resolveCallerAgent: () => ({ id: "agent-1", cwd: "/repo" }), - }); - - const response = await tool(harness, "browser_list_tabs").handler({}); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - command: { command: "list_tabs", args: {} }, - }); - expect(response.structuredContent?.context).toEqual({ agentId: "agent-1", cwd: "/repo" }); - }); - - it("uses empty browser context when there is no caller agent", async () => { - const harness = createHarness({ - callerAgentId: null, - resolveCallerAgent: () => null, - }); - - const response = await tool(harness, "browser_list_tabs").handler({}); - - expect(harness.execute).toHaveBeenCalledWith({ - command: { command: "list_tabs", args: {} }, - }); - expect(response.structuredContent?.context).toEqual({}); - }); - - it("registers browser_new_tab and returns the created tab handle", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-new-tab", - ok: true, - result: { - command: "new_tab", - browserId: "browser-new", - workspaceId: "wks_workspace_a", - url: "https://example.com", - }, - }, - }); - - const response = await tool(harness, "browser_new_tab").handler({ url: "https://example.com" }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "new_tab", - args: { workspaceId: "wks_workspace_a", url: "https://example.com" }, + browserId: BROWSER_ID, + workspaceId: "wks_workspace_a", + url: "https://example.com", }, }); + + const response = await harness.execute("browser_new_tab", { url: "https://example.com" }); + + expect(harness.broker.calls).toEqual([ + { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + command: { command: "new_tab", args: { url: "https://example.com" } }, + }, + ]); expect(response.content).toEqual([ { type: "text", - text: "Created browser tab browserId=browser-new url=https://example.com. Use this browserId exactly, or omit browserId to use this active tab.", + text: `Created browser tab browserId=${BROWSER_ID} url=https://example.com. Use this browserId for tab-scoped browser tools.`, }, ]); }); - it("routes browser_page_info through the broker with explicit browserId", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-2", - ok: true, - result: { - command: "page_info", - tab: { - browserId: "browser-2", - url: "https://example.com/docs", - title: "Docs", - isActive: false, - isLoading: false, - }, - }, + test("snapshot rejects calls without a browser id", () => { + const harness = new BrowserToolHarness(); + + const parsed = harness.validate("browser_snapshot", {}); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] }, + }); + }); + + test("snapshot rejects hallucinated browser ids", () => { + const harness = new BrowserToolHarness(); + + const parsed = harness.validate("browser_snapshot", { browserId: "default" }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] }, + }); + }); + + test("snapshot sends browser id in command args only", async () => { + const harness = new BrowserToolHarness(); + harness.broker.setResponse({ + requestId: "req-snapshot", + ok: true, + result: { + command: "snapshot", + browserId: BROWSER_ID, + workspaceId: "wks_workspace_a", + url: "https://example.com", + title: "Example", + elements: [], }, }); - const response = await tool(harness, "browser_page_info").handler({ browserId: "browser-2" }); + const response = await harness.execute("browser_snapshot", { browserId: BROWSER_ID }); - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - browserId: "browser-2", - command: { - command: "page_info", - args: { workspaceId: "wks_workspace_a", browserId: "browser-2" }, + expect(harness.broker.calls).toEqual([ + { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Current page browserId=browser-2: Docs — https://example.com/docs" }, ]); expect(response.structuredContent).toEqual({ ok: true, result: { - command: "page_info", - tab: { - browserId: "browser-2", - url: "https://example.com/docs", - title: "Docs", - isActive: false, - isLoading: false, - }, + command: "snapshot", + browserId: BROWSER_ID, + workspaceId: "wks_workspace_a", + url: "https://example.com", + title: "Example", + elements: [], }, context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a", - browserId: "browser-2", + browserId: BROWSER_ID, }, }); }); - it("routes browser_snapshot through the broker with caller workspace", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-snapshot", - ok: true, - result: { - command: "snapshot", - browserId: "browser-1", - workspaceId: "wks_workspace_a", - url: "https://example.com/form", - title: "Fixture", - elements: [ - { - ref: "@e1", - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name" }, - }, - ], - }, - }, + test("wait rejects calls without a condition", () => { + const harness = new BrowserToolHarness(); + + const parsed = harness.validate("browser_wait", { browserId: BROWSER_ID }); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: WAIT_CONDITION_MESSAGE })] }, + }); + }); + + test("wait rejects empty calls", () => { + const harness = new BrowserToolHarness(); + + const parsed = harness.validate("browser_wait", {}); + + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: BROWSER_ID_MESSAGE })] }, + }); + }); + + test("wait rejects calls with both text and url", () => { + const harness = new BrowserToolHarness(); + + const parsed = harness.validate("browser_wait", { + browserId: BROWSER_ID, + text: "Ready", + url: "/ready", }); - const response = await tool(harness, "browser_snapshot").handler({}); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "snapshot", args: { workspaceId: "wks_workspace_a" } }, + expect(parsed).toMatchObject({ + success: false, + error: { issues: [expect.objectContaining({ message: WAIT_CONDITION_MESSAGE })] }, }); - expect(response.content).toEqual([{ type: "text", text: "Snapshot captured 1 element." }]); - expect(response.structuredContent).toEqual({ + }); + + test("wait sends the text condition and extends the broker timeout", async () => { + const harness = new BrowserToolHarness(); + harness.broker.setResponse({ + requestId: "req-wait", ok: true, - result: { - command: "snapshot", - browserId: "browser-1", - workspaceId: "wks_workspace_a", - url: "https://example.com/form", - title: "Fixture", - elements: [ - { - ref: "@e1", - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name" }, - }, - ], - }, - context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" }, - }); - }); - - it("routes browser_set_background through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-bg", - ok: true, - result: { command: "set_background", browserId: "browser-1", color: "red" }, - }, + result: { command: "wait", browserId: BROWSER_ID, matched: "text" }, }); - const response = await tool(harness, "browser_set_background").handler({ color: "red" }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "set_background", - args: { workspaceId: "wks_workspace_a", color: "red" }, - }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Set browser page background to red." }, - ]); - }); - - it("routes browser_click through the broker with a snapshot ref", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-click", - ok: true, - result: { command: "click", browserId: "browser-1", ref: "@e2" }, - }, - }); - - const response = await tool(harness, "browser_click").handler({ ref: "@e2" }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "click", args: { workspaceId: "wks_workspace_a", ref: "@e2" } }, - }); - expect(response.content).toEqual([{ type: "text", text: "Clicked browser element @e2." }]); - expect(response.structuredContent).toEqual({ - ok: true, - result: { command: "click", browserId: "browser-1", ref: "@e2" }, - context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" }, - }); - }); - - it("routes browser_fill through the broker with a snapshot ref and value", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-fill", - ok: true, - result: { command: "fill", browserId: "browser-1", ref: "@e1" }, - }, - }); - - const response = await tool(harness, "browser_fill").handler({ ref: "@e1", value: "Ada" }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "fill", - args: { workspaceId: "wks_workspace_a", ref: "@e1", value: "Ada" }, - }, - }); - expect(response.content).toEqual([{ type: "text", text: "Filled browser element @e1." }]); - expect(response.structuredContent).toEqual({ - ok: true, - result: { command: "fill", browserId: "browser-1", ref: "@e1" }, - context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" }, - }); - }); - - it("routes browser_wait through the broker with text and timeout", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-wait", - ok: true, - result: { command: "wait", browserId: "browser-1", matched: "text" }, - }, - }); - - const response = await tool(harness, "browser_wait").handler({ + const response = await harness.execute("browser_wait", { + browserId: BROWSER_ID, text: "Ready", timeoutMs: 1000, }); - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - timeoutMs: 2000, - command: { - command: "wait", - args: { workspaceId: "wks_workspace_a", text: "Ready", timeoutMs: 1000 }, + expect(harness.broker.calls).toEqual([ + { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + timeoutMs: 2000, + command: { + command: "wait", + args: { browserId: BROWSER_ID, text: "Ready", timeoutMs: 1000 }, + }, }, - }); + ]); expect(response.content).toEqual([{ type: "text", text: "Browser wait matched text." }]); + }); + + test("tools keep empty context when there is no caller agent", async () => { + const harness = new BrowserToolHarness(null, null); + + const response = await harness.execute("browser_list_tabs", {}); + + expect(harness.broker.calls).toEqual([{ command: { command: "list_tabs", args: {} } }]); expect(response.structuredContent).toEqual({ ok: true, - result: { command: "wait", browserId: "browser-1", matched: "text" }, - context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" }, - }); - }); - - it("routes browser_type through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-type", - ok: true, - result: { command: "type", browserId: "browser-1", ref: "@e1" }, - }, - }); - - const response = await tool(harness, "browser_type").handler({ ref: "@e1", text: "Ada" }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "type", - args: { workspaceId: "wks_workspace_a", ref: "@e1", text: "Ada" }, - }, - }); - expect(response.content).toEqual([{ type: "text", text: "Typed into browser element @e1." }]); - }); - - it("routes browser_keypress through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-keypress", - ok: true, - result: { command: "keypress", browserId: "browser-1", ref: "@e1", key: "Enter" }, - }, - }); - - const response = await tool(harness, "browser_keypress").handler({ - ref: "@e1", - key: "Enter", - }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "keypress", - args: { workspaceId: "wks_workspace_a", ref: "@e1", key: "Enter" }, - }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Pressed Enter on browser element @e1." }, - ]); - }); - - it("routes browser_navigate through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-nav", - ok: true, - result: { command: "navigate", browserId: "browser-1", url: "https://example.com/next" }, - }, - }); - - const response = await tool(harness, "browser_navigate").handler({ - url: "https://example.com/next", - }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "navigate", - args: { workspaceId: "wks_workspace_a", url: "https://example.com/next" }, - }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Navigated browser to https://example.com/next." }, - ]); - }); - - it("routes browser_back through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-back", - ok: true, - result: { command: "back", browserId: "browser-1" }, - }, - }); - - const response = await tool(harness, "browser_back").handler({}); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "back", args: { workspaceId: "wks_workspace_a" } }, - }); - expect(response.content).toEqual([{ type: "text", text: "Browser back complete." }]); - }); - - it("routes browser_screenshot through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-shot", - ok: true, - result: { - command: "screenshot", - browserId: "browser-1", - mimeType: "image/png", - dataBase64: "iVBORw0KGgo=", - width: 100, - height: 50, - }, - }, - }); - - const response = await tool(harness, "browser_screenshot").handler({}); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "screenshot", args: { workspaceId: "wks_workspace_a" } }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Captured browser screenshot (100x50)." }, - { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }, - ]); - }); - - it("routes browser_logs through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-logs", - ok: true, - result: { - command: "logs", - browserId: "browser-1", - console: [{ level: "info", message: "ready", timestamp: 1 }], - network: [ - { - url: "https://example.com/app.js", - type: "script", - startTime: 2, - duration: 3, - }, - ], - }, - }, - }); - - const response = await tool(harness, "browser_logs").handler({ maxEntries: 25 }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "logs", args: { workspaceId: "wks_workspace_a", maxEntries: 25 } }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Read 1 console log and 1 network entry." }, - ]); - }); - - it("routes browser_storage through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-storage", - ok: true, - result: { - command: "storage", - browserId: "browser-1", - url: "https://example.com", - cookies: [{ name: "theme", value: "dark" }], - localStorage: [{ key: "token", value: "abc" }], - sessionStorage: [{ key: "tab", value: "1" }], - }, - }, - }); - - const response = await tool(harness, "browser_storage").handler({}); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "storage", args: { workspaceId: "wks_workspace_a" } }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Read 1 cookie, 1 localStorage entry, and 1 sessionStorage entry." }, - ]); - }); - - it("routes browser_environment through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-environment", - ok: true, - result: { - command: "environment", - browserId: "browser-1", - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - }); - - const response = await tool(harness, "browser_environment").handler({ - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "environment", - args: { - workspaceId: "wks_workspace_a", - viewport: { width: 390, height: 844, deviceScaleFactor: 3 }, - geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 }, - }, - }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Browser environment viewport is 390x844." }, - ]); - }); - - it("routes browser_full_page_screenshot through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-full-page", - ok: true, - result: { - command: "full_page_screenshot", - browserId: "browser-1", - mimeType: "image/png", - dataBase64: "iVBORw0KGgo=", - width: 390, - height: 1200, - }, - }, - }); - - const response = await tool(harness, "browser_full_page_screenshot").handler({}); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "full_page_screenshot", args: { workspaceId: "wks_workspace_a" } }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Captured full-page browser screenshot (390x1200)." }, - { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }, - ]); - }); - - it("routes browser_pdf through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-pdf", - ok: true, - result: { - command: "pdf", - browserId: "browser-1", - mimeType: "application/pdf", - dataBase64: "JVBERg==", - }, - }, - }); - - const response = await tool(harness, "browser_pdf").handler({ landscape: true }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "pdf", - args: { workspaceId: "wks_workspace_a", landscape: true, printBackground: true }, - }, - }); - expect(response.content).toEqual([{ type: "text", text: "Exported browser page PDF." }]); - }); - - it("routes browser_download through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-download", - ok: true, - result: { - command: "download", - browserId: "browser-1", - url: "https://example.com/file.txt", - filePath: "/tmp/file.txt", - totalBytes: 5, - state: "completed", - }, - }, - }); - - const response = await tool(harness, "browser_download").handler({ - url: "https://example.com/file.txt", - fileName: "file.txt", - }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "download", - args: { - workspaceId: "wks_workspace_a", - url: "https://example.com/file.txt", - fileName: "file.txt", - }, - }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Downloaded browser file to /tmp/file.txt." }, - ]); - }); - - it("routes browser_upload through the broker", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-upload", - ok: true, - result: { - command: "upload", - browserId: "browser-1", - ref: "@e1", - filePaths: ["/tmp/file.txt"], - }, - }, - }); - - const response = await tool(harness, "browser_upload").handler({ - ref: "@e1", - filePaths: ["/tmp/file.txt"], - }); - - expect(harness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "upload", - args: { workspaceId: "wks_workspace_a", ref: "@e1", filePaths: ["/tmp/file.txt"] }, - }, - }); - expect(response.content).toEqual([ - { type: "text", text: "Uploaded 1 file to browser element @e1." }, - ]); - }); - - it("routes browser form control tools through the broker", async () => { - const focusHarness = createHarness({ - brokerResponse: { - requestId: "req-focus", - ok: true, - result: { command: "focus", browserId: "browser-1", ref: "@e1" }, - }, - }); - const focusResponse = await tool(focusHarness, "browser_focus").handler({ ref: "@e1" }); - expect(focusHarness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "focus", args: { workspaceId: "wks_workspace_a", ref: "@e1" } }, - }); - expect(focusResponse.content).toEqual([{ type: "text", text: "Focused browser element @e1." }]); - - const checkHarness = createHarness({ - brokerResponse: { - requestId: "req-check", - ok: true, - result: { command: "check", browserId: "browser-1", ref: "@e2", checked: false }, - }, - }); - const checkResponse = await tool(checkHarness, "browser_check").handler({ - ref: "@e2", - checked: false, - }); - expect(checkHarness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "check", - args: { workspaceId: "wks_workspace_a", ref: "@e2", checked: false }, - }, - }); - expect(checkResponse.content).toEqual([ - { type: "text", text: "Unchecked browser element @e2." }, - ]); - - const selectHarness = createHarness({ - brokerResponse: { - requestId: "req-select", - ok: true, - result: { command: "select", browserId: "browser-1", ref: "@e3", value: "us" }, - }, - }); - const selectResponse = await tool(selectHarness, "browser_select").handler({ - ref: "@e3", - value: "us", - }); - expect(selectHarness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "select", - args: { workspaceId: "wks_workspace_a", ref: "@e3", value: "us" }, - }, - }); - expect(selectResponse.content).toEqual([ - { type: "text", text: "Selected us in browser element @e3." }, - ]); - - const hoverHarness = createHarness({ - brokerResponse: { - requestId: "req-hover", - ok: true, - result: { command: "hover", browserId: "browser-1", ref: "@e4" }, - }, - }); - const hoverResponse = await tool(hoverHarness, "browser_hover").handler({ ref: "@e4" }); - expect(hoverHarness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { command: "hover", args: { workspaceId: "wks_workspace_a", ref: "@e4" } }, - }); - expect(hoverResponse.content).toEqual([{ type: "text", text: "Hovered browser element @e4." }]); - - const dragHarness = createHarness({ - brokerResponse: { - requestId: "req-drag", - ok: true, - result: { command: "drag", browserId: "browser-1", sourceRef: "@e4", targetRef: "@e5" }, - }, - }); - const dragResponse = await tool(dragHarness, "browser_drag").handler({ - sourceRef: "@e4", - targetRef: "@e5", - }); - expect(dragHarness.execute).toHaveBeenCalledWith({ - agentId: "agent-1", - cwd: "/repo", - workspaceId: "wks_workspace_a", - command: { - command: "drag", - args: { workspaceId: "wks_workspace_a", sourceRef: "@e4", targetRef: "@e5" }, - }, - }); - expect(dragResponse.content).toEqual([ - { type: "text", text: "Dragged browser element @e4 to @e5." }, - ]); - }); - - it("returns model-actionable disabled errors with structured content", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-disabled", - ok: false, - error: { - code: "browser_disabled", - message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.", - retryable: false, - }, - }, - }); - - const response = await tool(harness, "browser_list_tabs").handler({}); - - expect(response.content).toEqual([ - { - type: "text", - text: "Browser tools are disabled. Enable desktop browser tools on the host, then try again.", - }, - ]); - expect(response.structuredContent).toEqual({ - ok: false, - error: { - code: "browser_disabled", - message: "Browser tools are disabled. Enable daemon.browserTools.enabled to use them.", - retryable: false, - }, - context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" }, - }); - }); - - it("preserves typed broker errors", async () => { - const harness = createHarness({ - brokerResponse: { - requestId: "req-timeout", - ok: false, - error: { - code: "browser_timeout", - message: "Browser automation timed out after 15000ms.", - retryable: true, - }, - }, - }); - - const response = await tool(harness, "browser_page_info").handler({}); - - expect(response.content).toEqual([ - { - type: "text", - text: "The browser did not respond before the timeout. Try again or check the desktop app.", - }, - ]); - expect(response.structuredContent?.error).toEqual({ - code: "browser_timeout", - message: "Browser automation timed out after 15000ms.", - retryable: true, - }); - }); - - it("preserves screenshot no-frame broker messages", async () => { - const message = - "The browser tab has no painted frame. Focus the tab in the app, then try again."; - const harness = createHarness({ - brokerResponse: { - requestId: "req-no-frame", - ok: false, - error: { - code: "screenshot_no_frame", - message, - retryable: false, - }, - }, - }); - - const response = await tool(harness, "browser_screenshot").handler({}); - - expect(response.content).toEqual([{ type: "text", text: message }]); - expect(response.structuredContent?.error).toEqual({ - code: "screenshot_no_frame", - message, - retryable: false, + result: listTabsPayload().result, + context: {}, }); }); }); diff --git a/packages/server/src/server/browser-tools/tools.ts b/packages/server/src/server/browser-tools/tools.ts index 3d8236cb9..befa5a224 100644 --- a/packages/server/src/server/browser-tools/tools.ts +++ b/packages/server/src/server/browser-tools/tools.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import type { BrowserToolsBroker } from "./broker.js"; import type { BrowserToolsResponsePayload } from "./errors.js"; import type { @@ -49,6 +50,16 @@ const BrowserToolOutputSchema = z.object({ }); const BrowserRefInputSchema = z.string().regex(/^@e\d+$/); +const BrowserWaitInputSchema = z + .object({ + text: z.string().min(1).optional(), + url: z.string().min(1).optional(), + timeoutMs: z.number().int().positive().max(30_000).optional(), + browserId: BrowserAutomationBrowserIdSchema, + }) + .refine((input) => Number(Boolean(input.text)) + Number(Boolean(input.url)) === 1, { + message: "browser_wait requires exactly one of text or url", + }); export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { options.registerTool( @@ -56,7 +67,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "List browser tabs", description: - "List open Paseo desktop browser tabs for this agent's workspace context. Use the returned browserId values exactly; do not use 'default'. If a tool omits browserId, it targets the active workspace tab.", + "List open Paseo desktop browser tabs for this agent's workspace context. Use the returned browserId values for tab-scoped browser tools.", inputSchema: {}, outputSchema: BrowserToolOutputSchema, }, @@ -68,7 +79,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), command: { command: "list_tabs", - args: context.workspaceId ? { workspaceId: context.workspaceId } : {}, + args: {}, }, }); return browserToolResult({ payload, context }); @@ -80,7 +91,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Create browser tab", description: - "Create and focus a new Paseo desktop browser tab in this agent's workspace. Optionally pass an http(s) URL to open immediately. Returns the new browserId; use that exact ID for later calls or omit browserId to use the active tab.", + "Create and focus a new Paseo desktop browser tab in this agent's workspace. Optionally pass an http(s) URL to open immediately. Returns the browserId for later tab-scoped browser tools.", inputSchema: { url: z.string().url().optional(), }, @@ -94,10 +105,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), command: { command: "new_tab", - args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(url ? { url } : {}), - }, + args: url ? { url } : {}, }, }); return browserToolResult({ payload, context }); @@ -109,9 +117,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Get browser page info", description: - "Get page info for a Paseo desktop browser tab. Omit browserId to use the active workspace tab. If you pass browserId, use a real ID returned by browser_list_tabs or browser_new_tab; never pass 'default'.", + "Get page info for a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.", inputSchema: { - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -121,12 +129,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "page_info", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, }, }, }); @@ -139,9 +146,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Snapshot browser page", description: - "Return a model-readable snapshot of the active Paseo desktop browser tab for this agent's workspace. Snapshot refs like @e1 are valid until the page changes or a new snapshot is taken.", + "Return a model-readable snapshot of a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs. Snapshot refs like @e1 are valid until the page changes or a new snapshot is taken.", inputSchema: { - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -151,12 +158,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "snapshot", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, }, }, }); @@ -172,7 +178,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void "Click an element ref from the latest browser_snapshot for this agent's workspace browser tab.", inputSchema: { ref: BrowserRefInputSchema, - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -182,12 +188,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "click", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ref, }, }, @@ -205,7 +210,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { ref: BrowserRefInputSchema, value: z.string(), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -215,12 +220,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "fill", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ref, value, }, @@ -235,13 +239,8 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Wait for browser condition", description: - "Wait until the active Paseo desktop browser tab contains text or reaches a URL fragment.", - inputSchema: { - text: z.string().min(1).optional(), - url: z.string().min(1).optional(), - timeoutMs: z.number().int().positive().max(30_000).optional(), - browserId: z.string().min(1).optional(), - }, + "Wait until a Paseo desktop browser tab contains text or reaches a URL fragment. Pass browserId from browser_new_tab or browser_list_tabs.", + inputSchema: BrowserWaitInputSchema, outputSchema: BrowserToolOutputSchema, }, async ({ text, url, timeoutMs, browserId }) => { @@ -250,13 +249,12 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + ...(timeoutMs ? { timeoutMs: timeoutMs + 1_000 } : {}), command: { command: "wait", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ...(text ? { text } : {}), ...(url ? { url } : {}), ...(timeoutMs ? { timeoutMs } : {}), @@ -276,7 +274,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { text: z.string(), ref: BrowserRefInputSchema.optional(), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -286,12 +284,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "type", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ...(ref ? { ref } : {}), text, }, @@ -310,7 +307,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { key: z.string().min(1), ref: BrowserRefInputSchema.optional(), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -320,12 +317,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "keypress", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ...(ref ? { ref } : {}), key, }, @@ -339,8 +335,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void "browser_navigate", { title: "Navigate browser", - description: "Navigate the active Paseo desktop browser tab to a URL.", - inputSchema: { url: z.string().min(1), browserId: z.string().min(1).optional() }, + description: + "Navigate a Paseo desktop browser tab to a URL. Pass browserId from browser_new_tab or browser_list_tabs.", + inputSchema: { url: z.string().min(1), browserId: BrowserAutomationBrowserIdSchema }, outputSchema: BrowserToolOutputSchema, }, async ({ url, browserId }) => { @@ -349,12 +346,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "navigate", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, url, }, }, @@ -369,8 +365,8 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void name, { title: `Browser ${command}`, - description: `${command} the active Paseo desktop browser tab.`, - inputSchema: { browserId: z.string().min(1).optional() }, + description: `${command} a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.`, + inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, outputSchema: BrowserToolOutputSchema, }, async ({ browserId }) => { @@ -379,12 +375,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command, args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, }, }, }); @@ -397,8 +392,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void "browser_screenshot", { title: "Capture browser screenshot", - description: "Capture a PNG screenshot of the active Paseo desktop browser tab.", - inputSchema: { browserId: z.string().min(1).optional() }, + description: + "Capture a PNG screenshot of a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.", + inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, outputSchema: BrowserToolOutputSchema, }, async ({ browserId }) => { @@ -407,12 +403,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "screenshot", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, }, }, }); @@ -425,7 +420,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Capture full-page browser screenshot", description: "Capture a full-page PNG screenshot of a Paseo desktop browser tab.", - inputSchema: { browserId: z.string().min(1).optional() }, + inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, outputSchema: BrowserToolOutputSchema, }, async ({ browserId }) => { @@ -434,12 +429,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "full_page_screenshot", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, }, }, }); @@ -451,9 +445,10 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void "browser_pdf", { title: "Export browser page PDF", - description: "Export the current Paseo desktop browser tab as a PDF.", + description: + "Export a Paseo desktop browser tab as a PDF. Pass browserId from browser_new_tab or browser_list_tabs.", inputSchema: { - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, landscape: z.boolean().optional(), printBackground: z.boolean().default(true), }, @@ -465,12 +460,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "pdf", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ...(landscape !== undefined ? { landscape } : {}), printBackground: printBackground ?? true, }, @@ -488,7 +482,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { url: z.string().min(1), fileName: z.string().min(1).optional(), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -498,12 +492,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "download", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, url, ...(fileName ? { fileName } : {}), }, @@ -522,7 +515,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { ref: BrowserRefInputSchema, filePaths: z.array(z.string().min(1)).min(1), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -532,12 +525,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "upload", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ref, filePaths, }, @@ -572,7 +564,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: toolConfig.title, description: toolConfig.description, - inputSchema: { ref: BrowserRefInputSchema, browserId: z.string().min(1).optional() }, + inputSchema: { ref: BrowserRefInputSchema, browserId: BrowserAutomationBrowserIdSchema }, outputSchema: BrowserToolOutputSchema, }, async ({ ref, browserId }) => { @@ -581,12 +573,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: toolConfig.command, args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ref, }, }, @@ -604,7 +595,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { ref: BrowserRefInputSchema, checked: z.boolean().default(true), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -614,12 +605,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "check", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ref, checked, }, @@ -637,7 +627,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { ref: BrowserRefInputSchema, value: z.string(), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -647,12 +637,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "select", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ref, value, }, @@ -671,7 +660,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { sourceRef: BrowserRefInputSchema, targetRef: BrowserRefInputSchema, - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -681,12 +670,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "drag", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, sourceRef, targetRef, }, @@ -704,7 +692,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void "Read recent console messages and browser performance network entries for a Paseo desktop browser tab.", inputSchema: { maxEntries: z.number().int().positive().max(200).optional(), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -714,12 +702,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "logs", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, maxEntries: maxEntries ?? 50, }, }, @@ -734,7 +721,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void title: "Read browser storage", description: "Read cookies plus localStorage and sessionStorage for a Paseo desktop browser tab.", - inputSchema: { browserId: z.string().min(1).optional() }, + inputSchema: { browserId: BrowserAutomationBrowserIdSchema }, outputSchema: BrowserToolOutputSchema, }, async ({ browserId }) => { @@ -743,12 +730,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "storage", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, }, }, }); @@ -776,7 +762,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void accuracy: z.number().positive().optional(), }) .optional(), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -786,12 +772,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "environment", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, ...(viewport ? { viewport } : {}), ...(geolocation ? { geolocation } : {}), }, @@ -806,10 +791,10 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Set browser background", description: - "Set the current page background color in the active Paseo desktop browser tab. Omit browserId to use the active workspace tab; pass a returned browserId only when targeting a specific tab.", + "Set the current page background color in a Paseo desktop browser tab. Pass browserId from browser_new_tab or browser_list_tabs.", inputSchema: { color: z.string().min(1), - browserId: z.string().min(1).optional(), + browserId: BrowserAutomationBrowserIdSchema, }, outputSchema: BrowserToolOutputSchema, }, @@ -819,12 +804,11 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + command: { command: "set_background", args: { - ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), - ...(browserId ? { browserId } : {}), + browserId, color, }, }, @@ -941,20 +925,20 @@ function summarizeBrowserSuccess( if (payload.result.command === "list_tabs") { const count = payload.result.tabs.length; if (count === 0) { - return "No Paseo browser tabs are open. Call browser_new_tab to create one, then use the returned browserId or omit browserId for the active tab."; + return "No Paseo browser tabs are open. Call browser_new_tab to create one."; } const tabLines = payload.result.tabs.map((tab) => { const active = tab.isActive ? " active" : ""; return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url}`; }); return [ - `Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values exactly; do not use 'default'. You may omit browserId to use the active tab.`, + `Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values for tab-scoped browser tools.`, ...tabLines, ].join("\n"); } if (payload.result.command === "new_tab") { - return `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId exactly, or omit browserId to use this active tab.`; + return `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`; } if (payload.result.command === "snapshot") { @@ -1112,8 +1096,6 @@ function summarizeBrowserError( return "Browser tools are disabled. Enable desktop browser tools on the host, then try again."; case "browser_no_desktop": return "No desktop browser automation client is connected. Open the Paseo desktop app and try again."; - case "browser_no_tab": - return "No active browser tab is available. Call browser_new_tab to create one, then omit browserId for the active tab or use the returned browserId."; case "browser_timeout": return "The browser did not respond before the timeout. Try again or check the desktop app."; case "screenshot_no_frame": diff --git a/packages/server/src/server/websocket-server.browser-tools.test.ts b/packages/server/src/server/websocket-server.browser-tools.test.ts index 90c3eebad..f3c8d0c97 100644 --- a/packages/server/src/server/websocket-server.browser-tools.test.ts +++ b/packages/server/src/server/websocket-server.browser-tools.test.ts @@ -51,6 +51,7 @@ interface QueuedBrowserRequests { } const harnesses: BrowserToolsDaemonHarness[] = []; +const BROWSER_ID = "11111111-1111-4111-8111-111111111111"; afterEach(async () => { await Promise.all(harnesses.splice(0).map((harness) => harness.stop())); @@ -131,7 +132,7 @@ describe("WebSocketServer browser tools wiring", () => { }); const resultPromise = harness.broker.execute({ - command: { command: "click", args: { ref: "@e1" } }, + command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } }, }); const request = await resumedDesktop.nextBrowserRequest(); resumedDesktop.respondToBrowserRequest({ @@ -139,13 +140,13 @@ describe("WebSocketServer browser tools wiring", () => { payload: { requestId: request.requestId, ok: true, - result: { command: "click", browserId: "browser-1", ref: "@e1" }, + result: { command: "click", browserId: BROWSER_ID, ref: "@e1" }, }, }); await expect(resultPromise).resolves.toMatchObject({ ok: true, - result: { command: "click", browserId: "browser-1", ref: "@e1" }, + result: { command: "click", browserId: BROWSER_ID, ref: "@e1" }, }); }); });