diff --git a/docs/browser-capture-harness.md b/docs/browser-capture-harness.md index dd09d5a02..24da98263 100644 --- a/docs/browser-capture-harness.md +++ b/docs/browser-capture-harness.md @@ -21,6 +21,18 @@ Run it with the repo Electron: npm run capture-harness --workspace=@getpaseo/desktop ``` +Run the browser automation fixture with: + +```bash +PASEO_CAPTURE_HARNESS_GROUP=automation npm run capture-harness --workspace=@getpaseo/desktop +``` + +The automation group uses a real guest webview to verify the page-side ref contract: +ARIA-like snapshot text includes headings, static text, and controls; refs survive +`pushState` when the element still matches; same-URL rerenders stale old refs; and a +file-input ref can be resolved to a CDP backend node id for upload. It also verifies +page-context evaluation, including passing a resolved ref element as the function argument. + On macOS the harness process must set `app.setActivationPolicy("accessory")` and hide the Dock icon before creating any window. `showInactive()` only prevents window focus; a normal Electron app launch can still activate the app and steal focus. diff --git a/packages/app/src/browser-automation/handler.test.ts b/packages/app/src/browser-automation/handler.test.ts index 756c7262b..a40fb7699 100644 --- a/packages/app/src/browser-automation/handler.test.ts +++ b/packages/app/src/browser-automation/handler.test.ts @@ -75,6 +75,8 @@ class FakeBrowserBridge { public readonly executedRequests: BrowserAutomationExecuteRequest[] = []; public readonly registeredWorkspaceBrowsers: Array<{ browserId: string; workspaceId: string }> = []; + public readonly unregisteredWorkspaceBrowsers: string[] = []; + public readonly clearedPartitions: string[] = []; public readonly activeWorkspaceBrowsers: Array<{ browserId: string | null; workspaceId: string; @@ -99,6 +101,14 @@ class FakeBrowserBridge { this.registeredWorkspaceBrowsers.push(input); }; + public unregisterWorkspaceBrowser = async (browserId: string): Promise => { + this.unregisteredWorkspaceBrowsers.push(browserId); + }; + + public clearPartition = async (browserId: string): Promise => { + this.clearedPartitions.push(browserId); + }; + public setWorkspaceActiveBrowser = async (input: { browserId: string | null; workspaceId: string; @@ -175,6 +185,35 @@ function browserNewTabRequest(): BrowserAutomationExecuteRequest { }; } +function browserResizeRequest( + browserId: string, + input: { workspaceId?: string } = {}, +): BrowserAutomationExecuteRequest { + return { + type: "browser.automation.execute.request", + requestId: "req-resize", + agentId: "agent-1", + workspaceId: input.workspaceId ?? "wks_workspace_a", + command: { + command: "resize", + args: { browserId, width: 1024, height: 768 }, + }, + }; +} + +function browserCloseTabRequest(browserId: string): BrowserAutomationExecuteRequest { + return { + type: "browser.automation.execute.request", + requestId: "req-close-tab", + agentId: "agent-1", + workspaceId: "wks_workspace_a", + command: { + command: "close_tab", + args: { browserId }, + }, + }; +} + function emptyListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload { return { requestId, @@ -355,6 +394,106 @@ describe("mountBrowserAutomationHandler", () => { ]); }); + test("browser_resize updates resident webview dimensions", async () => { + const browser = new BrowserAutomationHandlerHarness(); + browser.mount({ serverId: "server-1" }); + + browser.receive(browserNewTabRequest()); + await flushAsyncWork(); + const result = newTabResultFrom(browser.client.payloadAt(0)); + + browser.receive(browserResizeRequest(result.browserId)); + await flushAsyncWork(); + + expect(browser.client.payloadAt(1)).toEqual({ + requestId: "req-resize", + ok: true, + result: { + command: "resize", + browserId: result.browserId, + width: 1024, + height: 768, + }, + }); + expect(browser.browser.executedRequests).toHaveLength(1); + }); + + test("browser_resize returns not found for a tab outside the request workspace", async () => { + const browser = new BrowserAutomationHandlerHarness(); + browser.mount({ serverId: "server-1" }); + + browser.receive(browserNewTabRequest()); + await flushAsyncWork(); + const result = newTabResultFrom(browser.client.payloadAt(0)); + + browser.receive(browserResizeRequest(result.browserId, { workspaceId: "wks_workspace_b" })); + await flushAsyncWork(); + + expect(browser.client.payloadAt(1)).toEqual({ + requestId: "req-resize", + ok: false, + error: { + code: "browser_tab_not_found", + message: `No browser tab found for ID: ${result.browserId}`, + retryable: false, + }, + }); + }); + + test("browser_close_tab removes the workspace tab, browser record, resident webview, registry entry, and partition", async () => { + const browser = new BrowserAutomationHandlerHarness(); + const workspaceKey = buildWorkspaceTabPersistenceKey({ + serverId: "server-1", + workspaceId: "wks_workspace_a", + }); + if (!workspaceKey) { + throw new Error("Expected workspace key"); + } + browser.mount({ serverId: "server-1" }); + + browser.receive(browserNewTabRequest()); + await flushAsyncWork(); + const result = newTabResultFrom(browser.client.payloadAt(0)); + + browser.receive(browserCloseTabRequest(result.browserId)); + await flushAsyncWork(); + + expect(browser.client.payloadAt(1)).toEqual({ + requestId: "req-close-tab", + ok: true, + result: { command: "close_tab", browserId: result.browserId }, + }); + expect(workspaceBrowserTabs(workspaceKey, result.browserId)).toEqual([]); + expect(useBrowserStore.getState().browsersById[result.browserId]).toBeUndefined(); + expect(browser.browser.unregisteredWorkspaceBrowsers).toEqual([result.browserId]); + expect(browser.browser.clearedPartitions).toEqual([result.browserId]); + expect(currentBrowserTabs()).toEqual([]); + }); + + test("browser_close_tab returns not found after the tab is gone", async () => { + const browser = new BrowserAutomationHandlerHarness(); + browser.mount({ serverId: "server-1" }); + + browser.receive(browserNewTabRequest()); + await flushAsyncWork(); + const result = newTabResultFrom(browser.client.payloadAt(0)); + + browser.receive(browserCloseTabRequest(result.browserId)); + await flushAsyncWork(); + browser.receive(browserCloseTabRequest(result.browserId)); + await flushAsyncWork(); + + expect(browser.client.payloadAt(2)).toEqual({ + requestId: "req-close-tab", + ok: false, + error: { + code: "browser_tab_not_found", + message: `No browser tab found for ID: ${result.browserId}`, + retryable: false, + }, + }); + }); + test("non-new-tab requests send the desktop bridge response", async () => { const browser = new BrowserAutomationHandlerHarness(); browser.browser.response = { diff --git a/packages/app/src/browser-automation/handler.ts b/packages/app/src/browser-automation/handler.ts index 44677ac0a..d728248ea 100644 --- a/packages/app/src/browser-automation/handler.ts +++ b/packages/app/src/browser-automation/handler.ts @@ -1,9 +1,14 @@ import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host"; -import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident"; -import { createWorkspaceBrowser } from "@/stores/browser-store"; +import { + ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault, + removeResidentBrowserWebview, + resizeResidentBrowserWebview, +} from "@/components/browser-webview-resident"; +import { createWorkspaceBrowser, getBrowserRecord, useBrowserStore } from "@/stores/browser-store"; import { buildWorkspaceTabPersistenceKey, + collectAllTabs, useWorkspaceLayoutStore, } from "@/stores/workspace-layout-store"; @@ -114,6 +119,33 @@ async function handleBrowserAutomationRequest(params: { return; } + if (request.command.command === "resize") { + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: resizeBrowserTabForRequest({ request, serverId }), + }); + return; + } + + if (request.command.command === "close_tab") { + try { + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: await closeBrowserTabForRequest({ + request, + serverId, + browserHost, + }), + }); + } catch (error) { + client.sendBrowserAutomationExecuteResponse({ + type: "browser.automation.execute.response", + payload: normalizeThrownBridgeError(request.requestId, error), + }); + } + return; + } + if (!executeAutomationCommand) { client.sendBrowserAutomationExecuteResponse({ type: "browser.automation.execute.response", @@ -140,6 +172,127 @@ async function handleBrowserAutomationRequest(params: { } } +function resizeBrowserTabForRequest(params: { + request: BrowserAutomationExecuteRequest; + serverId?: string; +}): BrowserAutomationResponsePayload { + const { request, serverId } = params; + const command = request.command as Extract< + BrowserAutomationExecuteRequest["command"], + { command: "resize" } + >; + const browserId = command.args.browserId; + if (!getBrowserRecord(browserId)) { + return browserAutomationFailure({ + requestId: request.requestId, + code: "browser_tab_not_found", + message: `No browser tab found for ID: ${browserId}`, + }); + } + + const workspaceId = request.workspaceId; + if (serverId && workspaceId && !findWorkspaceBrowserTab({ serverId, workspaceId, browserId })) { + return browserAutomationFailure({ + requestId: request.requestId, + code: "browser_tab_not_found", + message: `No browser tab found for ID: ${browserId}`, + }); + } + + const dimensions = resizeResidentBrowserWebview({ + browserId, + width: command.args.width, + height: command.args.height, + }); + if (!dimensions) { + return browserAutomationFailure({ + requestId: request.requestId, + code: "browser_tab_not_found", + message: `No browser tab found for ID: ${browserId}`, + }); + } + + return { + requestId: request.requestId, + ok: true, + result: { + command: "resize", + browserId, + width: dimensions.width, + height: dimensions.height, + }, + }; +} + +async function closeBrowserTabForRequest(params: { + request: BrowserAutomationExecuteRequest; + serverId?: string; + browserHost: DesktopHostBridge["browser"] | undefined; +}): Promise { + const { request, serverId, browserHost } = params; + const command = request.command as Extract< + BrowserAutomationExecuteRequest["command"], + { command: "close_tab" } + >; + const browserId = command.args.browserId; + const workspaceId = request.workspaceId; + const workspaceTab = serverId + ? findWorkspaceBrowserTab({ serverId, workspaceId, browserId }) + : null; + if (!workspaceTab && (!serverId || !workspaceId)) { + return browserAutomationFailure({ + requestId: request.requestId, + code: "browser_unsupported", + message: "Cannot close a browser tab without a workspace context.", + }); + } + if (!workspaceTab || !getBrowserRecord(browserId)) { + return browserAutomationFailure({ + requestId: request.requestId, + code: "browser_tab_not_found", + message: `No browser tab found for ID: ${browserId}`, + }); + } + + useWorkspaceLayoutStore.getState().closeTab(workspaceTab.workspaceKey, workspaceTab.tabId); + useBrowserStore.getState().removeBrowser(browserId); + removeResidentBrowserWebview(browserId); + await browserHost?.unregisterWorkspaceBrowser?.(browserId); + await browserHost?.clearPartition?.(browserId); + + return { + requestId: request.requestId, + ok: true, + result: { command: "close_tab", browserId }, + }; +} + +function findWorkspaceBrowserTab(input: { + serverId: string; + workspaceId: string | undefined; + browserId: string; +}): { workspaceKey: string; tabId: string } | null { + if (!input.workspaceId) { + return null; + } + const workspaceKey = buildWorkspaceTabPersistenceKey({ + serverId: input.serverId, + workspaceId: input.workspaceId, + }); + if (!workspaceKey) { + return null; + } + const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey]; + const tab = layout + ? collectAllTabs(layout.root).find((candidate) => { + return ( + candidate.target.kind === "browser" && candidate.target.browserId === input.browserId + ); + }) + : null; + return tab ? { workspaceKey, tabId: tab.tabId } : null; +} + async function openBrowserTabForRequest(params: { request: BrowserAutomationExecuteRequest; serverId?: string; diff --git a/packages/app/src/components/browser-webview-resident.ts b/packages/app/src/components/browser-webview-resident.ts index ac752593f..63bbed7de 100644 --- a/packages/app/src/components/browser-webview-resident.ts +++ b/packages/app/src/components/browser-webview-resident.ts @@ -4,6 +4,7 @@ const RESIDENT_VIEWPORT_WIDTH = 1280; const RESIDENT_VIEWPORT_HEIGHT = 800; const residentWebviewsByBrowserId = new Map(); +const residentWebviewSizesByBrowserId = new Map(); interface BrowserWebviewElement extends HTMLElement { src: string; @@ -66,11 +67,24 @@ function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLEle return null; } -function applyResidentWebviewStyle(webview: HTMLElement): void { +function dimensionsForBrowser(browserId: string | null): { width: number; height: number } { + if (!browserId) { + return { width: RESIDENT_VIEWPORT_WIDTH, height: RESIDENT_VIEWPORT_HEIGHT }; + } + return ( + residentWebviewSizesByBrowserId.get(browserId) ?? { + width: RESIDENT_VIEWPORT_WIDTH, + height: RESIDENT_VIEWPORT_HEIGHT, + } + ); +} + +function applyResidentWebviewStyle(webview: HTMLElement, browserId: string | null): void { + const dimensions = dimensionsForBrowser(browserId); webview.style.display = "inline-flex"; webview.style.flex = "0 0 auto"; - webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`; - webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`; + webview.style.width = `${dimensions.width}px`; + webview.style.height = `${dimensions.height}px`; webview.style.border = "0"; webview.style.background = "transparent"; webview.style.position = "absolute"; @@ -163,10 +177,33 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl } residentWebviewsByBrowserId.set(normalizedBrowserId, webview); - applyResidentWebviewStyle(webview); + applyResidentWebviewStyle(webview, normalizedBrowserId); getResidentBrowserHost(ownerDocument).appendChild(webview); } +export function resizeResidentBrowserWebview(input: { + browserId: string; + width: number; + height: number; +}): { width: number; height: number } | null { + const normalizedBrowserId = trimNonEmpty(input.browserId); + if (!normalizedBrowserId) { + return null; + } + const width = Math.max(1, Math.round(input.width)); + const height = Math.max(1, Math.round(input.height)); + residentWebviewSizesByBrowserId.set(normalizedBrowserId, { width, height }); + + const ownerDocument = readDocument(); + const webview = ownerDocument ? findBrowserWebview(normalizedBrowserId, ownerDocument) : null; + if (webview) { + webview.style.width = `${width}px`; + webview.style.height = `${height}px`; + } + + return { width, height }; +} + export function removeResidentBrowserWebview(browserId: string): void { const normalizedBrowserId = trimNonEmpty(browserId); if (!normalizedBrowserId) { @@ -175,6 +212,7 @@ export function removeResidentBrowserWebview(browserId: string): void { const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null; residentWebviewsByBrowserId.delete(normalizedBrowserId); + residentWebviewSizesByBrowserId.delete(normalizedBrowserId); resident?.remove(); } @@ -183,5 +221,6 @@ export function clearResidentBrowserWebviewsForTests(): void { webview.remove(); } residentWebviewsByBrowserId.clear(); + residentWebviewSizesByBrowserId.clear(); readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove(); } diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index 2380fcc81..6b3f0832b 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -128,6 +128,7 @@ export interface DesktopBrowserNewTabRequestEvent { export interface DesktopBrowserBridge { registerWorkspaceBrowser?: (input: { browserId: string; workspaceId: string }) => Promise; + unregisterWorkspaceBrowser?: (browserId: string) => Promise; setWorkspaceActiveBrowser?: (input: { workspaceId: string; browserId: string | null; diff --git a/packages/desktop/capture-harness/main.js b/packages/desktop/capture-harness/main.js index 84294df4e..e3e9cb54f 100644 --- a/packages/desktop/capture-harness/main.js +++ b/packages/desktop/capture-harness/main.js @@ -1095,12 +1095,823 @@ async function runPermanentParkingGroup() { return results; } +function automationFixtureUrl() { + const html = ` + + Automation Fixture + + +
+

Settings

+
+

Connected as Maya

+ + + Read docs + + + + + + + Drop target + + + + + +
+
+ + + `; + return `data:text/html;charset=utf-8,${encodeURIComponent(html)}`; +} + +const AUTOMATION_SNAPSHOT_PROBE = String.raw`(() => { + const refs = new Map(); + const lines = ['- document "Automation Fixture"']; + let nextRef = 1; + function text(value) { + return String(value || '').replace(/\s+/g, ' ').trim(); + } + function fingerprint(element, role, name) { + return { role, name, tagName: element.tagName.toLowerCase(), type: element.getAttribute('type') || '', ariaLabel: element.getAttribute('aria-label') || '' }; + } + function runtime() { + const api = { + refs, + resolve(ref, expected) { + const element = refs.get(ref); + if (!element || !element.isConnected) return { ok: false, reason: 'stale_ref' }; + const role = roleFor(element); + const name = nameFor(element, role); + const current = fingerprint(element, role, name); + return current.role === expected.role && current.name === expected.name && current.tagName === expected.tagName && current.type === expected.type && current.ariaLabel === expected.ariaLabel + ? { ok: true, element } + : { ok: false, reason: 'stale_ref' }; + } + }; + Object.defineProperty(window, '__PASEO_BROWSER_AUTOMATION__', { configurable: true, value: api }); + return api; + } + function roleFor(element) { + const tag = element.tagName.toLowerCase(); + if (/^h[1-6]$/.test(tag)) return 'heading'; + if (tag === 'input') return element.type === 'file' ? 'button' : 'textbox'; + if (tag === 'button') return 'button'; + if (element.id === 'drag-target') return 'button'; + if (tag === 'a') return 'link'; + if (tag === 'section') return 'region'; + return ''; + } + function nameFor(element, role) { + if (element.getAttribute('aria-label')) return element.getAttribute('aria-label'); + if (element.id) { + const label = document.querySelector('label[for="' + element.id + '"]'); + if (label) return text(label.textContent); + } + return role === 'textbox' ? text(element.value || element.placeholder) : text(element.textContent); + } + const api = runtime(); + const heading = document.querySelector('h1'); + lines.push(' - heading "' + nameFor(heading, 'heading') + '" [level=1]'); + lines.push(' - text: "Connected as Maya"'); + for (const element of document.querySelectorAll('input, a, button, #drag-target')) { + const role = roleFor(element); + const name = nameFor(element, role); + const ref = '@e' + nextRef++; + const fp = fingerprint(element, role, name); + api.refs.set(ref, element); + lines.push(' - ' + role + ' "' + name + '" [ref=' + ref + ']'); + } + return { + snapshot: lines.join('\n'), + refs: Array.from(api.refs.entries()).map(([ref, element]) => { + const role = roleFor(element); + return { ref, fingerprint: fingerprint(element, role, nameFor(element, role)) }; + }) + }; +})()`; + +async function attachAutomationDebugger(guest) { + if (!guest.debugger.isAttached()) { + guest.debugger.attach("1.3"); + } + return (command, params = {}) => guest.debugger.sendCommand(command, params); +} + +async function automationRefPoint(guest, ref, fingerprint) { + const result = await guest.executeJavaScript( + String.raw`(async () => { + const fingerprint = ${JSON.stringify(fingerprint)}; + const ref = ${JSON.stringify(ref)}; + const deadline = performance.now() + 5000; + const nextFrame = () => new Promise((resolve) => requestAnimationFrame(resolve)); + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const sameRect = (a, b) => + Math.abs(a.x - b.x) < 0.25 && + Math.abs(a.y - b.y) < 0.25 && + Math.abs(a.width - b.width) < 0.25 && + Math.abs(a.height - b.height) < 0.25; + const isDisabled = (element) => + Boolean(element.closest?.('[aria-disabled="true"]')) || + Boolean('disabled' in element && element.disabled); + const isVisible = (element, rect) => { + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none'; + }; + while (performance.now() <= deadline) { + const resolved = window.__PASEO_BROWSER_AUTOMATION__.resolve(ref, fingerprint); + if (!resolved.ok || !resolved.element?.isConnected) return { ok: false, reason: 'stale_ref' }; + const element = resolved.element; + const rect = element.getBoundingClientRect(); + if (!isVisible(element, rect) || isDisabled(element)) { + await sleep(25); + continue; + } + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + await nextFrame(); + const first = element.getBoundingClientRect(); + await nextFrame(); + const second = element.getBoundingClientRect(); + if (!sameRect(first, second)) continue; + const x = Math.min(Math.max(second.left + second.width / 2, 0), Math.max(window.innerWidth - 1, 0)); + const y = Math.min(Math.max(second.top + second.height / 2, 0), Math.max(window.innerHeight - 1, 0)); + const hit = document.elementFromPoint(x, y); + if (hit && (hit === element || element.contains(hit))) return { ok: true, x, y }; + await sleep(25); + } + return { ok: false, reason: 'timeout' }; + })()`, + true, + ); + if (!result || result.ok !== true) { + fail(`automation ref ${ref} was not actionable: ${JSON.stringify(result)}`); + } + return { x: result.x, y: result.y }; +} + +async function automationClick(guest, refEntry, options = {}) { + const send = await attachAutomationDebugger(guest); + const point = await automationRefPoint(guest, refEntry.ref, refEntry.fingerprint); + const button = options.button || "left"; + const modifiers = automationModifierMask(options.modifiers || []); + const clickCount = options.doubleClick ? 2 : 1; + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + button: "none", + modifiers, + }); + await send("Input.dispatchMouseEvent", { + type: "mousePressed", + x: point.x, + y: point.y, + button, + buttons: automationButtonMask(button), + clickCount, + modifiers, + }); + await send("Input.dispatchMouseEvent", { + type: "mouseReleased", + x: point.x, + y: point.y, + button, + buttons: 0, + clickCount, + modifiers, + }); + return point; +} + +async function automationHover(guest, refEntry) { + const send = await attachAutomationDebugger(guest); + const point = await automationRefPoint(guest, refEntry.ref, refEntry.fingerprint); + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + button: "none", + }); +} + +async function automationScroll(guest, deltaX, deltaY) { + const send = await attachAutomationDebugger(guest); + const point = await guest.executeJavaScript( + "({ x: Math.max(0, (window.innerWidth || 1) / 2), y: Math.max(0, (window.innerHeight || 1) / 2) })", + true, + ); + await send("Input.dispatchMouseEvent", { + type: "mouseWheel", + x: point.x, + y: point.y, + deltaX, + deltaY, + }); +} + +async function automationDrag(guest, sourceRef, targetRef) { + const send = await attachAutomationDebugger(guest); + const source = await automationRefPoint(guest, sourceRef.ref, sourceRef.fingerprint); + const target = await automationRefPoint(guest, targetRef.ref, targetRef.fingerprint); + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: source.x, + y: source.y, + button: "none", + }); + await send("Input.dispatchMouseEvent", { + type: "mousePressed", + x: source.x, + y: source.y, + button: "left", + buttons: 1, + clickCount: 1, + }); + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: target.x, + y: target.y, + button: "left", + buttons: 1, + }); + await send("Input.dispatchMouseEvent", { + type: "mouseReleased", + x: target.x, + y: target.y, + button: "left", + buttons: 0, + clickCount: 1, + }); +} + +async function automationType(guest, refEntry, text) { + const send = await attachAutomationDebugger(guest); + await automationClick(guest, refEntry); + await send("Input.insertText", { text }); +} + +async function automationEvaluate(guest, functionSource, refEntry) { + return guest.executeJavaScript( + String.raw`(async () => { + const userFunction = (0, eval)(${JSON.stringify(`(${functionSource})`)}); + const args = []; + ${ + refEntry + ? `const resolved = window.__PASEO_BROWSER_AUTOMATION__.resolve(${JSON.stringify(refEntry.ref)}, ${JSON.stringify(refEntry.fingerprint)}); + if (!resolved.ok) return { ok: false, reason: "stale_ref" }; + args.push(resolved.element);` + : "" + } + return JSON.stringify(await userFunction(...args)); + })()`, + true, + ); +} + +const AUTOMATION_DIALOG_POLICY = { + alert: { action: "accepted", accept: true }, + confirm: { action: "dismissed", accept: false }, + prompt: { action: "dismissed", accept: false }, + beforeunload: { action: "dismissed", accept: false }, +}; + +const AUTOMATION_PROMPT_SHIM_INSTALL = String.raw`(() => { + const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__"; + const state = window[stateKey] || { prompts: [], installed: false }; + window[stateKey] = state; + if (state.installed) return true; + window.prompt = (message = "", defaultValue = "") => { + state.prompts.push({ + type: "prompt", + message: String(message ?? ""), + defaultValue: String(defaultValue ?? ""), + action: "dismissed", + timestamp: Date.now(), + }); + return null; + }; + state.installed = true; + return true; +})()`; + +const AUTOMATION_PROMPT_SHIM_DRAIN = String.raw`(() => { + const state = window.__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__; + if (!state || !Array.isArray(state.prompts)) return []; + return state.prompts.splice(0); +})()`; + +async function captureAutomationDialogs(guest, action, expectedCount) { + const send = await attachAutomationDebugger(guest); + await send("Page.enable"); + await send("Runtime.evaluate", { + expression: AUTOMATION_PROMPT_SHIM_INSTALL, + returnByValue: true, + }); + const dialogs = []; + const listener = (_event, method, params = {}) => { + if (method !== "Page.javascriptDialogOpening") return; + const type = AUTOMATION_DIALOG_POLICY[params.type] ? params.type : "alert"; + const policy = AUTOMATION_DIALOG_POLICY[type]; + dialogs.push({ + type, + message: String(params.message || ""), + ...(typeof params.defaultPrompt === "string" ? { defaultValue: params.defaultPrompt } : {}), + action: policy.action, + timestamp: Date.now(), + }); + void send("Page.handleJavaScriptDialog", { accept: policy.accept }); + }; + guest.debugger.on("message", listener); + try { + await action(); + const promptResult = await send("Runtime.evaluate", { + expression: AUTOMATION_PROMPT_SHIM_DRAIN, + returnByValue: true, + }); + if (Array.isArray(promptResult.result?.value)) { + dialogs.push(...promptResult.result.value); + } + await waitForDialogCount(dialogs, expectedCount); + return dialogs; + } finally { + guest.debugger.removeListener?.("message", listener); + } +} + +async function waitForDialogCount(dialogs, expectedCount) { + const deadline = Date.now() + 1000; + do { + if (dialogs.length >= expectedCount) return; + await delay(25); + } while (Date.now() < deadline); + fail(`automation observed ${dialogs.length}/${expectedCount} dialogs`); +} + +function assertAutomationDialog(dialogs, expected) { + const dialog = dialogs.find((entry) => entry.type === expected.type); + if (!dialog) { + fail(`automation did not report ${expected.type} dialog: ${JSON.stringify(dialogs)}`); + } + for (const [key, value] of Object.entries(expected)) { + if (dialog[key] !== value) { + fail(`automation ${expected.type} dialog ${key}=${JSON.stringify(dialog[key])}`); + } + } +} + +function automationButtonMask(button) { + if (button === "right") return 2; + if (button === "middle") return 4; + return 1; +} + +function automationModifierMask(modifiers) { + const masks = { Alt: 1, Control: 2, Meta: 4, Shift: 8 }; + return modifiers.reduce((mask, modifier) => mask | masks[modifier], 0); +} + +function automationRefByName(snapshot, name) { + const entry = snapshot.refs.find((ref) => ref.fingerprint.name === name); + if (!entry) { + fail(`automation ref missing for ${name}`); + } + return entry; +} + +async function automationLog(guest) { + return guest.executeJavaScript("window.fixtureLog", true); +} + +async function waitForAutomationLog(guest, predicate, label) { + const deadline = Date.now() + 1000; + do { + const log = await automationLog(guest); + if (Array.isArray(log) && log.some(predicate)) { + return log; + } + await delay(25); + } while (Date.now() < deadline); + fail(`automation log never observed ${label}`); +} + +async function runAutomationGroup() { + const results = []; + const handle = createInactiveHarnessWindow({ + width: 1000, + height: 700, + backgroundColor: "#202020", + webPreferences: { + webviewTag: true, + contextIsolation: true, + nodeIntegration: false, + sandbox: false, + }, + }); + const { win } = handle; + installHarnessWebviewGuards(win); + const tracker = trackAttachedGuests(win, { disableGuestBackgroundThrottlingAtAttach: true }); + try { + await withTimeout( + win.loadFile(path.join(ROOT, "index.html"), { + query: { + webviewCount: "0", + permanentParkingState: "p1-overflow-1x1", + targetUrl: automationFixtureUrl(), + }, + }), + "automation harness window loadFile", + ); + await waitForInactiveReveal(handle, "automation harness window"); + const { guest } = await appendPermanentWebview({ + win, + tracker, + state: { id: "p1-overflow-1x1" }, + sourceUrl: automationFixtureUrl(), + }); + await waitForGuestLoad(guest); + + const first = await guest.executeJavaScript(AUTOMATION_SNAPSHOT_PROBE, true); + assertAutomationSnapshot(first); + pass("automation snapshot renders headings static text controls and refs"); + results.push({ group: "automation", check: "snapshot", pass: true }); + + const saveRef = first.refs.find((ref) => ref.fingerprint.name === "Save changes"); + if (!saveRef) { + fail("automation save ref missing"); + } + await guest.executeJavaScript("window.pushFixtureState()", true); + const pushStateResult = await guest.executeJavaScript( + `window.__PASEO_BROWSER_AUTOMATION__.resolve(${JSON.stringify(saveRef.ref)}, ${JSON.stringify(saveRef.fingerprint)}).ok`, + true, + ); + if (pushStateResult !== true) { + fail("automation ref did not survive pushState"); + } + pass("automation ref resolves after pushState"); + results.push({ group: "automation", check: "pushState-ref", pass: true }); + + const pageEvaluate = await automationEvaluate(guest, "() => ({ title: document.title })"); + const refEvaluate = await automationEvaluate( + guest, + "(element) => ({ text: element.textContent.trim(), tag: element.tagName.toLowerCase() })", + saveRef, + ); + if ( + pageEvaluate !== '{"title":"Automation Fixture"}' || + refEvaluate !== '{"text":"Save changes","tag":"button"}' + ) { + fail(`automation evaluate returned page=${pageEvaluate} ref=${refEvaluate}`); + } + pass("automation evaluate runs in page context and receives ref element"); + results.push({ group: "automation", check: "evaluate", pass: true }); + + await guest.executeJavaScript("window.scrollTo(0, 0)", true); + await automationScroll(guest, 0, 500); + await delay(100); + const scrollY = await guest.executeJavaScript("window.scrollY", true); + if (!(scrollY > 0)) { + fail(`automation scroll did not change window.scrollY: ${scrollY}`); + } + pass("automation scroll changes the viewport scroll position"); + results.push({ group: "automation", check: "scroll", pass: true }); + + const nameRef = automationRefByName(first, "Name"); + await automationType(guest, nameRef, " Ada"); + await automationClick(guest, saveRef); + await waitForAutomationLog( + guest, + (entry) => entry.event === "input-name" && entry.trusted === true, + "trusted input event", + ); + await waitForAutomationLog( + guest, + (entry) => entry.event === "click-save" && entry.trusted === true, + "trusted click event", + ); + pass("automation trusted click and text input events reach the page"); + results.push({ group: "automation", check: "trusted-input", pass: true }); + + const hoverRef = automationRefByName(first, "Reveal actions"); + await automationHover(guest, hoverRef); + const hoverVisible = await guest.executeJavaScript( + "getComputedStyle(document.getElementById('hover-target')).display !== 'none'", + true, + ); + if (hoverVisible !== true) { + fail("automation hover did not reveal the hover target"); + } + pass("automation hover reveals CSS :hover content"); + results.push({ group: "automation", check: "hover-reveal", pass: true }); + + const delayedRef = automationRefByName(first, "Delayed save"); + await automationClick(guest, delayedRef); + const delayedLog = await automationLog(guest); + if (!delayedLog.some((entry) => entry.event === "click-delayed" && entry.trusted === true)) { + fail("automation delayed enabled button did not receive trusted click"); + } + pass("automation action waits for delayed enabled state"); + results.push({ group: "automation", check: "delayed-enable", pass: true }); + + const movingRef = automationRefByName(first, "Moving target"); + await automationClick(guest, movingRef); + const movingLog = await automationLog(guest); + if (!movingLog.some((entry) => entry.event === "click-moving" && entry.trusted === true)) { + fail("automation moving button did not receive trusted click"); + } + pass("automation action waits for moving element stabilization"); + results.push({ group: "automation", check: "moving-stable", pass: true }); + + const dragSourceRef = automationRefByName(first, "Drag source"); + const dragTargetRef = automationRefByName(first, "Drop target"); + await automationDrag(guest, dragSourceRef, dragTargetRef); + const dragLog = await automationLog(guest); + if ( + !dragLog.some((entry) => entry.event === "drag-down" && entry.trusted === true) || + !dragLog.some((entry) => entry.event === "drag-up" && entry.trusted === true) + ) { + fail("automation pointer drag did not produce trusted pointer events"); + } + pass("automation pointer-event drag reaches source and target"); + results.push({ group: "automation", check: "pointer-drag", pass: true }); + + await automationClick(guest, saveRef, { + button: "right", + doubleClick: true, + modifiers: ["Control", "Shift"], + }); + const optionsLog = await automationLog(guest); + if ( + !optionsLog.some( + (entry) => + entry.event === "down-save" && + entry.trusted === true && + entry.button === 2 && + entry.detail === 2 && + entry.ctrlKey === true && + entry.shiftKey === true, + ) + ) { + fail("automation click options did not affect button/modifiers/double-click count"); + } + pass("automation click options affect button modifiers and double-click count"); + results.push({ group: "automation", check: "click-options", pass: true }); + + const alertRef = automationRefByName(first, "Open alert"); + const alertDialogs = await captureAutomationDialogs( + guest, + () => automationClick(guest, alertRef), + 1, + ); + assertAutomationDialog(alertDialogs, { + type: "alert", + message: "Alert opened", + action: "accepted", + }); + await waitForAutomationLog(guest, (entry) => entry.event === "alert-returned", "alert return"); + pass("automation alert dialogs are accepted and reported"); + results.push({ group: "automation", check: "dialog-alert", pass: true }); + + const confirmRef = automationRefByName(first, "Open confirm"); + const confirmDialogs = await captureAutomationDialogs( + guest, + () => automationClick(guest, confirmRef), + 1, + ); + assertAutomationDialog(confirmDialogs, { + type: "confirm", + message: "Confirm action?", + action: "dismissed", + }); + await waitForAutomationLog( + guest, + (entry) => entry.event === "confirm-result" && entry.value === false, + "confirm dismissal", + ); + pass("automation confirm dialogs are dismissed and reported"); + results.push({ group: "automation", check: "dialog-confirm", pass: true }); + + const promptRef = automationRefByName(first, "Open prompt"); + const promptDialogs = await captureAutomationDialogs( + guest, + () => automationClick(guest, promptRef), + 1, + ); + assertAutomationDialog(promptDialogs, { + type: "prompt", + message: "Prompt value?", + defaultValue: "Maya", + action: "dismissed", + }); + await waitForAutomationLog( + guest, + (entry) => entry.event === "prompt-result" && entry.value === null, + "prompt dismissal", + ); + pass("automation prompt dialogs are dismissed and reported"); + results.push({ group: "automation", check: "dialog-prompt", pass: true }); + + const beforeUnloadRef = automationRefByName(first, "Arm beforeunload"); + await automationClick(guest, beforeUnloadRef); + const beforeUrl = guest.getURL(); + const beforeUnloadDialogs = await captureAutomationDialogs( + guest, + async () => { + await guest.loadURL(automationFixtureUrl()).catch(() => {}); + }, + 1, + ); + assertAutomationDialog(beforeUnloadDialogs, { + type: "beforeunload", + action: "dismissed", + }); + if (guest.getURL() !== beforeUrl) { + fail("automation beforeunload dismissal did not cancel navigation"); + } + pass("automation beforeunload dialogs are dismissed and reported"); + results.push({ group: "automation", check: "dialog-beforeunload", pass: true }); + + await guest.executeJavaScript("window.sameUrlRerender()", true); + const rerenderResult = await guest.executeJavaScript( + `window.__PASEO_BROWSER_AUTOMATION__.resolve(${JSON.stringify(saveRef.ref)}, ${JSON.stringify(saveRef.fingerprint)}).reason`, + true, + ); + if (rerenderResult !== "stale_ref") { + fail(`automation same-url rerender returned ${rerenderResult}`); + } + pass("automation same-url rerender stales old ref"); + results.push({ group: "automation", check: "same-url-stale-ref", pass: true }); + + const second = await guest.executeJavaScript(AUTOMATION_SNAPSHOT_PROBE, true); + const uploadRef = second.refs.find((ref) => ref.fingerprint.name === "Upload receipt"); + if (!uploadRef) { + fail("automation upload ref missing"); + } + if (!guest.debugger.isAttached()) { + guest.debugger.attach("1.3"); + } + const evaluated = await guest.debugger.sendCommand("Runtime.evaluate", { + expression: `(() => window.__PASEO_BROWSER_AUTOMATION__.resolve(${JSON.stringify(uploadRef.ref)}, ${JSON.stringify(uploadRef.fingerprint)}).element)()`, + objectGroup: "paseo-browser-automation", + returnByValue: false, + }); + const described = await guest.debugger.sendCommand("DOM.describeNode", { + objectId: evaluated.result.objectId, + }); + if (!described.node || typeof described.node.backendNodeId !== "number") { + fail("automation upload ref did not resolve to backendNodeId"); + } + pass("automation upload ref resolves to backendNodeId"); + results.push({ group: "automation", check: "upload-backend-node", pass: true }); + + // Resize is not harness-testable: the harness hosts webviews in the parked + // 1px resident host, and Electron does not propagate CSS-box resizes to a + // parked guest's capture surface (see docs/browser-capture-harness.md). + // The production resize path is app-owned webview sizing, covered by + // packages/app/src/browser-automation/handler.test.ts. + + await fsp.writeFile( + path.join(OUT_DIR, "automation-results.json"), + `${JSON.stringify({ generatedAt: new Date().toISOString(), results }, null, 2)}\n`, + ); + return results; + } finally { + await closeHarnessWindow(win); + } +} + +function assertAutomationSnapshot(snapshot) { + const text = snapshot && snapshot.snapshot; + if (typeof text !== "string") { + fail("automation snapshot returned no text"); + } + for (const expected of [ + 'heading "Settings"', + 'text: "Connected as Maya"', + 'textbox "Name" [ref=@e1]', + 'link "Read docs"', + 'button "Save changes"', + 'button "Upload receipt"', + ]) { + if (!text.includes(expected)) { + fail(`automation snapshot missing ${expected}`); + } + } + if (text.includes("selector")) { + fail("automation snapshot exposed selector text"); + } +} + async function main() { ensureDirSync(OUT_DIR); - if (!["all", "existing", "permanent-parking"].includes(HARNESS_GROUP)) { + if (!["all", "existing", "permanent-parking", "automation"].includes(HARNESS_GROUP)) { fail(`unknown harness group ${HARNESS_GROUP}`); } + if (HARNESS_GROUP === "automation") { + const automationResults = await runAutomationGroup(); + await fsp.writeFile( + path.join(OUT_DIR, "results.json"), + `${JSON.stringify( + { + generatedAt: new Date().toISOString(), + automationResults, + }, + null, + 2, + )}\n`, + ); + pass(`capture harness automation complete output=${OUT_DIR}`); + return; + } + if (HARNESS_GROUP === "permanent-parking") { const permanentParkingResults = await runPermanentParkingGroup(); await fsp.writeFile( diff --git a/packages/desktop/src/features/browser-automation/actionability.ts b/packages/desktop/src/features/browser-automation/actionability.ts new file mode 100644 index 000000000..b5e48db76 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/actionability.ts @@ -0,0 +1,198 @@ +import type { SnapshotPage } from "./snapshot-engine.js"; + +export interface ActionablePoint { + x: number; + y: number; +} + +export interface ActionableTarget { + point: ActionablePoint; + rect: { + x: number; + y: number; + width: number; + height: number; + }; +} + +export type ActionabilityResult = + | { ok: true; target: ActionableTarget } + | { ok: false; reason: "stale_ref" | "timeout"; detail?: string }; + +const DEFAULT_ACTIONABILITY_TIMEOUT_MS = 5_000; + +export async function waitForActionableTarget(input: { + page: SnapshotPage; + elementExpression: string; + editable?: boolean; + timeoutMs?: number; +}): Promise { + const result = await input.page.executeJavaScript( + buildActionabilityScript({ + elementExpression: input.elementExpression, + editable: input.editable === true, + timeoutMs: input.timeoutMs ?? DEFAULT_ACTIONABILITY_TIMEOUT_MS, + }), + ); + return readActionabilityResult(result); +} + +function readActionabilityResult(value: unknown): ActionabilityResult { + if (!value || typeof value !== "object") { + return { ok: false, reason: "timeout" }; + } + const record = value as Record; + if (record.ok === true && isActionableTarget(record.target)) { + return { ok: true, target: record.target }; + } + if (record.ok === false) { + const reason = record.reason; + if (reason === "stale_ref" || reason === "timeout") { + return { + ok: false, + reason, + ...(typeof record.detail === "string" ? { detail: record.detail } : {}), + }; + } + } + return { ok: false, reason: "timeout" }; +} + +function isActionableTarget(value: unknown): value is ActionableTarget { + if (!value || typeof value !== "object") { + return false; + } + const record = value as Record; + return isPoint(record.point) && isRect(record.rect); +} + +function isPoint(value: unknown): value is ActionablePoint { + if (!value || typeof value !== "object") { + return false; + } + const record = value as Record; + return isFiniteNumber(record.x) && isFiniteNumber(record.y); +} + +function isRect(value: unknown): value is ActionableTarget["rect"] { + if (!value || typeof value !== "object") { + return false; + } + const record = value as Record; + return ( + isFiniteNumber(record.x) && + isFiniteNumber(record.y) && + isFiniteNumber(record.width) && + isFiniteNumber(record.height) + ); +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function buildActionabilityScript(input: { + elementExpression: string; + editable: boolean; + timeoutMs: number; +}): string { + return String.raw`(async () => { + const deadline = performance.now() + ${JSON.stringify(input.timeoutMs)}; + const requiresEditable = ${JSON.stringify(input.editable)}; + + const nextFrame = () => new Promise((resolve) => requestAnimationFrame(resolve)); + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const nearlyEqual = (a, b) => Math.abs(a - b) < 0.25; + const sameRect = (a, b) => + nearlyEqual(a.x, b.x) && + nearlyEqual(a.y, b.y) && + nearlyEqual(a.width, b.width) && + nearlyEqual(a.height, b.height); + const rectPayload = (rect) => ({ + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + }); + const centerPoint = (rect) => ({ + x: Math.min(Math.max(rect.left + rect.width / 2, 0), Math.max(window.innerWidth - 1, 0)), + y: Math.min(Math.max(rect.top + rect.height / 2, 0), Math.max(window.innerHeight - 1, 0)), + }); + const isDisabled = (element) => { + if (element.closest?.('[aria-disabled="true"]')) return true; + if ('disabled' in element && element.disabled) return true; + const fieldset = element.closest?.('fieldset[disabled]'); + return Boolean(fieldset); + }; + const isEditable = (element) => { + if (element.isContentEditable) return true; + const tag = element.tagName?.toLowerCase(); + if (tag === 'textarea' || tag === 'select') return !element.readOnly && !isDisabled(element); + if (tag !== 'input') return false; + const type = (element.getAttribute('type') || 'text').toLowerCase(); + if (['button', 'checkbox', 'color', 'file', 'hidden', 'image', 'radio', 'range', 'reset', 'submit'].includes(type)) return false; + return !element.readOnly && !isDisabled(element); + }; + const isVisible = (element, rect) => { + const style = getComputedStyle(element); + return ( + rect.width > 0 && + rect.height > 0 && + style.visibility !== 'hidden' && + style.display !== 'none' && + Number(style.opacity || '1') !== 0 + ); + }; + const hitTargetReceivesEvents = (element, point) => { + const hit = document.elementFromPoint(point.x, point.y); + return Boolean(hit && (hit === element || element.contains(hit))); + }; + const resolveElement = () => (${input.elementExpression}); + + let detail = 'not actionable'; + while (performance.now() <= deadline) { + const element = resolveElement(); + if (!element || !element.isConnected) { + return { ok: false, reason: 'stale_ref', detail: 'ref no longer resolves' }; + } + + const rect = element.getBoundingClientRect(); + if (!isVisible(element, rect)) { + detail = 'not visible'; + await sleep(25); + continue; + } + if (isDisabled(element)) { + detail = 'disabled'; + await sleep(25); + continue; + } + if (requiresEditable && !isEditable(element)) { + detail = 'not editable'; + await sleep(25); + continue; + } + + element.scrollIntoView?.({ block: 'center', inline: 'center' }); + await nextFrame(); + const firstRect = element.getBoundingClientRect(); + await nextFrame(); + const secondRect = element.getBoundingClientRect(); + if (!sameRect(firstRect, secondRect)) { + detail = 'moving'; + continue; + } + + const point = centerPoint(secondRect); + if (!hitTargetReceivesEvents(element, point)) { + detail = 'covered'; + await sleep(25); + continue; + } + + return { ok: true, target: { point, rect: rectPayload(secondRect) } }; + } + + return { ok: false, reason: 'timeout', detail }; + })()`; +} diff --git a/packages/desktop/src/features/browser-automation/aria-snapshot-script.ts b/packages/desktop/src/features/browser-automation/aria-snapshot-script.ts new file mode 100644 index 000000000..1256bbba2 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/aria-snapshot-script.ts @@ -0,0 +1,311 @@ +export const ARIA_SNAPSHOT_SCRIPT_MARKER = "__PASEO_ARIA_SNAPSHOT__"; + +// Adapted from Playwright's injected ARIA snapshot model. +// Copyright (c) Microsoft Corporation. Licensed under the Apache License, Version 2.0. +export const ARIA_SNAPSHOT_SCRIPT = String.raw`(() => { + const MARKER = ${JSON.stringify(ARIA_SNAPSHOT_SCRIPT_MARKER)}; + const MAX_NODES = 1500; + const MAX_REFS = 500; + const MAX_TEXT_LENGTH = 80000; + const ACTIONABLE_ROLES = new Set([ + 'button', + 'checkbox', + 'combobox', + 'link', + 'menuitem', + 'option', + 'radio', + 'searchbox', + 'slider', + 'spinbutton', + 'switch', + 'tab', + 'textbox', + 'treeitem' + ]); + + function normalizeText(value) { + return String(value || '').replace(/[\u200b\u00ad]/g, '').replace(/[\r\n\s\t]+/g, ' ').trim(); + } + + function visibilityFor(element) { + if (!(element instanceof Element)) return false; + const style = window.getComputedStyle(element); + if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) return false; + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0 ? 'box' : 'boxless'; + } + + function explicitRole(element) { + const role = element.getAttribute('role'); + if (!role || role === 'presentation' || role === 'none') return null; + return role.split(/\s+/)[0].toLowerCase(); + } + + function implicitRole(element) { + const tag = element.tagName.toLowerCase(); + if (/^h[1-6]$/.test(tag)) return 'heading'; + if (tag === 'a' && element.hasAttribute('href')) return 'link'; + if (tag === 'button') return 'button'; + if (tag === 'select') return 'combobox'; + if (tag === 'textarea') return 'textbox'; + if (element instanceof HTMLElement && element.isContentEditable) return 'textbox'; + if (tag === 'summary') return 'button'; + if (tag === 'main') return 'main'; + if (tag === 'nav') return 'navigation'; + if (tag === 'header') return 'banner'; + if (tag === 'footer') return 'contentinfo'; + if (tag === 'section') return element.getAttribute('aria-label') ? 'region' : null; + if (tag === 'ul' || tag === 'ol') return 'list'; + if (tag === 'li') return 'listitem'; + if (tag === 'table') return 'table'; + if (tag === 'tr') return 'row'; + if (tag === 'th') return 'columnheader'; + if (tag === 'td') return 'cell'; + if (tag === 'iframe') return 'iframe'; + if (tag === 'input') { + const type = (element.getAttribute('type') || 'text').toLowerCase(); + if (type === 'checkbox') return 'checkbox'; + if (type === 'radio') return 'radio'; + if (type === 'button' || type === 'submit' || type === 'reset') return 'button'; + if (type === 'range') return 'slider'; + if (type === 'number') return 'spinbutton'; + if (type === 'search') return 'searchbox'; + if (type === 'hidden') return null; + return 'textbox'; + } + return null; + } + + function roleFor(element) { + return explicitRole(element) || implicitRole(element); + } + + function labelText(element) { + if (!(element instanceof HTMLElement)) return ''; + if (element.id) { + const escapedId = window.CSS && typeof window.CSS.escape === 'function' + ? window.CSS.escape(element.id) + : String(element.id).replace(/"/g, '\\"'); + const label = document.querySelector('label[for="' + escapedId + '"]'); + if (label) return normalizeText(label.textContent); + } + const closestLabel = element.closest('label'); + return closestLabel ? normalizeText(closestLabel.textContent) : ''; + } + + function nameFor(element, role) { + const tag = element.tagName.toLowerCase(); + const labelledBy = element.getAttribute('aria-labelledby'); + if (labelledBy) { + const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.textContent || '').join(' '); + const normalized = normalizeText(text); + if (normalized) return normalized; + } + const pieces = [ + element.getAttribute('aria-label'), + labelText(element), + element.getAttribute('alt'), + element.getAttribute('title'), + tag === 'input' || tag === 'textarea' ? element.getAttribute('placeholder') : null, + tag === 'input' || tag === 'textarea' ? element.value : null, + role === 'button' || role === 'link' || role === 'heading' || ACTIONABLE_ROLES.has(role) + ? element.textContent + : null + ]; + return normalizeText(pieces.find((piece) => normalizeText(piece).length > 0) || ''); + } + + function fingerprintNameFor(element, role, name) { + const tag = element.tagName.toLowerCase(); + if (tag !== 'input' && tag !== 'textarea') return name; + const mutableValue = normalizeText(element.value); + if (!mutableValue || name !== mutableValue) return name; + return ''; + } + + function inheritedDisabled(element) { + if (!(element instanceof Element)) return false; + if (element.hasAttribute('disabled') || element.getAttribute('aria-disabled') === 'true') return true; + if (element.closest('fieldset[disabled]')) return true; + return element.closest('[aria-disabled="true"]') !== null; + } + + function isActionable(element, role) { + if (!role || !ACTIONABLE_ROLES.has(role)) return false; + if (visibilityFor(element) !== 'box') return false; + if (inheritedDisabled(element)) return false; + return true; + } + + function fingerprintFor(element, role, name) { + return { + role, + name: fingerprintNameFor(element, role, name), + tagName: element.tagName.toLowerCase(), + type: element.getAttribute('type') || '', + ariaLabel: element.getAttribute('aria-label') || '' + }; + } + + function fingerprintMatches(element, fingerprint) { + const role = roleFor(element) || 'generic'; + const name = nameFor(element, role); + const current = fingerprintFor(element, role, name); + return current.role === fingerprint.role && + current.name === fingerprint.name && + current.tagName === fingerprint.tagName && + current.type === fingerprint.type && + current.ariaLabel === fingerprint.ariaLabel; + } + + function ensureRuntime() { + const runtime = { + refs: new Map(), + resolve(ref, fingerprint) { + const element = this.refs.get(ref); + if (!element || !element.isConnected || !fingerprintMatches(element, fingerprint)) { + return { ok: false, reason: 'stale_ref' }; + } + return { ok: true, element }; + } + }; + Object.defineProperty(window, '__PASEO_BROWSER_AUTOMATION__', { + configurable: true, + enumerable: false, + value: runtime + }); + return runtime; + } + + function stateAttributes(element, role) { + const attrs = []; + if (role === 'heading') { + const tag = element.tagName.toLowerCase(); + const level = /^h[1-6]$/.test(tag) ? Number(tag.slice(1)) : Number(element.getAttribute('aria-level') || 0); + if (level) attrs.push('level=' + level); + } + if (element.matches?.('input[type="checkbox"], input[type="radio"]')) { + attrs.push('checked=' + (element.checked ? 'true' : 'false')); + } + if (element.getAttribute('aria-checked')) attrs.push('checked=' + element.getAttribute('aria-checked')); + if (element.getAttribute('aria-expanded')) attrs.push('expanded=' + element.getAttribute('aria-expanded')); + if (element.getAttribute('aria-pressed')) attrs.push('pressed=' + element.getAttribute('aria-pressed')); + if (element.getAttribute('aria-selected')) attrs.push('selected=' + element.getAttribute('aria-selected')); + return attrs; + } + + function textNode(text) { + return { kind: 'text', text }; + } + + function elementNode(element, role, name) { + return { + kind: 'role', + role: role || 'generic', + name, + tagName: element.tagName.toLowerCase(), + attributes: stateAttributes(element, role), + children: [] + }; + } + + let nodeCount = 0; + let refCount = 0; + let iframeCount = 0; + let maxDepth = 0; + let truncated = false; + let textBudget = MAX_TEXT_LENGTH; + const runtime = ensureRuntime(); + + function countNode(depth) { + if (nodeCount >= MAX_NODES) { + truncated = true; + return false; + } + nodeCount += 1; + maxDepth = Math.max(maxDepth, depth); + return true; + } + + function cappedText(text) { + if (text.length <= textBudget) { + textBudget -= text.length; + return text; + } + truncated = true; + const capped = text.slice(0, Math.max(0, textBudget)); + textBudget = 0; + return capped; + } + + function visitNode(domNode, depth) { + if (!countNode(depth)) return null; + if (domNode.nodeType === Node.TEXT_NODE) { + const text = cappedText(normalizeText(domNode.textContent)); + if (!text) return null; + return textNode(text); + } + if (!(domNode instanceof Element)) return null; + const visibility = visibilityFor(domNode); + if (!visibility) return null; + if (domNode.getAttribute('aria-hidden') === 'true') return null; + + const role = roleFor(domNode); + const name = role ? nameFor(domNode, role) : ''; + const children = []; + for (const child of Array.from(domNode.childNodes)) { + const childSnapshot = visitNode(child, depth + 1); + if (childSnapshot) children.push(childSnapshot); + if (truncated) break; + } + + if (domNode.tagName.toLowerCase() === 'iframe') { + iframeCount += 1; + } + + if (visibility === 'boxless') { + return children.length > 0 ? { kind: 'group', children } : null; + } + if (!role && children.length === 0) return null; + const snapshotNode = role ? elementNode(domNode, role, name) : { kind: 'group', children: [] }; + snapshotNode.children = children; + if (role && isActionable(domNode, role) && refCount < MAX_REFS) { + const ref = '@e' + (refCount + 1); + const fingerprint = fingerprintFor(domNode, role, name); + refCount += 1; + runtime.refs.set(ref, domNode); + snapshotNode.ref = ref; + snapshotNode.fingerprint = fingerprint; + } else if (role && isActionable(domNode, role)) { + truncated = true; + } + return snapshotNode; + } + + const root = { + kind: 'role', + role: 'document', + name: normalizeText(document.title), + tagName: 'document', + attributes: [], + children: [] + }; + for (const child of Array.from(document.body ? document.body.childNodes : document.documentElement.childNodes)) { + const childSnapshot = visitNode(child, 1); + if (childSnapshot) root.children.push(childSnapshot); + if (truncated) break; + } + + return JSON.stringify({ + marker: MARKER, + root, + refs: Array.from(runtime.refs.entries()).map(([ref, element]) => { + const role = roleFor(element) || 'generic'; + const name = nameFor(element, role); + return { ref, fingerprint: fingerprintFor(element, role, name) }; + }), + truncated, + stats: { nodeCount, refCount, textLength: 0, iframeCount, maxDepth } + }); +})()`; diff --git a/packages/desktop/src/features/browser-automation/cdp-session-queue.ts b/packages/desktop/src/features/browser-automation/cdp-session-queue.ts new file mode 100644 index 000000000..7e7167ec2 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/cdp-session-queue.ts @@ -0,0 +1,28 @@ +export type CdpCommandSender = ( + command: string, + params?: Record, +) => Promise; + +export class CdpSessionQueue { + private queue: Promise = Promise.resolve(); + + public async run(task: () => Promise): Promise { + const previous = this.queue; + let releaseCurrent = () => {}; + const current = new Promise((resolve) => { + releaseCurrent = resolve; + }); + const tail = previous.catch(() => {}).then(() => current); + this.queue = tail; + + await previous.catch(() => {}); + try { + return await task(); + } finally { + releaseCurrent(); + if (this.queue === tail) { + this.queue = Promise.resolve(); + } + } + } +} diff --git a/packages/desktop/src/features/browser-automation/dialog-handling.ts b/packages/desktop/src/features/browser-automation/dialog-handling.ts new file mode 100644 index 000000000..d23b8f156 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/dialog-handling.ts @@ -0,0 +1,89 @@ +import type { BrowserAutomationDialogEvent } from "@getpaseo/protocol/browser-automation/rpc-schemas"; + +export const MAX_DIALOGS_PER_COMMAND = 20; + +export interface JavaScriptDialogOpening { + type?: unknown; + message?: unknown; + defaultPrompt?: unknown; +} + +interface DialogPolicy { + readonly action: BrowserAutomationDialogEvent["action"]; + readonly accept: boolean; +} + +const DIALOG_POLICIES: Record = { + alert: { action: "accepted", accept: true }, + confirm: { action: "dismissed", accept: false }, + prompt: { action: "dismissed", accept: false }, + beforeunload: { action: "dismissed", accept: false }, +}; + +export function handledDialogEvent(opening: JavaScriptDialogOpening): BrowserAutomationDialogEvent { + const type = normalizeDialogType(opening.type); + const policy = DIALOG_POLICIES[type]; + return { + type, + message: typeof opening.message === "string" ? opening.message : String(opening.message ?? ""), + ...(typeof opening.defaultPrompt === "string" ? { defaultValue: opening.defaultPrompt } : {}), + action: policy.action, + timestamp: Date.now(), + }; +} + +export function dialogAcceptValue(type: BrowserAutomationDialogEvent["type"]): boolean { + return DIALOG_POLICIES[type].accept; +} + +export function promptShimInstallScript(): string { + return String.raw`(() => { + const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__"; + const state = window[stateKey] || { prompts: [], installed: false }; + window[stateKey] = state; + if (state.installed) return true; + const originalPrompt = window.prompt; + Object.defineProperty(state, "originalPrompt", { configurable: true, value: originalPrompt }); + const promptShim = (message = "", defaultValue = "") => { + state.prompts.push({ + type: "prompt", + message: String(message ?? ""), + defaultValue: String(defaultValue ?? ""), + action: "dismissed", + timestamp: Date.now(), + }); + return null; + }; + Object.defineProperty(state, "promptShim", { configurable: true, value: promptShim }); + window.prompt = promptShim; + state.installed = true; + return true; + })()`; +} + +export function promptShimDrainScript(): string { + return String.raw`(() => { + const state = window.__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__; + if (!state || !Array.isArray(state.prompts)) return []; + return state.prompts.splice(0); + })()`; +} + +export function promptShimRestoreScript(): string { + return String.raw`(() => { + const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__"; + const state = window[stateKey]; + if (!state || !state.installed) return true; + if (window.prompt === state.promptShim && typeof state.originalPrompt === "function") { + window.prompt = state.originalPrompt; + } + delete window[stateKey]; + return true; + })()`; +} + +function normalizeDialogType(value: unknown): BrowserAutomationDialogEvent["type"] { + return value === "alert" || value === "confirm" || value === "prompt" || value === "beforeunload" + ? value + : "alert"; +} diff --git a/packages/desktop/src/features/browser-automation/ipc.test.ts b/packages/desktop/src/features/browser-automation/ipc.test.ts index b3ed24994..e1f0b0a43 100644 --- a/packages/desktop/src/features/browser-automation/ipc.test.ts +++ b/packages/desktop/src/features/browser-automation/ipc.test.ts @@ -1,5 +1,5 @@ import type { Rectangle } from "electron"; -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import type { TabImage } from "./service.js"; import { adaptWebContents } from "./ipc.js"; @@ -16,6 +16,15 @@ class FakeImage implements TabImage { class FakeDebugger { public attachedProtocolVersions: string[] = []; public commands: Array<{ command: string; params: Record }> = []; + public blockCommands = false; + public readonly blockedCommandNames = new Set(); + public readonly failedCommandNames = new Set(); + public readonly promptDialogs: unknown[] = []; + public failPromptDrain = false; + private messageListener: + | ((event: unknown, method: string, params?: Record) => void) + | null = null; + private readonly blockedCommands: Array<() => void> = []; public isAttached(): boolean { return this.attachedProtocolVersions.length > 0; @@ -27,8 +36,48 @@ class FakeDebugger { public async sendCommand(command: string, params?: Record): Promise { this.commands.push({ command, params: params ?? {} }); + if (this.failedCommandNames.has(command)) { + throw new Error(`${command} failed`); + } + if (this.blockCommands || this.blockedCommandNames.has(command)) { + await new Promise((resolve) => { + this.blockedCommands.push(resolve); + }); + } + if (command === "Runtime.evaluate" && typeof params?.expression === "string") { + if (params.expression.includes("state.prompts.splice(0)")) { + if (this.failPromptDrain) { + throw new Error("execution context destroyed"); + } + return { result: { value: this.promptDialogs.splice(0) } }; + } + return { result: { value: true } }; + } return { ok: true }; } + + public on( + event: "message", + listener: (event: unknown, method: string, params?: Record) => void, + ): void { + expect(event).toBe("message"); + this.messageListener = listener; + } + + public emitMessage(method: string, params?: Record): void { + if (!this.messageListener) { + throw new Error("Debugger message listener was not registered"); + } + this.messageListener({}, method, params); + } + + public finishNextCommand(): void { + const resolve = this.blockedCommands.shift(); + if (!resolve) { + throw new Error("No command is blocked"); + } + resolve(); + } } type ConsoleMessageListener = ( @@ -181,4 +230,323 @@ describe("browser automation IPC adapter", () => { { command: "Page.captureScreenshot", params: { format: "png" } }, ]); }); + + test("serializes CDP commands per guest contents", async () => { + const contents = new FakeWebContents(23); + contents.debugger.blockCommands = true; + const tab = adaptWebContents(contents); + + const first = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mouseMoved" }); + const second = tab.sendDebugCommand?.("Page.captureScreenshot", { format: "png" }); + await flushMicrotasks(); + + expect(contents.debugger.commands).toEqual([ + { command: "Input.dispatchMouseEvent", params: { type: "mouseMoved" } }, + ]); + + contents.debugger.finishNextCommand(); + await flushMicrotasks(); + + expect(contents.debugger.commands).toEqual([ + { command: "Input.dispatchMouseEvent", params: { type: "mouseMoved" } }, + { command: "Page.captureScreenshot", params: { format: "png" } }, + ]); + + contents.debugger.finishNextCommand(); + await expect(first).resolves.toEqual({ ok: true }); + await expect(second).resolves.toEqual({ ok: true }); + }); + + test("handles JavaScript dialogs through the per-tab CDP queue", async () => { + const contents = new FakeWebContents(24); + const tab = adaptWebContents(contents); + + const captured = tab.captureDialogs?.(async () => { + contents.debugger.blockCommands = true; + const input = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mouseReleased" }); + await flushMicrotasks(); + + contents.debugger.emitMessage("Page.javascriptDialogOpening", { + type: "confirm", + message: "Delete item?", + }); + await flushMicrotasks(); + + expect(contents.debugger.commands).toEqual([ + { command: "Page.enable", params: {} }, + { + command: "Runtime.evaluate", + params: { expression: expect.any(String), returnByValue: true }, + }, + { command: "Input.dispatchMouseEvent", params: { type: "mouseReleased" } }, + { command: "Page.handleJavaScriptDialog", params: { accept: false } }, + ]); + + contents.debugger.blockCommands = false; + contents.debugger.finishNextCommand(); + await input; + await flushMicrotasks(); + return "done"; + }); + + await expect(captured).resolves.toEqual({ + result: "done", + dialogs: [ + { + type: "confirm", + message: "Delete item?", + action: "dismissed", + timestamp: expect.any(Number), + }, + ], + }); + expect(contents.debugger.commands).toEqual([ + { command: "Page.enable", params: {} }, + { + command: "Runtime.evaluate", + params: { expression: expect.any(String), returnByValue: true }, + }, + { command: "Input.dispatchMouseEvent", params: { type: "mouseReleased" } }, + { command: "Page.handleJavaScriptDialog", params: { accept: false } }, + { + command: "Runtime.evaluate", + params: { expression: expect.any(String), returnByValue: true }, + }, + { + command: "Runtime.evaluate", + params: { expression: expect.any(String), returnByValue: true }, + }, + ]); + }); + + test("handles JavaScript dialogs while the triggering CDP input command is still in flight", async () => { + const contents = new FakeWebContents(25); + const tab = adaptWebContents(contents); + + const captured = tab.captureDialogs?.(async () => { + contents.debugger.blockedCommandNames.add("Input.dispatchMouseEvent"); + const input = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mousePressed" }); + await flushMicrotasks(); + + contents.debugger.emitMessage("Page.javascriptDialogOpening", { + type: "alert", + message: "Saved", + }); + await flushMicrotasks(); + + expect(contents.debugger.commands).toEqual([ + { command: "Page.enable", params: {} }, + { + command: "Runtime.evaluate", + params: { expression: expect.any(String), returnByValue: true }, + }, + { command: "Input.dispatchMouseEvent", params: { type: "mousePressed" } }, + { command: "Page.handleJavaScriptDialog", params: { accept: true } }, + ]); + + contents.debugger.blockedCommandNames.clear(); + contents.debugger.finishNextCommand(); + await input; + return "done"; + }); + + await expect(captured).resolves.toEqual({ + result: "done", + dialogs: [ + { + type: "alert", + message: "Saved", + action: "accepted", + timestamp: expect.any(Number), + }, + ], + }); + expect(contents.debugger.commands.at(-1)).toEqual({ + command: "Runtime.evaluate", + params: { + expression: expect.stringContaining("delete window[stateKey]"), + returnByValue: true, + }, + }); + }); + + test("keeps the prompt shim installed until overlapping captures finish", async () => { + const contents = new FakeWebContents(26); + const tab = adaptWebContents(contents); + const firstStarted = deferred(); + const secondStarted = deferred(); + const finishFirst = deferred(); + const finishSecond = deferred(); + + const first = tab.captureDialogs?.(async () => { + firstStarted.resolve(); + await finishFirst.promise; + return "first"; + }); + await firstStarted.promise; + + const second = tab.captureDialogs?.(async () => { + secondStarted.resolve(); + await finishSecond.promise; + return "second"; + }); + await secondStarted.promise; + + contents.debugger.promptDialogs.push( + { + type: "prompt", + message: "First?", + defaultValue: "one", + action: "dismissed", + timestamp: 1, + }, + { + type: "prompt", + message: "Second?", + defaultValue: "two", + action: "dismissed", + timestamp: 2, + }, + ); + + finishFirst.resolve(); + await expect(first).resolves.toEqual({ + result: "first", + dialogs: [ + { + type: "prompt", + message: "First?", + defaultValue: "one", + action: "dismissed", + timestamp: 1, + }, + { + type: "prompt", + message: "Second?", + defaultValue: "two", + action: "dismissed", + timestamp: 2, + }, + ], + }); + expect( + contents.debugger.commands.some( + (entry) => + entry.command === "Runtime.evaluate" && + typeof entry.params.expression === "string" && + entry.params.expression.includes("delete window[stateKey]"), + ), + ).toBe(false); + + finishSecond.resolve(); + await expect(second).resolves.toEqual({ + result: "second", + dialogs: [ + { + type: "prompt", + message: "First?", + defaultValue: "one", + action: "dismissed", + timestamp: 1, + }, + { + type: "prompt", + message: "Second?", + defaultValue: "two", + action: "dismissed", + timestamp: 2, + }, + ], + }); + expect(contents.debugger.commands.at(-1)).toEqual({ + command: "Runtime.evaluate", + params: { + expression: expect.stringContaining("delete window[stateKey]"), + returnByValue: true, + }, + }); + }); + + test("leaves JavaScript dialogs alone when no capture is active", async () => { + const contents = new FakeWebContents(28); + const tab = adaptWebContents(contents); + + await expect(tab.captureDialogs?.(async () => "done")).resolves.toEqual({ + result: "done", + dialogs: [], + }); + contents.debugger.emitMessage("Page.javascriptDialogOpening", { + type: "confirm", + message: "Unsaved changes?", + }); + await flushMicrotasks(); + + expect(contents.debugger.commands).not.toContainEqual({ + command: "Page.handleJavaScriptDialog", + params: { accept: false }, + }); + }); + + test("treats prompt shim drain failures after navigation as no dialogs", async () => { + const contents = new FakeWebContents(29); + contents.debugger.failPromptDrain = true; + const tab = adaptWebContents(contents); + + await expect(tab.captureDialogs?.(async () => "navigated")).resolves.toEqual({ + result: "navigated", + dialogs: [], + }); + + expect(contents.debugger.commands).toEqual([ + { command: "Page.enable", params: {} }, + { + command: "Runtime.evaluate", + params: { expression: expect.any(String), returnByValue: true }, + }, + { + command: "Runtime.evaluate", + params: { expression: expect.any(String), returnByValue: true }, + }, + { + command: "Runtime.evaluate", + params: { + expression: expect.stringContaining("delete window[stateKey]"), + returnByValue: true, + }, + }, + ]); + }); + + test("runs the command without dialog capture when CDP setup fails", async () => { + const contents = new FakeWebContents(30); + contents.debugger.failedCommandNames.add("Page.enable"); + const tab = adaptWebContents(contents); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + await expect(tab.captureDialogs?.(async () => "done")).resolves.toEqual({ + result: "done", + dialogs: [], + }); + + expect(warn).toHaveBeenCalledWith( + "[browser-automation] Dialog capture unavailable; running command without it", + { contentsId: 30, error: expect.any(Error) }, + ); + warn.mockRestore(); + }); }); + +async function flushMicrotasks(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 00a56b28b..0d7d526e2 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -1,8 +1,20 @@ import type { Rectangle } from "electron"; import { ipcMain } from "electron"; import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; -import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import type { + BrowserAutomationConsoleLogEntry, + BrowserAutomationDialogEvent, +} from "@getpaseo/protocol/browser-automation/rpc-schemas"; import type { TabContents, BrowserRegistry, TabImage } from "./service.js"; +import { CdpSessionQueue } from "./cdp-session-queue.js"; +import { + dialogAcceptValue, + handledDialogEvent, + MAX_DIALOGS_PER_COMMAND, + promptShimDrainScript, + promptShimInstallScript, + promptShimRestoreScript, +} from "./dialog-handling.js"; import { executeAutomationCommand } from "./service.js"; import { listRegisteredPaseoBrowserIds, @@ -14,6 +26,8 @@ import { const MAX_CONSOLE_MESSAGES_PER_TAB = 200; const consoleMessagesByContentsId = new Map(); +const cdpQueuesByContentsId = new Map(); +const dialogMonitorsByContentsId = new Map(); const observedContentsIds = new Set(); interface IpcHandlerRegistry { @@ -24,6 +38,10 @@ interface WebContentsDebugger { isAttached(): boolean; attach(protocolVersion?: string): void; sendCommand(command: string, params?: Record): Promise; + on?( + event: "message", + listener: (event: unknown, method: string, params?: Record) => void, + ): void; } interface ConsoleMessageEmitter { @@ -60,6 +78,8 @@ interface BrowserAutomationWebContents extends ConsoleMessageEmitter { export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents { observeConsoleMessages(contents); + const cdpQueue = getCdpQueue(contents.id); + const dialogMonitor = getDialogMonitor(contents, cdpQueue); return { id: contents.id, getURL: () => contents.getURL(), @@ -76,15 +96,27 @@ export function adaptWebContents(contents: BrowserAutomationWebContents): TabCon capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions), invalidate: () => contents.invalidate(), getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [], - sendDebugCommand: async (command: string, params?: Record) => { - if (!contents.debugger.isAttached()) { - contents.debugger.attach("1.3"); - } - return contents.debugger.sendCommand(command, params ?? {}); - }, + captureDialogs: (task) => dialogMonitor.capture(task), + sendDebugCommand: (command: string, params?: Record) => + cdpQueue.run(async () => { + if (!contents.debugger.isAttached()) { + contents.debugger.attach("1.3"); + } + return contents.debugger.sendCommand(command, params ?? {}); + }), }; } +function getCdpQueue(contentsId: number): CdpSessionQueue { + const existing = cdpQueuesByContentsId.get(contentsId); + if (existing) { + return existing; + } + const queue = new CdpSessionQueue(); + cdpQueuesByContentsId.set(contentsId, queue); + return queue; +} + function observeConsoleMessages(contents: BrowserAutomationWebContents): void { if (observedContentsIds.has(contents.id)) { return; @@ -99,6 +131,192 @@ function observeConsoleMessages(contents: BrowserAutomationWebContents): void { contents.once("destroyed", () => { observedContentsIds.delete(contents.id); consoleMessagesByContentsId.delete(contents.id); + cdpQueuesByContentsId.delete(contents.id); + dialogMonitorsByContentsId.delete(contents.id); + }); +} + +function getDialogMonitor( + contents: BrowserAutomationWebContents, + cdpQueue: CdpSessionQueue, +): DialogMonitor { + const existing = dialogMonitorsByContentsId.get(contents.id); + if (existing) { + return existing; + } + const monitor = new DialogMonitor(contents, cdpQueue); + dialogMonitorsByContentsId.set(contents.id, monitor); + return monitor; +} + +class DialogMonitor { + private enabled = false; + private listenerRegistered = false; + private readonly activeCollectors: DialogCollector[] = []; + + public constructor( + private readonly contents: BrowserAutomationWebContents, + private readonly cdpQueue: CdpSessionQueue, + ) {} + + public async capture( + task: () => Promise, + ): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> { + const collector: DialogCollector = { dialogs: [] }; + try { + await this.enable(); + await this.installPromptShim(); + } catch (error) { + console.warn("[browser-automation] Dialog capture unavailable; running command without it", { + contentsId: this.contents.id, + error, + }); + return { result: await task(), dialogs: [] }; + } + this.activeCollectors.push(collector); + try { + const result = await task(); + this.recordPromptShimDialogs(await this.drainPromptShim()); + return { result, dialogs: collector.dialogs }; + } finally { + const index = this.activeCollectors.indexOf(collector); + if (index >= 0) { + this.activeCollectors.splice(index, 1); + } + if (this.activeCollectors.length === 0) { + await this.restorePromptShim(); + } + } + } + + private async enable(): Promise { + if (this.enabled) { + return; + } + if (!this.contents.debugger.on) { + return; + } + if (!this.listenerRegistered) { + this.listenerRegistered = true; + this.contents.debugger.on("message", (_event, method, params) => { + if (method !== "Page.javascriptDialogOpening") { + return; + } + if (this.activeCollectors.length === 0) { + return; + } + void this.handleOpening(params ?? {}); + }); + } + await this.sendDebugCommand("Page.enable"); + this.enabled = true; + } + + private async handleOpening(params: Record): Promise { + const event = handledDialogEvent(params); + for (const collector of this.activeCollectors) { + this.recordDialogs(collector, [event]); + } + await this.sendDialogResponseCommand("Page.handleJavaScriptDialog", { + accept: dialogAcceptValue(event.type), + }); + } + + private async installPromptShim(): Promise { + await this.sendDebugCommand("Runtime.evaluate", { + expression: promptShimInstallScript(), + returnByValue: true, + }); + } + + private async drainPromptShim(): Promise { + try { + const result = (await this.sendDebugCommand("Runtime.evaluate", { + expression: promptShimDrainScript(), + returnByValue: true, + })) as { result?: { value?: unknown } }; + return parsePromptShimDialogs(result.result?.value); + } catch { + return []; + } + } + + private async restorePromptShim(): Promise { + try { + await this.sendDebugCommand("Runtime.evaluate", { + expression: promptShimRestoreScript(), + returnByValue: true, + }); + } catch { + // Navigation can destroy the execution context before cleanup runs; the next page has no shim. + } + } + + private recordDialogs(collector: DialogCollector, dialogs: BrowserAutomationDialogEvent[]): void { + for (const dialog of dialogs) { + if (collector.dialogs.length >= MAX_DIALOGS_PER_COMMAND) { + return; + } + collector.dialogs.push(dialog); + } + } + + private recordPromptShimDialogs(dialogs: BrowserAutomationDialogEvent[]): void { + for (const collector of this.activeCollectors) { + this.recordDialogs(collector, dialogs); + } + } + + private async sendDebugCommand( + command: string, + params?: Record, + ): Promise { + return this.cdpQueue.run(async () => { + if (!this.contents.debugger.isAttached()) { + this.contents.debugger.attach("1.3"); + } + return this.contents.debugger.sendCommand(command, params ?? {}); + }); + } + + private async sendDialogResponseCommand( + command: string, + params?: Record, + ): Promise { + // Dialogs can block the CDP command that opened them, so the unblocker must not wait behind + // the per-tab command queue. + if (!this.contents.debugger.isAttached()) { + this.contents.debugger.attach("1.3"); + } + return this.contents.debugger.sendCommand(command, params ?? {}); + } +} + +interface DialogCollector { + dialogs: BrowserAutomationDialogEvent[]; +} + +function parsePromptShimDialogs(value: unknown): BrowserAutomationDialogEvent[] { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap((entry): BrowserAutomationDialogEvent[] => { + if (!entry || typeof entry !== "object") { + return []; + } + const record = entry as Record; + if (record.type !== "prompt" || record.action !== "dismissed") { + return []; + } + return [ + { + type: "prompt", + message: typeof record.message === "string" ? record.message : "", + ...(typeof record.defaultValue === "string" ? { defaultValue: record.defaultValue } : {}), + action: "dismissed", + timestamp: typeof record.timestamp === "number" ? record.timestamp : Date.now(), + }, + ]; }); } diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index c89b696f8..aefb629e5 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -4,6 +4,7 @@ import { describe, expect, test, vi } from "vitest"; import type { BrowserAutomationCommand, BrowserAutomationConsoleLogEntry, + BrowserAutomationDialogEvent, BrowserAutomationExecuteRequest, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; @@ -41,10 +42,26 @@ class FakeTab implements TabContents { public destroyed = false; public bodyText = ""; - public snapshotElements: unknown[] = []; + public snapshotNodes: Array<{ + role: string; + name: string; + tagName: string; + ref?: string; + attributes?: string[]; + children?: unknown[]; + }> = []; public actionScriptResult: unknown = true; + public evaluateScriptResult: unknown = { ok: true, resultJson: "null" }; + public evaluateScriptThrows = false; + public evaluateScriptErrorMessage = "page failed"; + public rejectEditableActionability = false; + public actionabilityResult: unknown = { + ok: true, + target: { point: { x: 40, y: 30 }, rect: { x: 20, y: 10, width: 40, height: 40 } }, + }; public networkEntries: unknown[] = []; public consoleMessages: BrowserAutomationConsoleLogEntry[] = []; + public dialogsToCapture: BrowserAutomationDialogEvent[] = []; public captureNeverPaints = false; public captureThrows = false; public captureErrorMessage = "capture failed"; @@ -60,6 +77,9 @@ class FakeTab implements TabContents { public fullPageScreenshotData = "fullPagePng"; public documentNodeId = 1; public queriedNodeId = 2; + public canNavigateBack = true; + public canNavigateForward = false; + public keypressTargetEditable = false; public constructor( public readonly id: number, @@ -76,11 +96,11 @@ class FakeTab implements TabContents { } public canGoBack(): boolean { - return true; + return this.canNavigateBack; } public canGoForward(): boolean { - return false; + return this.canNavigateForward; } public isLoading(): boolean { @@ -96,12 +116,30 @@ class FakeTab implements TabContents { if (code.includes("document.body.innerText")) { return this.bodyText; } - if (code.includes("CANDIDATE_SELECTOR")) { - return JSON.stringify(this.snapshotElements); + if (code.includes("__PASEO_ARIA_SNAPSHOT__")) { + return JSON.stringify(snapshotResult(this.snapshotNodes)); + } + if (code.includes("Timed out waiting") || code.includes("performance.now()")) { + if (this.rejectEditableActionability && code.includes("const requiresEditable = true")) { + return { ok: false, reason: "timeout", detail: "not editable" }; + } + return this.actionabilityResult; } if (code.includes("performance.getEntriesByType")) { return JSON.stringify(this.networkEntries); } + if (code.includes("window.innerWidth") && code.includes("window.innerHeight")) { + return { x: 640, y: 400 }; + } + if (code.includes("element.focus({ preventScroll: true })")) { + return { editable: this.keypressTargetEditable }; + } + if (code.includes("__PASEO_BROWSER_EVALUATE__")) { + if (this.evaluateScriptThrows) { + throw new Error(this.evaluateScriptErrorMessage); + } + return this.evaluateScriptResult; + } return this.actionScriptResult; } @@ -151,6 +189,12 @@ class FakeTab implements TabContents { return this.consoleMessages; } + public async captureDialogs( + task: () => Promise, + ): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> { + return { result: await task(), dialogs: this.dialogsToCapture }; + } + public async sendDebugCommand( command: string, params?: Record, @@ -170,11 +214,11 @@ class FakeTab implements TabContents { } return { data: this.fullPageScreenshotData }; } - if (command === "DOM.getDocument") { - return { root: { nodeId: this.documentNodeId } }; + if (command === "Runtime.evaluate") { + return { result: { objectId: "object-1" } }; } - if (command === "DOM.querySelector") { - return { nodeId: this.queriedNodeId }; + if (command === "DOM.describeNode") { + return { node: { backendNodeId: this.queriedNodeId, nodeName: "INPUT" } }; } return {}; } @@ -293,41 +337,86 @@ function formElements() { { role: "textbox", tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, + name: "Name", + ref: "@e1", }, { role: "checkbox", tagName: "input", - text: "Agree", - selector: "#agree", - attributes: { id: "agree", type: "checkbox" }, + name: "Agree", + ref: "@e2", + attributes: ["checked=false"], }, { role: "combobox", tagName: "select", - text: "Country", - selector: "#country", - attributes: { id: "country" }, + name: "Country", + ref: "@e3", }, { role: "button", tagName: "button", - text: "Source", - selector: "#source", - attributes: { id: "source" }, + name: "Source", + ref: "@e4", }, { role: "button", tagName: "button", - text: "Target", - selector: "#target", - attributes: { id: "target" }, + name: "Target", + ref: "@e5", }, ]; } +function snapshotResult(nodes: FakeTab["snapshotNodes"]) { + const refs = nodes.flatMap((node) => + node.ref + ? [ + { + ref: node.ref, + fingerprint: { + role: node.role, + name: node.name, + tagName: node.tagName, + type: "", + ariaLabel: "", + }, + }, + ] + : [], + ); + return { + marker: "__PASEO_ARIA_SNAPSHOT__", + root: { + kind: "role", + role: "document", + name: "Fixture", + tagName: "document", + attributes: [], + children: nodes.map((node) => ({ + kind: "role", + role: node.role, + name: node.name, + tagName: node.tagName, + attributes: node.attributes ?? [], + ...(node.ref + ? { ref: node.ref, fingerprint: refs.find((ref) => ref.ref === node.ref)?.fingerprint } + : {}), + children: node.children ?? [], + })), + }, + refs, + truncated: false, + stats: { + nodeCount: nodes.length + 1, + refCount: refs.length, + textLength: 0, + iframeCount: 0, + maxDepth: 1, + }, + }; +} + function containsScript(tab: FakeTab, ...parts: string[]): boolean { return tab.scripts.some((script) => parts.every((part) => script.includes(part))); } @@ -342,48 +431,17 @@ function requireSnapshotRefs(result: Awaited { test("snapshot and click use refs from the same explicit tab", async () => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = [ + browser.tab.snapshotNodes = [ { role: "button", tagName: "button", - text: "Submit", - selector: "#submit", - attributes: { id: "submit" }, + name: "Submit", + ref: "@e1", }, ]; @@ -556,24 +613,149 @@ describe("executeAutomationCommand", () => { workspaceId: WORKSPACE_A, url: "https://a.test/form", title: "Fixture", - elements: [ - { - ref: "@e1", - role: "button", - tagName: "button", - text: "Submit", - selector: "#submit", - attributes: { id: "submit" }, - }, - ], + format: "aria-yaml", + snapshot: '- document "Fixture"\n - button "Submit" [ref=@e1]', + truncated: false, + stats: { nodeCount: 2, refCount: 1, textLength: 50, iframeCount: 0, maxDepth: 1 }, }, }); expect(click).toEqual({ requestId: "req-click", ok: true, - result: { command: "click", browserId: BROWSER_A, ref: "@e1" }, + result: { command: "click", browserId: BROWSER_A, ref: "@e1", x: 40, y: 30 }, + }); + expect(browser.tab.debugCommands).toEqual([ + { + command: "Input.dispatchMouseEvent", + params: { type: "mouseMoved", x: 40, y: 30, button: "none", modifiers: 0 }, + }, + { + command: "Input.dispatchMouseEvent", + params: { + type: "mousePressed", + x: 40, + y: 30, + button: "left", + buttons: 1, + clickCount: 1, + modifiers: 0, + }, + }, + { + command: "Input.dispatchMouseEvent", + params: { + type: "mouseReleased", + x: 40, + y: 30, + button: "left", + buttons: 0, + clickCount: 1, + modifiers: 0, + }, + }, + ]); + }); + + test("click options choose the trusted mouse button, double-click count, and modifiers", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = [ + { + role: "button", + tagName: "button", + name: "Open menu", + ref: "@e1", + }, + ]; + + await browser.snapshot(); + const click = await browser.execute({ + command: "click", + args: { + browserId: BROWSER_A, + ref: "@e1", + button: "right", + doubleClick: true, + modifiers: ["Control", "Shift"], + }, + }); + + expect(click).toEqual({ + requestId: "req-click", + ok: true, + result: { command: "click", browserId: BROWSER_A, ref: "@e1", x: 40, y: 30 }, + }); + expect(browser.tab.debugCommands.map((entry) => entry.params)).toEqual([ + { type: "mouseMoved", x: 40, y: 30, button: "none", modifiers: 10 }, + { + type: "mousePressed", + x: 40, + y: 30, + button: "right", + buttons: 2, + clickCount: 1, + modifiers: 10, + }, + { + type: "mouseReleased", + x: 40, + y: 30, + button: "right", + buttons: 0, + clickCount: 1, + modifiers: 10, + }, + { + type: "mousePressed", + x: 40, + y: 30, + button: "right", + buttons: 2, + clickCount: 2, + modifiers: 10, + }, + { + type: "mouseReleased", + x: 40, + y: 30, + button: "right", + buttons: 0, + clickCount: 2, + modifiers: 10, + }, + ]); + }); + + test("commands include dialogs handled during the command", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + + requireSnapshotRefs(await browser.snapshot()); + browser.tab.dialogsToCapture = [ + { + type: "alert", + message: "Saved", + action: "accepted", + timestamp: 123, + }, + ]; + const click = await browser.execute({ + command: "click", + args: { browserId: BROWSER_A, ref: "@e1" }, + }); + + expect(click).toEqual({ + requestId: "req-click", + ok: true, + result: { command: "click", browserId: BROWSER_A, ref: "@e1", x: 40, y: 30 }, + dialogs: [ + { + type: "alert", + message: "Saved", + action: "accepted", + timestamp: 123, + }, + ], }); - expect(containsScript(browser.tab, "#submit", ".click()")).toBe(true); }); test.each([ @@ -581,32 +763,17 @@ describe("executeAutomationCommand", () => { name: "fill updates a ref from the latest snapshot", command: { command: "fill", args: { browserId: BROWSER_A, ref: "@e1", value: "Ada" } }, result: { command: "fill", browserId: BROWSER_A, ref: "@e1" }, - scriptParts: ["#name", "Ada"], + scriptParts: ['"@e1"', "Ada"], }, { name: "select sets the requested value on a ref", command: { command: "select", args: { browserId: BROWSER_A, ref: "@e3", value: "us" } }, result: { command: "select", browserId: BROWSER_A, ref: "@e3", value: "us" }, - scriptParts: ["#country", "us"], - }, - { - name: "hover dispatches hover events to a ref", - command: { command: "hover", args: { browserId: BROWSER_A, ref: "@e4" } }, - result: { command: "hover", browserId: BROWSER_A, ref: "@e4" }, - scriptParts: ["#source", "mouseover"], - }, - { - name: "drag dispatches drag events between refs", - command: { - command: "drag", - args: { browserId: BROWSER_A, sourceRef: "@e4", targetRef: "@e5" }, - }, - result: { command: "drag", browserId: BROWSER_A, sourceRef: "@e4", targetRef: "@e5" }, - scriptParts: ["#source", "#target", "dragstart"], + scriptParts: ['"@e3"', "us"], }, ] as const)("$name", async ({ command, result, scriptParts }) => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = formElements(); + browser.tab.snapshotNodes = formElements(); requireSnapshotRefs(await browser.snapshot()); const action = await browser.execute(command); @@ -619,9 +786,140 @@ describe("executeAutomationCommand", () => { expect(containsScript(browser.tab, ...scriptParts)).toBe(true); }); + test("hover moves the trusted browser pointer to the actionable point", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + + requireSnapshotRefs(await browser.snapshot()); + const action = await browser.execute({ + command: "hover", + args: { browserId: BROWSER_A, ref: "@e4" }, + }); + + expect(action).toEqual({ + requestId: "req-hover", + ok: true, + result: { command: "hover", browserId: BROWSER_A, ref: "@e4", x: 40, y: 30 }, + }); + expect(browser.tab.debugCommands).toEqual([ + { + command: "Input.dispatchMouseEvent", + params: { type: "mouseMoved", x: 40, y: 30, button: "none" }, + }, + ]); + }); + + test("scroll sends trusted wheel input at the viewport center", async () => { + const browser = new BrowserAutomationHarness(); + + const action = await browser.execute({ + command: "scroll", + args: { browserId: BROWSER_A, deltaX: 10, deltaY: 400 }, + }); + + expect(action).toEqual({ + requestId: "req-scroll", + ok: true, + result: { + command: "scroll", + browserId: BROWSER_A, + deltaX: 10, + deltaY: 400, + x: 640, + y: 400, + }, + }); + expect(browser.tab.debugCommands).toEqual([ + { + command: "Input.dispatchMouseEvent", + params: { type: "mouseWheel", x: 640, y: 400, deltaX: 10, deltaY: 400 }, + }, + ]); + }); + + test("scroll with a ref sends trusted wheel input at the actionable point", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + + requireSnapshotRefs(await browser.snapshot()); + const action = await browser.execute({ + command: "scroll", + args: { browserId: BROWSER_A, ref: "@e4", deltaX: 0, deltaY: -120 }, + }); + + expect(action).toEqual({ + requestId: "req-scroll", + ok: true, + result: { + command: "scroll", + browserId: BROWSER_A, + ref: "@e4", + deltaX: 0, + deltaY: -120, + x: 40, + y: 30, + }, + }); + expect(browser.tab.debugCommands.at(-1)).toEqual({ + command: "Input.dispatchMouseEvent", + params: { type: "mouseWheel", x: 40, y: 30, deltaX: 0, deltaY: -120 }, + }); + }); + + test("drag sends trusted pointer input between actionable points", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + browser.tab.actionabilityResult = { + ok: true, + target: { point: { x: 100, y: 50 }, rect: { x: 80, y: 30, width: 40, height: 40 } }, + }; + + requireSnapshotRefs(await browser.snapshot()); + const action = await browser.execute({ + command: "drag", + args: { browserId: BROWSER_A, sourceRef: "@e4", targetRef: "@e5" }, + }); + + expect(action).toEqual({ + requestId: "req-drag", + ok: true, + result: { + command: "drag", + browserId: BROWSER_A, + sourceRef: "@e4", + targetRef: "@e5", + sourceX: 100, + sourceY: 50, + targetX: 100, + targetY: 50, + }, + }); + expect(browser.tab.debugCommands.map((entry) => entry.command)).toEqual([ + "Input.dispatchMouseEvent", + "Input.dispatchMouseEvent", + "Input.dispatchMouseEvent", + "Input.dispatchMouseEvent", + "Input.dispatchMouseEvent", + ]); + expect(browser.tab.debugCommands.at(1)?.params).toMatchObject({ + type: "mousePressed", + x: 100, + y: 50, + button: "left", + buttons: 1, + }); + expect(browser.tab.debugCommands.at(-1)?.params).toMatchObject({ + type: "mouseReleased", + x: 100, + y: 50, + button: "left", + buttons: 0, + }); + }); + test("fill with an empty string clears a ref through the regular fill path", async () => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = formElements(); + browser.tab.snapshotNodes = formElements(); requireSnapshotRefs(await browser.snapshot()); const action = await browser.execute({ @@ -634,12 +932,12 @@ describe("executeAutomationCommand", () => { ok: true, result: { command: "fill", browserId: BROWSER_A, ref: "@e1" }, }); - expect(containsScript(browser.tab, "#name", 'const nextValue = "";')).toBe(true); + expect(containsScript(browser.tab, '"@e1"', 'const nextValue = "";')).toBe(true); }); test("refs become stale after navigation changes the tab URL", async () => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = formElements(); + browser.tab.snapshotNodes = formElements(); requireSnapshotRefs(await browser.snapshot()); await browser.execute({ @@ -664,7 +962,7 @@ describe("executeAutomationCommand", () => { test("refs become stale when the same URL no longer contains the element", async () => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = formElements(); + browser.tab.snapshotNodes = formElements(); requireSnapshotRefs(await browser.snapshot()); browser.tab.actionScriptResult = false; @@ -684,6 +982,69 @@ describe("executeAutomationCommand", () => { }); }); + test("click returns a retryable browser timeout when the ref never becomes actionable", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + browser.tab.actionabilityResult = { ok: false, reason: "timeout", detail: "disabled" }; + + requireSnapshotRefs(await browser.snapshot()); + const click = await browser.execute({ + command: "click", + args: { browserId: BROWSER_A, ref: "@e2" }, + }); + + expect(click).toEqual({ + requestId: "req-click", + ok: false, + error: { + code: "browser_timeout", + message: "Timed out waiting for browser element @e2 to become actionable.", + retryable: true, + }, + }); + expect(browser.tab.debugCommands).toEqual([]); + }); + + test("command failures include dialogs handled before the failure", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + browser.tab.actionabilityResult = { ok: false, reason: "timeout", detail: "disabled" }; + + requireSnapshotRefs(await browser.snapshot()); + browser.tab.dialogsToCapture = [ + { + type: "prompt", + message: "Name?", + defaultValue: "Maya", + action: "dismissed", + timestamp: 124, + }, + ]; + const click = await browser.execute({ + command: "click", + args: { browserId: BROWSER_A, ref: "@e2" }, + }); + + expect(click).toEqual({ + requestId: "req-click", + ok: false, + error: { + code: "browser_timeout", + message: "Timed out waiting for browser element @e2 to become actionable.", + retryable: true, + }, + dialogs: [ + { + type: "prompt", + message: "Name?", + defaultValue: "Maya", + action: "dismissed", + timestamp: 124, + }, + ], + }); + }); + test("wait resolves when the explicit tab contains the requested text", async () => { const browser = new BrowserAutomationHarness(); browser.tab.bodyText = "Ready"; @@ -720,36 +1081,104 @@ describe("executeAutomationCommand", () => { }); }); - test.each([ - { - name: "type writes text into a ref", - command: { command: "type", args: { browserId: BROWSER_A, ref: "@e1", text: "Ada" } }, - result: { command: "type", browserId: BROWSER_A, ref: "@e1" }, - scriptParts: ["#name", "Ada"], - needsSnapshot: true, - }, - { - name: "keypress dispatches a key to the focused element when ref is omitted", - command: { command: "keypress", args: { browserId: BROWSER_A, key: "Enter" } }, - result: { command: "keypress", browserId: BROWSER_A, key: "Enter" }, - scriptParts: ["document.activeElement", "Enter"], - needsSnapshot: false, - }, - ] as const)("$name", async ({ command, result, scriptParts, needsSnapshot }) => { + test("type writes trusted text into a ref", async () => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = formElements(); - if (needsSnapshot) { - requireSnapshotRefs(await browser.snapshot()); - } + browser.tab.snapshotNodes = formElements(); - const action = await browser.execute(command); + requireSnapshotRefs(await browser.snapshot()); + const action = await browser.execute({ + command: "type", + args: { browserId: BROWSER_A, ref: "@e1", text: "Ada" }, + }); expect(action).toEqual({ - requestId: `req-${command.command}`, + requestId: "req-type", ok: true, - result, + result: { command: "type", browserId: BROWSER_A, ref: "@e1", x: 40, y: 30 }, + }); + expect(browser.tab.debugCommands.at(-1)).toEqual({ + command: "Input.insertText", + params: { text: "Ada" }, + }); + expect(browser.tab.debugCommands.slice(0, 3).map((entry) => entry.command)).toEqual([ + "Input.dispatchMouseEvent", + "Input.dispatchMouseEvent", + "Input.dispatchMouseEvent", + ]); + }); + + test("keypress dispatches a trusted key to the focused element when ref is omitted", async () => { + const browser = new BrowserAutomationHarness(); + + const action = await browser.execute({ + command: "keypress", + args: { browserId: BROWSER_A, key: "Enter" }, + }); + + expect(action).toEqual({ + requestId: "req-keypress", + ok: true, + result: { command: "keypress", browserId: BROWSER_A, key: "Enter" }, + }); + expect(browser.tab.debugCommands).toEqual([ + { + command: "Input.dispatchKeyEvent", + params: { + type: "keyDown", + key: "Enter", + code: "Enter", + windowsVirtualKeyCode: 13, + nativeVirtualKeyCode: 13, + text: "\r", + unmodifiedText: "\r", + }, + }, + { + command: "Input.dispatchKeyEvent", + params: { + type: "keyUp", + key: "Enter", + code: "Enter", + windowsVirtualKeyCode: 13, + nativeVirtualKeyCode: 13, + }, + }, + ]); + }); + + test("keypress focuses a non-editable ref without clicking before the trusted key", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + browser.tab.rejectEditableActionability = true; + + requireSnapshotRefs(await browser.snapshot()); + const action = await browser.execute({ + command: "keypress", + args: { browserId: BROWSER_A, ref: "@e4", key: "Enter" }, + }); + + expect(action).toEqual({ + requestId: "req-keypress", + ok: true, + result: { command: "keypress", browserId: BROWSER_A, key: "Enter", ref: "@e4", x: 40, y: 30 }, + }); + expect(containsScript(browser.tab, "element.focus({ preventScroll: true })")).toBe(true); + expect(browser.tab.debugCommands.map((entry) => entry.command)).toEqual([ + "Input.dispatchKeyEvent", + "Input.dispatchKeyEvent", + ]); + expect(browser.tab.debugCommands.at(-2)).toEqual({ + command: "Input.dispatchKeyEvent", + params: { + type: "keyDown", + key: "Enter", + code: "Enter", + windowsVirtualKeyCode: 13, + nativeVirtualKeyCode: 13, + text: "\r", + unmodifiedText: "\r", + }, }); - expect(containsScript(browser.tab, ...scriptParts)).toBe(true); }); test("navigate loads the requested HTTP URL in the explicit tab", async () => { @@ -788,24 +1217,25 @@ describe("executeAutomationCommand", () => { expect(browser.tab.loadedUrls).toEqual([]); }); - test("navigation actions dispatch to the explicit tab", () => { + test("navigation actions dispatch to the explicit tab", async () => { const browser = new BrowserAutomationHarness(); + browser.tab.canNavigateForward = true; - const back = executeAutomationCommand( + const back = await executeAutomationCommand( automationRequest( { command: "back", args: { browserId: BROWSER_A } }, { requestId: "req-back" }, ), browser.registry, ); - const forward = executeAutomationCommand( + const forward = await executeAutomationCommand( automationRequest( { command: "forward", args: { browserId: BROWSER_A } }, { requestId: "req-forward" }, ), browser.registry, ); - const reload = executeAutomationCommand( + const reload = await executeAutomationCommand( automationRequest( { command: "reload", args: { browserId: BROWSER_A } }, { requestId: "req-reload" }, @@ -831,6 +1261,67 @@ describe("executeAutomationCommand", () => { expect(browser.tab.actions).toEqual(["back", "forward", "reload"]); }); + test("back and forward fail when the tab has no matching history entry", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.canNavigateBack = false; + browser.tab.canNavigateForward = false; + + const back = await browser.execute({ + command: "back", + args: { browserId: BROWSER_A }, + }); + const forward = await browser.execute({ + command: "forward", + args: { browserId: BROWSER_A }, + }); + + expect(back).toEqual({ + requestId: "req-back", + ok: false, + error: { + code: "browser_denied", + message: "There is nothing to go back to.", + retryable: false, + }, + }); + expect(forward).toEqual({ + requestId: "req-forward", + ok: false, + error: { + code: "browser_denied", + message: "There is nothing to go forward to.", + retryable: false, + }, + }); + expect(browser.tab.actions).toEqual([]); + }); + + test.each([ + { + command: { command: "resize", args: { browserId: BROWSER_A, width: 1024, height: 768 } }, + message: "browser_resize is handled by the app runtime.", + }, + { + command: { command: "close_tab", args: { browserId: BROWSER_A } }, + message: "browser_close_tab is handled by the app runtime.", + }, + ] satisfies Array<{ command: BrowserAutomationCommand; message: string }>)( + "$command.command is app-runtime owned", + async ({ command, message }) => { + const browser = new BrowserAutomationHarness(); + + await expect(browser.execute(command)).resolves.toEqual({ + requestId: `req-${command.command}`, + ok: false, + error: { + code: "browser_unsupported", + message, + retryable: false, + }, + }); + }, + ); + test("logs returns bounded console messages and network entries from the explicit tab", async () => { const browser = new BrowserAutomationHarness(); browser.tab.consoleMessages = [ @@ -877,6 +1368,158 @@ describe("executeAutomationCommand", () => { }); }); + test("evaluate returns primitive JSON from the page context", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.evaluateScriptResult = { ok: true, resultJson: "42" }; + + const result = await browser.execute({ + command: "evaluate", + args: { browserId: BROWSER_A, function: "() => 42" }, + }); + + expect(result).toEqual({ + requestId: "req-evaluate", + ok: true, + result: { + command: "evaluate", + browserId: BROWSER_A, + resultJson: "42", + truncated: false, + }, + }); + expect(containsScript(browser.tab, "__PASEO_BROWSER_EVALUATE__", "() => 42")).toBe(true); + }); + + test("evaluate returns object JSON from the page context", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.evaluateScriptResult = { + ok: true, + resultJson: '{"title":"Fixture","ready":true}', + }; + + const result = await browser.execute({ + command: "evaluate", + args: { browserId: BROWSER_A, function: "() => ({ title: document.title, ready: true })" }, + }); + + expect(result).toEqual({ + requestId: "req-evaluate", + ok: true, + result: { + command: "evaluate", + browserId: BROWSER_A, + resultJson: '{"title":"Fixture","ready":true}', + truncated: false, + }, + }); + }); + + test("evaluate passes the resolved ref element as the first argument", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + browser.tab.evaluateScriptResult = { ok: true, resultJson: '"Name"' }; + + requireSnapshotRefs(await browser.snapshot()); + const result = await browser.execute({ + command: "evaluate", + args: { browserId: BROWSER_A, ref: "@e1", function: "(element) => element.name" }, + }); + + expect(result).toEqual({ + requestId: "req-evaluate", + ok: true, + result: { + command: "evaluate", + browserId: BROWSER_A, + resultJson: '"Name"', + truncated: false, + }, + }); + expect(containsScript(browser.tab, '"@e1"', "__PASEO_BROWSER_AUTOMATION__?.resolve")).toBe( + true, + ); + }); + + test("evaluate returns stale ref when the target ref cannot be resolved", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.snapshotNodes = formElements(); + + requireSnapshotRefs(await browser.snapshot()); + const result = await browser.execute({ + command: "evaluate", + args: { browserId: BROWSER_A, ref: "@e9", function: "(element) => element.name" }, + }); + + expect(result).toEqual({ + requestId: "req-evaluate", + ok: false, + error: { + code: "browser_stale_ref", + message: "Browser element reference @e9 is stale. Take a new snapshot and try again.", + retryable: false, + }, + }); + }); + + test("evaluate converts thrown page code into a bounded unknown error", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.evaluateScriptResult = { error: `${"x".repeat(2_100)}boom` }; + + const result = await browser.execute({ + command: "evaluate", + args: { browserId: BROWSER_A, function: "() => { throw new Error('boom'); }" }, + }); + + expect(result).toEqual({ + requestId: "req-evaluate", + ok: false, + error: { + code: "browser_unknown_error", + message: "x".repeat(2_000), + retryable: false, + }, + }); + }); + + test("evaluate returns oversized results with explicit truncation", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.evaluateScriptResult = { ok: true, resultJson: "x".repeat(80_010) }; + + const result = await browser.execute({ + command: "evaluate", + args: { browserId: BROWSER_A, function: "() => 'large'" }, + }); + + expect(result).toEqual({ + requestId: "req-evaluate", + ok: true, + result: { + command: "evaluate", + browserId: BROWSER_A, + resultJson: JSON.stringify("x".repeat(79_000)), + truncated: true, + }, + }); + if (!result.ok) { + throw new Error("Expected evaluate to succeed"); + } + expect(JSON.parse(result.result.resultJson)).toBe("x".repeat(79_000)); + }); + + test("evaluate caps oversized results inside the page script", async () => { + const browser = new BrowserAutomationHarness(); + + await browser.execute({ + command: "evaluate", + args: { browserId: BROWSER_A, function: "() => 'large'" }, + }); + + expect( + containsScript(browser.tab, "__PASEO_BROWSER_EVALUATE__", "resultJson.length <= 80000"), + ).toBe(true); + expect(containsScript(browser.tab, "resultJson.slice(0, 79000)")).toBe(true); + }); + test("screenshot captures the painted viewport", async () => { const browser = new BrowserAutomationHarness(); @@ -1209,13 +1852,12 @@ describe("executeAutomationCommand", () => { test("upload resolves workspace files before setting them on the file input", async () => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = [ + browser.tab.snapshotNodes = [ { role: "textbox", tagName: "input", - text: "", - selector: "#file", - attributes: { id: "file", type: "file" }, + name: "Upload", + ref: "@e1", }, ]; const workspaceRoot = resolvePath("/workspace/project"); @@ -1240,24 +1882,30 @@ describe("executeAutomationCommand", () => { }, }); expect(browser.tab.debugCommands).toEqual([ - { command: "DOM.getDocument", params: { depth: -1, pierce: true } }, - { command: "DOM.querySelector", params: { nodeId: 1, selector: "#file" } }, + { + command: "Runtime.evaluate", + params: { + expression: expect.stringContaining('"@e1"'), + objectGroup: "paseo-browser-automation", + returnByValue: false, + }, + }, + { command: "DOM.describeNode", params: { objectId: "object-1" } }, { command: "DOM.setFileInputFiles", - params: { nodeId: 2, files: [resolvePath(workspaceRoot, "uploads/a.txt")] }, + params: { backendNodeId: 2, files: [resolvePath(workspaceRoot, "uploads/a.txt")] }, }, ]); }); test("upload denies paths outside the agent workspace", async () => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = [ + browser.tab.snapshotNodes = [ { role: "textbox", tagName: "input", - text: "", - selector: "#file", - attributes: { id: "file", type: "file" }, + name: "Upload", + ref: "@e1", }, ]; const workspaceRoot = resolvePath("/workspace/project"); @@ -1281,20 +1929,26 @@ describe("executeAutomationCommand", () => { }, }); expect(browser.tab.debugCommands).toEqual([ - { command: "DOM.getDocument", params: { depth: -1, pierce: true } }, - { command: "DOM.querySelector", params: { nodeId: 1, selector: "#file" } }, + { + command: "Runtime.evaluate", + params: { + expression: expect.stringContaining('"@e1"'), + objectGroup: "paseo-browser-automation", + returnByValue: false, + }, + }, + { command: "DOM.describeNode", params: { objectId: "object-1" } }, ]); }); test("upload reports missing cwd before setting files on the file input", async () => { const browser = new BrowserAutomationHarness(); - browser.tab.snapshotElements = [ + browser.tab.snapshotNodes = [ { role: "textbox", tagName: "input", - text: "", - selector: "#file", - attributes: { id: "file", type: "file" }, + name: "Upload", + ref: "@e1", }, ]; @@ -1314,8 +1968,15 @@ describe("executeAutomationCommand", () => { }, }); expect(browser.tab.debugCommands).toEqual([ - { command: "DOM.getDocument", params: { depth: -1, pierce: true } }, - { command: "DOM.querySelector", params: { nodeId: 1, selector: "#file" } }, + { + command: "Runtime.evaluate", + params: { + expression: expect.stringContaining('"@e1"'), + objectGroup: "paseo-browser-automation", + returnByValue: false, + }, + }, + { command: "DOM.describeNode", params: { objectId: "object-1" } }, ]); }); }); diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index 3a59bf447..72961f16f 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -3,12 +3,23 @@ import { isAbsolute, relative, resolve as resolvePath } from "node:path"; import type { BrowserAutomationCommand, BrowserAutomationConsoleLogEntry, + BrowserAutomationDialogEvent, BrowserAutomationErrorCode, BrowserAutomationExecuteResponse, BrowserAutomationExecuteRequest, BrowserAutomationNetworkLogEntry, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import { waitForActionableTarget, type ActionabilityResult } from "./actionability.js"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; +import { + dispatchTrustedClick, + dispatchTrustedDrag, + dispatchTrustedHover, + dispatchTrustedKey, + dispatchTrustedScroll, + dispatchTrustedText, + type ClickInputOptions, +} from "./trusted-input.js"; export interface TabContents { readonly id: number; @@ -26,6 +37,9 @@ export interface TabContents { capturePage(options?: TabCapturePageOptions): Promise; invalidate(): void; getConsoleMessages?(): BrowserAutomationConsoleLogEntry[]; + captureDialogs?( + task: () => Promise, + ): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }>; sendDebugCommand?(command: string, params?: Record): Promise; } @@ -56,6 +70,9 @@ const PIXEL_CAPTURE_TIMEOUT_MS = 5_000; const PIXEL_CAPTURE_RETRY_INTERVAL_MS = 200; const SCREENSHOT_NO_FRAME_MESSAGE = "The tab has not painted yet. Retry the screenshot."; const ALLOWED_PAGE_URL_PROTOCOLS = new Set(["http:", "https:"]); +const MAX_EVALUATE_RESULT_JSON_LENGTH = 80_000; +const MAX_EVALUATE_RESULT_PREVIEW_LENGTH = 79_000; +const MAX_EVALUATE_ERROR_MESSAGE_LENGTH = 2_000; let pixelCaptureQueue: Promise = Promise.resolve(); function fail( @@ -67,6 +84,17 @@ function fail( return { requestId, ok: false, error: { code, message, retryable } }; } +async function withDialogCapture( + contents: TabContents, + task: () => Promise, +): Promise { + if (!contents.captureDialogs) { + return task(); + } + const { result, dialogs } = await contents.captureDialogs(task); + return dialogs.length > 0 ? { ...result, dialogs } : result; +} + class ScreenshotNoFrameError extends Error { public constructor(message = SCREENSHOT_NO_FRAME_MESSAGE) { super(message); @@ -236,6 +264,11 @@ const commandHandlers: Record { + navigate: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const navigateCommand = command as Extract; return executeNavigate( requestId, @@ -298,9 +331,10 @@ const commandHandlers: Record { + back: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const backCommand = command as Extract; return executeNavigationAction( requestId, @@ -308,9 +342,10 @@ const commandHandlers: Record { + forward: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const forwardCommand = command as Extract; return executeNavigationAction( requestId, @@ -318,9 +353,10 @@ const commandHandlers: Record { + reload: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { const reloadCommand = command as Extract; return executeNavigationAction( requestId, @@ -328,6 +364,7 @@ const commandHandlers: Record { @@ -400,6 +437,35 @@ const commandHandlers: Record { + const evaluateCommand = command as Extract; + return executeEvaluate( + requestId, + workspaceId, + evaluateCommand.args.browserId, + evaluateCommand.args.function, + evaluateCommand.args.ref, + registry, + snapshotEngine, + ); + }, + scroll: ({ command, requestId, workspaceId, registry, snapshotEngine }) => { + const scrollCommand = command as Extract; + return executeScroll( + requestId, + workspaceId, + scrollCommand.args.browserId, + scrollCommand.args.ref, + scrollCommand.args.deltaX, + scrollCommand.args.deltaY, + registry, + snapshotEngine, + ); + }, + resize: ({ requestId }) => + fail(requestId, "browser_unsupported", "browser_resize is handled by the app runtime."), + close_tab: ({ requestId }) => + fail(requestId, "browser_unsupported", "browser_close_tab is handled by the app runtime."), }; interface ResolvedTabTarget { @@ -452,25 +518,27 @@ async function executeSnapshot( return target; } - const elements = await snapshotEngine.snapshot({ - browserId: target.browserId, - page: target.contents, - }); - - return { - requestId, - ok: true, - result: { - command: "snapshot", + return withDialogCapture(target.contents, async () => { + const snapshot = await snapshotEngine.snapshot({ browserId: target.browserId, - ...(registry.getBrowserWorkspaceId(target.browserId) - ? { workspaceId: registry.getBrowserWorkspaceId(target.browserId) ?? undefined } - : {}), - url: target.contents.getURL(), - title: target.contents.getTitle(), - elements, - }, - }; + page: target.contents, + }); + + return { + requestId, + ok: true, + result: { + command: "snapshot", + browserId: target.browserId, + ...(registry.getBrowserWorkspaceId(target.browserId) + ? { workspaceId: registry.getBrowserWorkspaceId(target.browserId) ?? undefined } + : {}), + url: target.contents.getURL(), + title: target.contents.getTitle(), + ...snapshot, + }, + }; + }); } async function executeClick( @@ -478,6 +546,7 @@ async function executeClick( workspaceId: string | undefined, browserId: string, ref: string, + options: ClickInputOptions, registry: BrowserRegistry, snapshotEngine: BrowserSnapshotEngine, ): Promise { @@ -485,15 +554,37 @@ async function executeClick( if ("ok" in target) { return target; } - const result = await snapshotEngine.click({ - browserId: target.browserId, - page: target.contents, - ref, + return withDialogCapture(target.contents, async () => { + if (!target.contents.sendDebugCommand) { + return fail(requestId, "browser_unsupported", "browser_click requires trusted browser input"); + } + const elementExpression = snapshotEngine.runtimeElementExpression({ + browserId: target.browserId, + ref, + }); + if (typeof elementExpression !== "string") { + return staleRefFailure(requestId, ref); + } + const actionable = await waitForActionableTarget({ + page: target.contents, + elementExpression, + }); + if (!actionable.ok) { + return actionabilityFailure(requestId, ref, actionable); + } + await dispatchTrustedClick(cdpSender(target.contents), actionable.target.point, options); + return { + requestId, + ok: true, + result: { + command: "click", + browserId: target.browserId, + ref, + x: actionable.target.point.x, + y: actionable.target.point.y, + }, + }; }); - if (!result.ok) { - return staleRefFailure(requestId, ref); - } - return { requestId, ok: true, result: { command: "click", browserId: target.browserId, ref } }; } async function executeFill( @@ -509,16 +600,18 @@ async function executeFill( if ("ok" in target) { return target; } - const result = await snapshotEngine.fill({ - browserId: target.browserId, - page: target.contents, - ref, - value, + return withDialogCapture(target.contents, async () => { + const result = await snapshotEngine.fill({ + browserId: target.browserId, + page: target.contents, + ref, + value, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { requestId, ok: true, result: { command: "fill", browserId: target.browserId, ref } }; }); - if (!result.ok) { - return staleRefFailure(requestId, ref); - } - return { requestId, ok: true, result: { command: "fill", browserId: target.browserId, ref } }; } async function executeSelect( @@ -534,20 +627,22 @@ async function executeSelect( if ("ok" in target) { return target; } - const result = await snapshotEngine.select({ - browserId: target.browserId, - page: target.contents, - ref, - value, + return withDialogCapture(target.contents, async () => { + const result = await snapshotEngine.select({ + browserId: target.browserId, + page: target.contents, + ref, + value, + }); + if (!result.ok) { + return staleRefFailure(requestId, ref); + } + return { + requestId, + ok: true, + result: { command: "select", browserId: target.browserId, ref, value }, + }; }); - if (!result.ok) { - return staleRefFailure(requestId, ref); - } - return { - requestId, - ok: true, - result: { command: "select", browserId: target.browserId, ref, value }, - }; } async function executeHover( @@ -562,15 +657,37 @@ async function executeHover( if ("ok" in target) { return target; } - const result = await snapshotEngine.hover({ - browserId: target.browserId, - page: target.contents, - ref, + return withDialogCapture(target.contents, async () => { + if (!target.contents.sendDebugCommand) { + return fail(requestId, "browser_unsupported", "browser_hover requires trusted browser input"); + } + const elementExpression = snapshotEngine.runtimeElementExpression({ + browserId: target.browserId, + ref, + }); + if (typeof elementExpression !== "string") { + return staleRefFailure(requestId, ref); + } + const actionable = await waitForActionableTarget({ + page: target.contents, + elementExpression, + }); + if (!actionable.ok) { + return actionabilityFailure(requestId, ref, actionable); + } + await dispatchTrustedHover(cdpSender(target.contents), actionable.target.point); + return { + requestId, + ok: true, + result: { + command: "hover", + browserId: target.browserId, + ref, + x: actionable.target.point.x, + y: actionable.target.point.y, + }, + }; }); - if (!result.ok) { - return staleRefFailure(requestId, ref); - } - return { requestId, ok: true, result: { command: "hover", browserId: target.browserId, ref } }; } async function executeDrag( @@ -586,20 +703,55 @@ async function executeDrag( if ("ok" in target) { return target; } - const result = await snapshotEngine.drag({ - browserId: target.browserId, - page: target.contents, - sourceRef, - targetRef, + return withDialogCapture(target.contents, async () => { + if (!target.contents.sendDebugCommand) { + return fail(requestId, "browser_unsupported", "browser_drag requires trusted browser input"); + } + const sourceExpression = snapshotEngine.runtimeElementExpression({ + browserId: target.browserId, + ref: sourceRef, + }); + const targetExpression = snapshotEngine.runtimeElementExpression({ + browserId: target.browserId, + ref: targetRef, + }); + if (typeof sourceExpression !== "string" || typeof targetExpression !== "string") { + return staleRefFailure(requestId, `${sourceRef}/${targetRef}`); + } + const source = await waitForActionableTarget({ + page: target.contents, + elementExpression: sourceExpression, + }); + if (!source.ok) { + return actionabilityFailure(requestId, sourceRef, source); + } + const dropTarget = await waitForActionableTarget({ + page: target.contents, + elementExpression: targetExpression, + }); + if (!dropTarget.ok) { + return actionabilityFailure(requestId, targetRef, dropTarget); + } + await dispatchTrustedDrag( + cdpSender(target.contents), + source.target.point, + dropTarget.target.point, + ); + return { + requestId, + ok: true, + result: { + command: "drag", + browserId: target.browserId, + sourceRef, + targetRef, + sourceX: source.target.point.x, + sourceY: source.target.point.y, + targetX: dropTarget.target.point.x, + targetY: dropTarget.target.point.y, + }, + }; }); - if (!result.ok) { - return staleRefFailure(requestId, `${sourceRef}/${targetRef}`); - } - return { - requestId, - ok: true, - result: { command: "drag", browserId: target.browserId, sourceRef, targetRef }, - }; } async function executeLogs( @@ -613,19 +765,153 @@ async function executeLogs( if ("ok" in target) { return target; } - const consoleMessages = target.contents.getConsoleMessages?.() ?? []; - const networkEntries = parseNetworkEntries( - await target.contents.executeJavaScript(NETWORK_PERFORMANCE_SCRIPT), + return withDialogCapture(target.contents, async () => { + const consoleMessages = target.contents.getConsoleMessages?.() ?? []; + const networkEntries = parseNetworkEntries( + await target.contents.executeJavaScript(NETWORK_PERFORMANCE_SCRIPT), + ); + return { + requestId, + ok: true, + result: { + command: "logs", + browserId: target.browserId, + console: consoleMessages.slice(-maxEntries), + network: networkEntries.slice(-maxEntries), + }, + }; + }); +} + +async function executeEvaluate( + requestId: string, + workspaceId: string | undefined, + browserId: string, + functionSource: string, + ref: string | undefined, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + return withDialogCapture(target.contents, async () => { + let elementExpression: string | undefined; + if (ref) { + const expression = snapshotEngine.runtimeElementExpression({ + browserId: target.browserId, + ref, + }); + if (typeof expression !== "string") { + return staleRefFailure(requestId, ref); + } + elementExpression = expression; + } + + let rawResult: unknown; + try { + rawResult = await target.contents.executeJavaScript( + buildEvaluateScript(functionSource, elementExpression), + ); + } catch (error) { + return fail(requestId, "browser_unknown_error", evaluateErrorMessage(error)); + } + + const result = readEvaluateScriptResult(rawResult); + if (result.status === "stale_ref") { + return staleRefFailure(requestId, ref ?? "unknown"); + } + if (result.status === "error") { + return fail(requestId, "browser_unknown_error", capEvaluateErrorMessage(result.message)); + } + + const capped = capEvaluateResultJson(result.resultJson); + return { + requestId, + ok: true, + result: { + command: "evaluate", + browserId: target.browserId, + resultJson: capped.resultJson, + truncated: result.truncated || capped.truncated, + }, + }; + }); +} + +async function executeScroll( + requestId: string, + workspaceId: string | undefined, + browserId: string, + ref: string | undefined, + deltaX: number, + deltaY: number, + registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, +): Promise { + const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); + if ("ok" in target) { + return target; + } + return withDialogCapture(target.contents, async () => { + if (!target.contents.sendDebugCommand) { + return fail( + requestId, + "browser_unsupported", + "browser_scroll requires trusted browser input", + ); + } + + let point: { x: number; y: number }; + if (ref) { + const elementExpression = snapshotEngine.runtimeElementExpression({ + browserId: target.browserId, + ref, + }); + if (typeof elementExpression !== "string") { + return staleRefFailure(requestId, ref); + } + const actionable = await waitForActionableTarget({ + page: target.contents, + elementExpression, + }); + if (!actionable.ok) { + return actionabilityFailure(requestId, ref, actionable); + } + point = actionable.target.point; + } else { + point = await readViewportCenter(target.contents); + } + + await dispatchTrustedScroll(cdpSender(target.contents), point, deltaX, deltaY); + return { + requestId, + ok: true, + result: { + command: "scroll", + browserId: target.browserId, + ...(ref ? { ref } : {}), + deltaX, + deltaY, + x: point.x, + y: point.y, + }, + }; + }); +} + +async function readViewportCenter(contents: TabContents): Promise<{ x: number; y: number }> { + const value = await contents.executeJavaScript( + "({ x: Math.max(0, (window.innerWidth || 1) / 2), y: Math.max(0, (window.innerHeight || 1) / 2) })", ); + if (!value || typeof value !== "object") { + return { x: 0, y: 0 }; + } + const record = value as Record; return { - requestId, - ok: true, - result: { - command: "logs", - browserId: target.browserId, - console: consoleMessages.slice(-maxEntries), - network: networkEntries.slice(-maxEntries), - }, + x: readNumber(record.x) ?? 0, + y: readNumber(record.y) ?? 0, }; } @@ -637,6 +923,26 @@ function staleRefFailure(requestId: string, ref: string): FailurePayload { ); } +function actionabilityFailure( + requestId: string, + ref: string, + result: Exclude, +): FailurePayload { + if (result.reason === "stale_ref") { + return staleRefFailure(requestId, ref); + } + return fail( + requestId, + "browser_timeout", + `Timed out waiting for browser element ${ref} to become actionable.`, + true, + ); +} + +function cdpSender(contents: TabContents): NonNullable { + return contents.sendDebugCommand?.bind(contents) as NonNullable; +} + async function executeWait( requestId: string, workspaceId: string | undefined, @@ -648,51 +954,52 @@ async function executeWait( if ("ok" in target) { return target; } - - if (!condition.text && !condition.url) { - return fail(requestId, "browser_unsupported", "browser_wait requires text or url"); - } - - const timeoutMs = condition.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; - const deadline = Date.now() + timeoutMs; - do { - if (condition.url && target.contents.getURL().includes(condition.url)) { - return { - requestId, - ok: true, - result: { command: "wait", browserId: target.browserId, matched: "url" }, - }; + return withDialogCapture(target.contents, async () => { + if (!condition.text && !condition.url) { + return fail(requestId, "browser_unsupported", "browser_wait requires text or url"); } - if (condition.text) { - const pageText = await target.contents.executeJavaScript("document.body.innerText || ''"); - if (typeof pageText === "string" && pageText.includes(condition.text)) { + + const timeoutMs = condition.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; + const deadline = Date.now() + timeoutMs; + do { + if (condition.url && target.contents.getURL().includes(condition.url)) { return { requestId, ok: true, - result: { command: "wait", browserId: target.browserId, matched: "text" }, + result: { command: "wait", browserId: target.browserId, matched: "url" }, }; } - } - await delay(WAIT_POLL_INTERVAL_MS); - } while (Date.now() < deadline); + if (condition.text) { + const pageText = await target.contents.executeJavaScript("document.body.innerText || ''"); + if (typeof pageText === "string" && pageText.includes(condition.text)) { + return { + requestId, + ok: true, + result: { command: "wait", browserId: target.browserId, matched: "text" }, + }; + } + } + await delay(WAIT_POLL_INTERVAL_MS); + } while (Date.now() < deadline); - if (condition.text) { - return fail( - requestId, - "browser_timeout", - `Timed out waiting for browser text: ${condition.text}`, - true, - ); - } - if (condition.url) { - return fail( - requestId, - "browser_timeout", - `Timed out waiting for browser URL: ${condition.url}`, - true, - ); - } - return fail(requestId, "browser_unsupported", "browser_wait requires text or url"); + if (condition.text) { + return fail( + requestId, + "browser_timeout", + `Timed out waiting for browser text: ${condition.text}`, + true, + ); + } + if (condition.url) { + return fail( + requestId, + "browser_timeout", + `Timed out waiting for browser URL: ${condition.url}`, + true, + ); + } + return fail(requestId, "browser_unsupported", "browser_wait requires text or url"); + }); } async function executeType( @@ -708,20 +1015,41 @@ async function executeType( if ("ok" in target) { return target; } - const result = await snapshotEngine.typeText({ - browserId: target.browserId, - page: target.contents, - ...(ref ? { ref } : {}), - text, + return withDialogCapture(target.contents, async () => { + if (!target.contents.sendDebugCommand) { + return fail(requestId, "browser_unsupported", "browser_type requires trusted browser input"); + } + let actionable: ActionabilityResult | null = null; + if (ref) { + const elementExpression = snapshotEngine.runtimeElementExpression({ + browserId: target.browserId, + ref, + }); + if (typeof elementExpression !== "string") { + return staleRefFailure(requestId, ref); + } + actionable = await waitForActionableTarget({ + page: target.contents, + elementExpression, + editable: true, + }); + if (!actionable.ok) { + return actionabilityFailure(requestId, ref, actionable); + } + await dispatchTrustedClick(cdpSender(target.contents), actionable.target.point); + } + await dispatchTrustedText(cdpSender(target.contents), text); + return { + requestId, + ok: true, + result: { + command: "type", + browserId: target.browserId, + ...(ref ? { ref } : {}), + ...(actionable?.ok ? { x: actionable.target.point.x, y: actionable.target.point.y } : {}), + }, + }; }); - if (!result.ok) { - return staleRefFailure(requestId, ref ?? "@e0"); - } - return { - requestId, - ok: true, - result: { command: "type", browserId: target.browserId, ...(ref ? { ref } : {}) }, - }; } async function executeKeypress( @@ -737,20 +1065,51 @@ async function executeKeypress( if ("ok" in target) { return target; } - const result = await snapshotEngine.keypress({ - browserId: target.browserId, - page: target.contents, - ...(ref ? { ref } : {}), - key, + return withDialogCapture(target.contents, async () => { + if (!target.contents.sendDebugCommand) { + return fail( + requestId, + "browser_unsupported", + "browser_keypress requires trusted browser input", + ); + } + let actionable: ActionabilityResult | null = null; + if (ref) { + const elementExpression = snapshotEngine.runtimeElementExpression({ + browserId: target.browserId, + ref, + }); + if (typeof elementExpression !== "string") { + return staleRefFailure(requestId, ref); + } + actionable = await waitForActionableTarget({ + page: target.contents, + elementExpression, + }); + if (!actionable.ok) { + return actionabilityFailure(requestId, ref, actionable); + } + const focused = await focusKeypressTarget(target.contents, elementExpression); + if (focused === "stale_ref") { + return staleRefFailure(requestId, ref); + } + if (focused === "editable") { + await dispatchTrustedClick(cdpSender(target.contents), actionable.target.point); + } + } + await dispatchTrustedKey(cdpSender(target.contents), key); + return { + requestId, + ok: true, + result: { + command: "keypress", + browserId: target.browserId, + key, + ...(ref ? { ref } : {}), + ...(actionable?.ok ? { x: actionable.target.point.x, y: actionable.target.point.y } : {}), + }, + }; }); - if (!result.ok) { - return staleRefFailure(requestId, ref ?? "@e0"); - } - return { - requestId, - ok: true, - result: { command: "keypress", browserId: target.browserId, key, ...(ref ? { ref } : {}) }, - }; } async function executeNavigate( @@ -759,43 +1118,63 @@ async function executeNavigate( browserId: string, url: string, registry: BrowserRegistry, + snapshotEngine: BrowserSnapshotEngine, ): Promise { const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); if ("ok" in target) { return target; } - if (!isAllowedPageUrl(url)) { - return fail( + return withDialogCapture(target.contents, async () => { + if (!isAllowedPageUrl(url)) { + return fail( + requestId, + "browser_denied", + "Browser navigation only supports http and https URLs.", + ); + } + snapshotEngine.clearBrowser(browserId); + await target.contents.loadURL(url); + return { requestId, - "browser_denied", - "Browser navigation only supports http and https URLs.", - ); - } - await target.contents.loadURL(url); - return { requestId, ok: true, result: { command: "navigate", browserId: target.browserId, url } }; + ok: true, + result: { command: "navigate", browserId: target.browserId, url }, + }; + }); } -function executeNavigationAction( +async function executeNavigationAction( requestId: string, workspaceId: string | undefined, browserId: string, action: "back" | "forward" | "reload", registry: BrowserRegistry, -): AutomationCommandPayload { + snapshotEngine: BrowserSnapshotEngine, +): Promise { const target = resolveTabTarget({ requestId, workspaceId, browserId, registry }); if ("ok" in target) { return target; } - if (action === "back") { - target.contents.goBack(); - return { requestId, ok: true, result: { command: "back", browserId: target.browserId } }; - } - if (action === "forward") { - target.contents.goForward(); - return { requestId, ok: true, result: { command: "forward", browserId: target.browserId } }; - } - target.contents.reload(); - return { requestId, ok: true, result: { command: "reload", browserId: target.browserId } }; + return withDialogCapture(target.contents, async () => { + if (action === "back") { + if (!target.contents.canGoBack()) { + return fail(requestId, "browser_denied", "There is nothing to go back to."); + } + snapshotEngine.clearBrowser(browserId); + target.contents.goBack(); + return { requestId, ok: true, result: { command: "back", browserId: target.browserId } }; + } + if (action === "forward") { + if (!target.contents.canGoForward()) { + return fail(requestId, "browser_denied", "There is nothing to go forward to."); + } + snapshotEngine.clearBrowser(browserId); + target.contents.goForward(); + return { requestId, ok: true, result: { command: "forward", browserId: target.browserId } }; + } + snapshotEngine.clearBrowser(browserId); + target.contents.reload(); + return { requestId, ok: true, result: { command: "reload", browserId: target.browserId } }; + }); } async function executeScreenshot( @@ -813,28 +1192,30 @@ async function executeScreenshot( if ("ok" in target) { return target; } - let image: TabImage; - try { - image = await capturePaintedViewport(target.contents); - } catch (error) { - if (isScreenshotNoFrameError(error)) { - return screenshotNoFrameFailure(requestId, error); + return withDialogCapture(target.contents, async () => { + let image: TabImage; + try { + image = await capturePaintedViewport(target.contents); + } catch (error) { + if (isScreenshotNoFrameError(error)) { + return screenshotNoFrameFailure(requestId, error); + } + throw error; } - throw error; - } - const size = image.getSize(); - return { - requestId, - ok: true, - result: { - command: "screenshot", - browserId: target.browserId, - mimeType: "image/png", - dataBase64: Buffer.from(image.toPNG()).toString("base64"), - width: size.width, - height: size.height, - }, - }; + const size = image.getSize(); + return { + requestId, + ok: true, + result: { + command: "screenshot", + browserId: target.browserId, + mimeType: "image/png", + dataBase64: Buffer.from(image.toPNG()).toString("base64"), + width: size.width, + height: size.height, + }, + }; + }); } interface CdpLayoutMetrics { @@ -860,6 +1241,21 @@ interface CdpCaptureScreenshotResult { data?: string; } +interface CdpRuntimeEvaluateResult { + result?: { + objectId?: string; + subtype?: string; + }; +} + +interface CdpDescribeNodeResult { + node?: { + backendNodeId?: number; + nodeId?: number; + nodeName?: string; + }; +} + async function getCdpLayoutMetrics(contents: TabContents): Promise<{ viewportWidth: number; viewportHeight: number; @@ -890,45 +1286,47 @@ async function executeFullPageScreenshot( if ("ok" in target) { return target; } - if (!target.contents.sendDebugCommand) { - return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP"); - } - const sendDebugCommand = target.contents.sendDebugCommand.bind(target.contents); - let screenshot: CdpCaptureScreenshotResult; - let width = 0; - let height = 0; - try { - screenshot = await runPaintedPixelCapture(target.contents, async () => { - const metrics = await getCdpLayoutMetrics(target.contents); - width = metrics.contentWidth; - height = metrics.contentHeight; - return (await sendDebugCommand("Page.captureScreenshot", { - format: "png", - captureBeyondViewport: true, - clip: { x: 0, y: 0, width, height, scale: 1 }, - })) as CdpCaptureScreenshotResult; - }); - } catch (error) { - if (isScreenshotNoFrameError(error)) { - return screenshotNoFrameFailure(requestId, error); + return withDialogCapture(target.contents, async () => { + if (!target.contents.sendDebugCommand) { + return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP"); } - throw error; - } - if (!screenshot.data) { - return fail(requestId, "browser_unsupported", "browser_screenshot fullPage returned no data"); - } - return { - requestId, - ok: true, - result: { - command: "screenshot", - browserId: target.browserId, - mimeType: "image/png", - dataBase64: screenshot.data, - width, - height, - }, - }; + const sendDebugCommand = target.contents.sendDebugCommand.bind(target.contents); + let screenshot: CdpCaptureScreenshotResult; + let width = 0; + let height = 0; + try { + screenshot = await runPaintedPixelCapture(target.contents, async () => { + const metrics = await getCdpLayoutMetrics(target.contents); + width = metrics.contentWidth; + height = metrics.contentHeight; + return (await sendDebugCommand("Page.captureScreenshot", { + format: "png", + captureBeyondViewport: true, + clip: { x: 0, y: 0, width, height, scale: 1 }, + })) as CdpCaptureScreenshotResult; + }); + } catch (error) { + if (isScreenshotNoFrameError(error)) { + return screenshotNoFrameFailure(requestId, error); + } + throw error; + } + if (!screenshot.data) { + return fail(requestId, "browser_unsupported", "browser_screenshot fullPage returned no data"); + } + return { + requestId, + ok: true, + result: { + command: "screenshot", + browserId: target.browserId, + mimeType: "image/png", + dataBase64: screenshot.data, + width, + height, + }, + }; + }); } function isAllowedPageUrl(value: string): boolean { @@ -952,59 +1350,61 @@ async function executeUpload( if ("ok" in target) { return target; } - if (!target.contents.sendDebugCommand) { - return fail(requestId, "browser_unsupported", "browser_upload requires CDP"); - } - const resolved = snapshotEngine.selectorForRef({ - browserId: target.browserId, - page: target.contents, - ref: input.ref, - }); - if (!resolved.ok) { - return staleRefFailure(requestId, input.ref); - } - const document = (await target.contents.sendDebugCommand("DOM.getDocument", { - depth: -1, - pierce: true, - })) as { root?: { nodeId?: number } }; - const rootNodeId = document.root?.nodeId; - if (typeof rootNodeId !== "number") { - return fail(requestId, "browser_unsupported", "browser_upload could not read DOM"); - } - const queried = (await target.contents.sendDebugCommand("DOM.querySelector", { - nodeId: rootNodeId, - selector: resolved.selector, - })) as { nodeId?: number }; - if (typeof queried.nodeId !== "number" || queried.nodeId <= 0) { - return staleRefFailure(requestId, input.ref); - } - const workspaceRoot = resolveUploadWorkspaceRoot(cwd); - if (!workspaceRoot) { - return fail(requestId, "browser_unsupported", "browser_upload requires request cwd"); - } - const filePaths = resolveWorkspaceFilePaths(input.filePaths, workspaceRoot); - if (!filePaths) { - return fail( - requestId, - "browser_unsupported", - "browser_upload only accepts files inside the agent workspace.", - ); - } - - await target.contents.sendDebugCommand("DOM.setFileInputFiles", { - nodeId: queried.nodeId, - files: filePaths, - }); - return { - requestId, - ok: true, - result: { - command: "upload", + return withDialogCapture(target.contents, async () => { + if (!target.contents.sendDebugCommand) { + return fail(requestId, "browser_unsupported", "browser_upload requires CDP"); + } + const expression = snapshotEngine.runtimeElementExpression({ browserId: target.browserId, ref: input.ref, - filePaths, - }, - }; + }); + if (typeof expression !== "string") { + return staleRefFailure(requestId, input.ref); + } + const evaluated = (await target.contents.sendDebugCommand("Runtime.evaluate", { + expression, + objectGroup: "paseo-browser-automation", + returnByValue: false, + })) as CdpRuntimeEvaluateResult; + const objectId = evaluated.result?.objectId; + if (!objectId || evaluated.result?.subtype === "null") { + return staleRefFailure(requestId, input.ref); + } + const described = (await target.contents.sendDebugCommand("DOM.describeNode", { + objectId, + })) as CdpDescribeNodeResult; + const backendNodeId = described.node?.backendNodeId; + if (typeof backendNodeId !== "number" || backendNodeId <= 0) { + return staleRefFailure(requestId, input.ref); + } + const workspaceRoot = resolveUploadWorkspaceRoot(cwd); + if (!workspaceRoot) { + return fail(requestId, "browser_unsupported", "browser_upload requires request cwd"); + } + const filePaths = resolveWorkspaceFilePaths(input.filePaths, workspaceRoot); + if (!filePaths) { + return fail( + requestId, + "browser_unsupported", + "browser_upload only accepts files inside the agent workspace.", + ); + } + + await target.contents.sendDebugCommand("DOM.setFileInputFiles", { + backendNodeId, + files: filePaths, + }); + return { + requestId, + ok: true, + result: { + command: "upload", + browserId: target.browserId, + ref: input.ref, + filePaths, + }, + }; + }); } function resolveUploadWorkspaceRoot(cwd: string | undefined): string | null { @@ -1064,6 +1464,111 @@ function parseNetworkEntries(value: unknown): BrowserAutomationNetworkLogEntry[] }); } +type EvaluateScriptResult = + | { status: "ok"; resultJson: string; truncated: boolean } + | { status: "stale_ref" } + | { status: "error"; message: string }; + +function readEvaluateScriptResult(value: unknown): EvaluateScriptResult { + if (!value || typeof value !== "object") { + return { status: "error", message: "Browser evaluate returned an invalid result." }; + } + const record = value as Record; + if (record.ok === true && typeof record.resultJson === "string") { + return { status: "ok", resultJson: record.resultJson, truncated: record.truncated === true }; + } + if (record.staleRef === true) { + return { status: "stale_ref" }; + } + if (typeof record.error === "string") { + return { status: "error", message: record.error }; + } + return { status: "error", message: "Browser evaluate returned an invalid result." }; +} + +function capEvaluateResultJson(resultJson: string): { resultJson: string; truncated: boolean } { + if (resultJson.length <= MAX_EVALUATE_RESULT_JSON_LENGTH) { + return { resultJson, truncated: false }; + } + return { + resultJson: JSON.stringify(resultJson.slice(0, MAX_EVALUATE_RESULT_PREVIEW_LENGTH)), + truncated: true, + }; +} + +function evaluateErrorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return capEvaluateErrorMessage(message); +} + +function capEvaluateErrorMessage(message: string): string { + return message.length <= MAX_EVALUATE_ERROR_MESSAGE_LENGTH + ? message + : message.slice(0, MAX_EVALUATE_ERROR_MESSAGE_LENGTH); +} + +function buildEvaluateScript( + functionSource: string, + elementExpression: string | undefined, +): string { + return String.raw`(async () => { + const __PASEO_BROWSER_EVALUATE__ = true; + try { + const userFunction = (0, eval)(${JSON.stringify(`(${functionSource})`)}); + if (typeof userFunction !== 'function') { + throw new Error('browser_evaluate input must evaluate to a function.'); + } + const args = []; + ${ + elementExpression + ? `const element = ${elementExpression}; + if (!element) return { staleRef: true }; + args.push(element);` + : "" + } + const value = await userFunction(...args); + const resultJson = JSON.stringify(value) ?? 'null'; + if (resultJson.length <= ${MAX_EVALUATE_RESULT_JSON_LENGTH}) { + return { ok: true, resultJson, truncated: false }; + } + return { + ok: true, + resultJson: JSON.stringify(resultJson.slice(0, ${MAX_EVALUATE_RESULT_PREVIEW_LENGTH})), + truncated: true + }; + } catch (error) { + return { error: error instanceof Error ? error.message : String(error) }; + } + })()`; +} + +async function focusKeypressTarget( + contents: TabContents, + elementExpression: string, +): Promise<"editable" | "focused" | "stale_ref"> { + const result = await contents.executeJavaScript(String.raw`(() => { + const element = ${elementExpression}; + if (!element) return { staleRef: true }; + const tagName = element.tagName ? element.tagName.toLowerCase() : ''; + const inputType = tagName === 'input' ? String(element.getAttribute('type') || 'text').toLowerCase() : ''; + const editableInput = tagName === 'textarea' || + (tagName === 'input' && !['button', 'checkbox', 'color', 'file', 'hidden', 'image', 'radio', 'range', 'reset', 'submit'].includes(inputType)); + const editable = editableInput || element.isContentEditable === true; + if (!editable && typeof element.focus === 'function') { + element.focus({ preventScroll: true }); + } + return { editable }; + })()`); + if (!result || typeof result !== "object") { + return "stale_ref"; + } + const record = result as Record; + if (record.staleRef === true) { + return "stale_ref"; + } + return record.editable === true ? "editable" : "focused"; +} + function readString(value: unknown): string | null { return typeof value === "string" ? value : null; } diff --git a/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts b/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts index eb0331326..969afb000 100644 --- a/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts +++ b/packages/desktop/src/features/browser-automation/snapshot-engine.test.ts @@ -3,79 +3,162 @@ import { BrowserSnapshotEngine, type SnapshotPage } from "./snapshot-engine.js"; class SnapshotFixture implements SnapshotPage { public currentUrl = "https://example.com/form"; - public actionResult: unknown = true; + public actionResult: unknown = { ok: true }; + public alreadyTruncated = false; + public snapshotNodes: unknown[] = [ + { + kind: "role", + role: "heading", + name: "Settings", + tagName: "h1", + attributes: ["level=1"], + children: [], + }, + { kind: "text", text: "Connected as Maya" }, + { + kind: "role", + role: "button", + name: "Save changes", + tagName: "button", + attributes: [], + ref: "@e1", + fingerprint: { + role: "button", + name: "Save changes", + tagName: "button", + type: "", + ariaLabel: "", + }, + children: [], + }, + ]; public getURL(): string { return this.currentUrl; } public async executeJavaScript(code: string): Promise { - if (code.includes("CANDIDATE_SELECTOR")) { - return JSON.stringify([ - { - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, + if (code.includes("__PASEO_ARIA_SNAPSHOT__")) { + return JSON.stringify({ + marker: "__PASEO_ARIA_SNAPSHOT__", + root: { + kind: "role", + role: "document", + name: "Fixture", + tagName: "document", + attributes: [], + children: this.snapshotNodes, }, - { - role: "button", - tagName: "button", - text: "Drop", - selector: "#drop", - attributes: { id: "drop" }, - }, - ]); + refs: [ + { + ref: "@e1", + fingerprint: { + role: "button", + name: "Save changes", + tagName: "button", + type: "", + ariaLabel: "", + }, + }, + ], + truncated: this.alreadyTruncated, + stats: { nodeCount: 4, refCount: 1, textLength: 0, iframeCount: 0, maxDepth: 1 }, + }); } return this.actionResult; } } describe("BrowserSnapshotEngine", () => { - it("treats a false result from a ref action script as a stale ref", async () => { + it("renders a hierarchical ARIA YAML snapshot with static text and actionable refs", async () => { const page = new SnapshotFixture(); const engine = new BrowserSnapshotEngine(); - await engine.snapshot({ browserId: "browser-1", page }); - page.actionResult = false; - - await expect(engine.click({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({ - ok: false, - reason: "stale_ref", - }); - await expect( - engine.select({ browserId: "browser-1", page, ref: "@e1", value: "us" }), - ).resolves.toEqual({ - ok: false, - reason: "stale_ref", + await expect(engine.snapshot({ browserId: "browser-1", page })).resolves.toEqual({ + format: "aria-yaml", + snapshot: [ + '- document "Fixture"', + ' - heading "Settings" [level=1]', + ' - text: "Connected as Maya"', + ' - button "Save changes" [ref=@e1]', + ].join("\n"), + truncated: false, + stats: { nodeCount: 4, refCount: 1, textLength: 119, iframeCount: 0, maxDepth: 1 }, }); }); - it("treats a false result from optional ref text/key actions as a stale ref", async () => { + it("builds a runtime ref expression with the snapshot fingerprint", async () => { const page = new SnapshotFixture(); const engine = new BrowserSnapshotEngine(); await engine.snapshot({ browserId: "browser-1", page }); - page.actionResult = false; + page.currentUrl = "https://example.com/form?panel=advanced"; - await expect( - engine.typeText({ browserId: "browser-1", page, ref: "@e1", text: "Ada" }), - ).resolves.toEqual({ ok: false, reason: "stale_ref" }); - await expect( - engine.keypress({ browserId: "browser-1", page, ref: "@e1", key: "Enter" }), - ).resolves.toEqual({ ok: false, reason: "stale_ref" }); + expect(engine.runtimeElementExpression({ browserId: "browser-1", ref: "@e1" })).toContain( + '"name":"Save changes"', + ); }); - it("treats a false result from drag as a stale ref", async () => { + it("treats missing host-side ref metadata as a stale ref", async () => { const page = new SnapshotFixture(); const engine = new BrowserSnapshotEngine(); await engine.snapshot({ browserId: "browser-1", page }); - page.actionResult = false; + expect(engine.runtimeElementExpression({ browserId: "browser-1", ref: "@e2" })).toEqual({ + ok: false, + reason: "missing_ref", + }); + }); - await expect( - engine.drag({ browserId: "browser-1", page, sourceRef: "@e1", targetRef: "@e2" }), - ).resolves.toEqual({ ok: false, reason: "stale_ref" }); + it("marks rendered output truncation explicitly and deterministically", async () => { + const page = new SnapshotFixture(); + page.snapshotNodes = [ + { + kind: "text", + text: "A".repeat(81_000), + }, + ]; + const engine = new BrowserSnapshotEngine(); + + const snapshot = await engine.snapshot({ browserId: "browser-1", page }); + + expect(snapshot.truncated).toBe(true); + expect(snapshot.snapshot.endsWith('- text: "Snapshot truncated."')).toBe(true); + expect(snapshot.stats.textLength).toBeLessThanOrEqual(80_000); + }); + + it("keeps the last rendered node when a short snapshot was capped by node count", async () => { + const page = new SnapshotFixture(); + page.alreadyTruncated = true; + page.snapshotNodes = [ + { + kind: "role", + role: "button", + name: "Final action", + tagName: "button", + attributes: [], + ref: "@e1", + fingerprint: { + role: "button", + name: "Final action", + tagName: "button", + type: "", + ariaLabel: "", + }, + children: [], + }, + ]; + const engine = new BrowserSnapshotEngine(); + + const snapshot = await engine.snapshot({ browserId: "browser-1", page }); + + expect(snapshot.truncated).toBe(true); + expect(snapshot.snapshot).toBe( + [ + '- document "Fixture"', + ' - button "Final action" [ref=@e1]', + '- text: "Snapshot truncated."', + ].join("\n"), + ); }); }); diff --git a/packages/desktop/src/features/browser-automation/snapshot-engine.ts b/packages/desktop/src/features/browser-automation/snapshot-engine.ts index 8e0fe1b7b..2ee1a68a5 100644 --- a/packages/desktop/src/features/browser-automation/snapshot-engine.ts +++ b/packages/desktop/src/features/browser-automation/snapshot-engine.ts @@ -1,24 +1,60 @@ +import { ARIA_SNAPSHOT_SCRIPT, ARIA_SNAPSHOT_SCRIPT_MARKER } from "./aria-snapshot-script.js"; + export interface SnapshotPage { getURL(): string; executeJavaScript(code: string): Promise; } -export interface BrowserSnapshotElement extends RawSnapshotElement { - ref: string; +export interface BrowserAriaSnapshot { + format: "aria-yaml"; + snapshot: string; + truncated: boolean; + stats: BrowserAriaSnapshotStats; } -interface RawSnapshotElement { +export interface BrowserAriaSnapshotStats { + nodeCount: number; + refCount: number; + textLength: number; + iframeCount?: number; + maxDepth?: number; +} + +interface SnapshotNode { + kind: "role" | "text" | "group"; + role?: string; + name?: string; + text?: string; + tagName?: string; + attributes?: string[]; + ref?: string; + fingerprint?: BrowserRefFingerprint; + children?: SnapshotNode[]; +} + +interface BrowserRefFingerprint { role: string; + name: string; tagName: string; - text: string; - selector: string; - attributes: Record; + type: string; + ariaLabel: string; +} + +interface RawAriaSnapshot { + marker: string; + root: SnapshotNode; + refs: BrowserRefMetadata[]; + truncated: boolean; + stats: BrowserAriaSnapshotStats; +} + +interface BrowserRefMetadata { + ref: string; + fingerprint: BrowserRefFingerprint; } interface BrowserRefState { - nextRefNumber: number; - url: string; - refs: Map; + refs: Map; } export type BrowserRefActionResult = @@ -26,45 +62,31 @@ export type BrowserRefActionResult = | { ok: false; reason: "stale_ref" | "missing_ref" }; type BrowserRefFailure = Extract; -type BrowserRefResolveResult = { ok: true; element: RawSnapshotElement } | BrowserRefFailure; +type BrowserRefResolveResult = { ok: true; metadata: BrowserRefMetadata } | BrowserRefFailure; + +const TRUNCATION_MARKER = '- text: "Snapshot truncated."'; +const MAX_RENDERED_TEXT_LENGTH = 80_000; export class BrowserSnapshotEngine { private readonly statesByBrowserId = new Map(); - async snapshot(input: { - browserId: string; - page: SnapshotPage; - }): Promise { - const rawElements = parseRawSnapshotElements( - await input.page.executeJavaScript(SNAPSHOT_SCRIPT), - ); - const state = { - nextRefNumber: 1, - url: input.page.getURL(), - refs: new Map(), - }; - const elements = rawElements.map((element) => { - const ref = `@e${state.nextRefNumber++}`; - state.refs.set(ref, element); - return { - ref, - role: element.role, - tagName: element.tagName, - text: element.text, - selector: element.selector, - attributes: element.attributes, - }; + async snapshot(input: { browserId: string; page: SnapshotPage }): Promise { + const rawSnapshot = parseAriaSnapshot(await input.page.executeJavaScript(ARIA_SNAPSHOT_SCRIPT)); + const rendered = renderSnapshot(rawSnapshot.root); + const capped = capRenderedSnapshot(rendered, rawSnapshot.truncated); + this.statesByBrowserId.set(input.browserId, { + refs: new Map(rawSnapshot.refs.map((ref) => [ref.ref, ref])), }); - this.statesByBrowserId.set(input.browserId, state); - return elements; - } - - async click(input: { - browserId: string; - page: SnapshotPage; - ref: string; - }): Promise { - return this.runRefScript(input, (selector) => buildClickScript(selector)); + return { + format: "aria-yaml", + snapshot: capped.snapshot, + truncated: capped.truncated, + stats: { + ...rawSnapshot.stats, + refCount: rawSnapshot.refs.length, + textLength: capped.snapshot.length, + }, + }; } async fill(input: { @@ -73,39 +95,7 @@ export class BrowserSnapshotEngine { ref: string; value: string; }): Promise { - return this.runRefScript(input, (selector) => buildFillScript(selector, input.value)); - } - - async typeText(input: { - browserId: string; - page: SnapshotPage; - ref?: string; - text: string; - }): Promise { - const selector = this.resolveOptionalRef(input); - if (!selector.ok) { - return selector; - } - const result = await input.page.executeJavaScript( - buildTypeScript(selector.selector, input.text), - ); - return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; - } - - async keypress(input: { - browserId: string; - page: SnapshotPage; - ref?: string; - key: string; - }): Promise { - const selector = this.resolveOptionalRef(input); - if (!selector.ok) { - return selector; - } - const result = await input.page.executeJavaScript( - buildKeypressScript(selector.selector, input.key), - ); - return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; + return this.runRefScript(input, (ref) => buildFillScript(ref, input.value)); } async select(input: { @@ -114,123 +104,214 @@ export class BrowserSnapshotEngine { ref: string; value: string; }): Promise { - return this.runRefScript(input, (selector) => buildSelectScript(selector, input.value)); - } - - async hover(input: { - browserId: string; - page: SnapshotPage; - ref: string; - }): Promise { - return this.runRefScript(input, (selector) => buildHoverScript(selector)); - } - - async drag(input: { - browserId: string; - page: SnapshotPage; - sourceRef: string; - targetRef: string; - }): Promise { - const source = this.resolveRef({ - browserId: input.browserId, - page: input.page, - ref: input.sourceRef, - }); - if (!source.ok) { - return source; - } - const target = this.resolveRef({ - browserId: input.browserId, - page: input.page, - ref: input.targetRef, - }); - if (!target.ok) { - return target; - } - const result = await input.page.executeJavaScript( - buildDragScript(source.element.selector, target.element.selector), - ); - return result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; + return this.runRefScript(input, (ref) => buildSelectScript(ref, input.value)); } clearBrowser(browserId: string): void { this.statesByBrowserId.delete(browserId); } - selectorForRef(input: { - browserId: string; - page: SnapshotPage; - ref: string; - }): { ok: true; selector: string } | BrowserRefFailure { + runtimeElementExpression(input: { browserId: string; ref: string }): string | BrowserRefFailure { const resolved = this.resolveRef(input); if (!resolved.ok) { return resolved; } - return { ok: true, selector: resolved.element.selector }; + return buildRuntimeElementExpression(resolved.metadata); } private async runRefScript( input: { browserId: string; page: SnapshotPage; ref: string }, - buildScript: (selector: string) => string, + buildScript: (ref: BrowserRefMetadata) => string, ): Promise { const resolved = this.resolveRef(input); if (!resolved.ok) { return resolved; } - const result = await input.page.executeJavaScript(buildScript(resolved.element.selector)); - return result === false ? { ok: false, reason: "stale_ref" } : { ok: true }; + const result = await input.page.executeJavaScript(buildScript(resolved.metadata)); + return readActionResult(result, true); } - private resolveRef(input: { - browserId: string; - page: SnapshotPage; - ref: string; - }): BrowserRefResolveResult { + private resolveRef(input: { browserId: string; ref: string }): BrowserRefResolveResult { const state = this.statesByBrowserId.get(input.browserId); - if (!state || state.url !== input.page.getURL()) { + if (!state) { return { ok: false, reason: "stale_ref" }; } - const element = state.refs.get(input.ref); - if (!element) { + const metadata = state.refs.get(input.ref); + if (!metadata) { return { ok: false, reason: "missing_ref" }; } - return { ok: true, element }; - } - - private resolveOptionalRef(input: { - browserId: string; - page: SnapshotPage; - ref?: string; - }): { ok: true; selector: string | undefined } | BrowserRefFailure { - if (!input.ref) { - return { ok: true, selector: undefined }; - } - const resolved = this.resolveRef({ - browserId: input.browserId, - page: input.page, - ref: input.ref, - }); - if (!resolved.ok) { - return resolved; - } - return { ok: true, selector: resolved.element.selector }; + return { ok: true, metadata }; } } -function buildClickScript(selector: string): string { - return String.raw`(() => { - const element = document.querySelector(${JSON.stringify(selector)}); - if (!element) return false; - element.scrollIntoView({ block: 'center', inline: 'center' }); - element.click(); - return true; - })()`; +function readActionResult(value: unknown, staleWhenFalse: boolean): BrowserRefActionResult { + if (value === false && staleWhenFalse) { + return { ok: false, reason: "stale_ref" }; + } + if (isActionFailure(value)) { + return { ok: false, reason: value.reason }; + } + return { ok: true }; } -function buildFillScript(selector: string, value: string): string { +function isActionFailure(value: unknown): value is BrowserRefFailure { + if (!value || typeof value !== "object") { + return false; + } + const reason = (value as Record).reason; + return reason === "stale_ref" || reason === "missing_ref"; +} + +function parseAriaSnapshot(value: unknown): RawAriaSnapshot { + const parsed = typeof value === "string" ? JSON.parse(value) : value; + if (!parsed || typeof parsed !== "object") { + return emptySnapshot(); + } + const record = parsed as Record; + if (record.marker !== ARIA_SNAPSHOT_SCRIPT_MARKER) { + return emptySnapshot(); + } + const root = parseSnapshotNode(record.root); + return { + marker: ARIA_SNAPSHOT_SCRIPT_MARKER, + root: root ?? emptySnapshot().root, + refs: parseRefs(record.refs), + truncated: record.truncated === true, + stats: parseStats(record.stats), + }; +} + +function emptySnapshot(): RawAriaSnapshot { + return { + marker: ARIA_SNAPSHOT_SCRIPT_MARKER, + root: { kind: "role", role: "document", name: "", tagName: "document", children: [] }, + refs: [], + truncated: false, + stats: { nodeCount: 0, refCount: 0, textLength: 0, iframeCount: 0, maxDepth: 0 }, + }; +} + +function parseSnapshotNode(value: unknown): SnapshotNode | null { + if (!value || typeof value !== "object") { + return null; + } + const record = value as Record; + const kind = record.kind; + if (kind !== "role" && kind !== "text" && kind !== "group") { + return null; + } + return { + kind, + ...(readString(record.role) ? { role: readString(record.role) ?? undefined } : {}), + ...(readString(record.name) ? { name: readString(record.name) ?? undefined } : {}), + ...(readString(record.text) ? { text: readString(record.text) ?? undefined } : {}), + ...(readString(record.tagName) ? { tagName: readString(record.tagName) ?? undefined } : {}), + ...(readString(record.ref) ? { ref: readString(record.ref) ?? undefined } : {}), + ...(parseFingerprint(record.fingerprint) + ? { fingerprint: parseFingerprint(record.fingerprint) ?? undefined } + : {}), + attributes: readStringArray(record.attributes), + children: Array.isArray(record.children) + ? record.children.flatMap((child): SnapshotNode[] => { + const parsed = parseSnapshotNode(child); + return parsed ? [parsed] : []; + }) + : [], + }; +} + +function parseRefs(value: unknown): BrowserRefMetadata[] { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap((item): BrowserRefMetadata[] => { + if (!item || typeof item !== "object") { + return []; + } + const record = item as Record; + const ref = readString(record.ref); + const fingerprint = parseFingerprint(record.fingerprint); + return ref && fingerprint ? [{ ref, fingerprint }] : []; + }); +} + +function parseFingerprint(value: unknown): BrowserRefFingerprint | null { + if (!value || typeof value !== "object") { + return null; + } + const record = value as Record; + const role = readString(record.role); + const name = readString(record.name); + const tagName = readString(record.tagName); + const type = readString(record.type); + const ariaLabel = readString(record.ariaLabel); + if (role === null || name === null || tagName === null || type === null || ariaLabel === null) { + return null; + } + return { role, name, tagName, type, ariaLabel }; +} + +function parseStats(value: unknown): BrowserAriaSnapshotStats { + if (!value || typeof value !== "object") { + return { nodeCount: 0, refCount: 0, textLength: 0 }; + } + const record = value as Record; + return { + nodeCount: readNumber(record.nodeCount) ?? 0, + refCount: readNumber(record.refCount) ?? 0, + textLength: readNumber(record.textLength) ?? 0, + ...(readNumber(record.iframeCount) !== null + ? { iframeCount: readNumber(record.iframeCount) ?? undefined } + : {}), + ...(readNumber(record.maxDepth) !== null + ? { maxDepth: readNumber(record.maxDepth) ?? undefined } + : {}), + }; +} + +function renderSnapshot(root: SnapshotNode): string { + const lines = renderNode(root, 0); + return lines.length > 0 ? lines.join("\n") : "- document"; +} + +function renderNode(node: SnapshotNode, depth: number): string[] { + if (node.kind === "group") { + return (node.children ?? []).flatMap((child) => renderNode(child, depth)); + } + const indent = " ".repeat(depth); + if (node.kind === "text") { + return [`${indent}- text: ${JSON.stringify(node.text ?? "")}`]; + } + const attrs = [...(node.attributes ?? [])]; + if (node.ref) { + attrs.push(`ref=${node.ref}`); + } + const suffix = attrs.length > 0 ? ` [${attrs.join(" ")}]` : ""; + const ownLine = `${indent}- ${node.role ?? "generic"}${node.name ? ` ${JSON.stringify(node.name)}` : ""}${suffix}`; + const childLines = (node.children ?? []).flatMap((child) => renderNode(child, depth + 1)); + return [ownLine, ...childLines]; +} + +function capRenderedSnapshot( + snapshot: string, + alreadyTruncated: boolean, +): { snapshot: string; truncated: boolean } { + if (snapshot.length <= MAX_RENDERED_TEXT_LENGTH && !alreadyTruncated) { + return { snapshot, truncated: false }; + } + if (snapshot.length + 1 + TRUNCATION_MARKER.length <= MAX_RENDERED_TEXT_LENGTH) { + return { snapshot: `${snapshot}\n${TRUNCATION_MARKER}`, truncated: true }; + } + const availableLength = MAX_RENDERED_TEXT_LENGTH - TRUNCATION_MARKER.length - 1; + const capped = snapshot.slice(0, Math.max(0, availableLength)).replace(/\n[^\n]*$/, ""); + return { snapshot: `${capped}\n${TRUNCATION_MARKER}`, truncated: true }; +} + +function buildFillScript(metadata: BrowserRefMetadata, value: string): string { return String.raw`(() => { - const element = document.querySelector(${JSON.stringify(selector)}); - if (!element) return false; + const resolved = ${buildResolveExpression(metadata)}; + if (!resolved.ok) return resolved; + const element = resolved.element; element.scrollIntoView({ block: 'center', inline: 'center' }); element.focus(); const nextValue = ${JSON.stringify(value)}; @@ -238,51 +319,19 @@ function buildFillScript(selector: string, value: string): string { element.value = nextValue; element.dispatchEvent(new Event('input', { bubbles: true })); element.dispatchEvent(new Event('change', { bubbles: true })); - return true; + return { ok: true }; } element.textContent = nextValue; element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: nextValue })); - return true; + return { ok: true }; })()`; } -function buildTypeScript(selector: string | undefined, text: string): string { +function buildSelectScript(metadata: BrowserRefMetadata, value: string): string { return String.raw`(() => { - const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"}; - if (!element) return false; - element.scrollIntoView?.({ block: 'center', inline: 'center' }); - element.focus?.(); - const text = ${JSON.stringify(text)}; - if ('value' in element) { - element.value = String(element.value || '') + text; - element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })); - element.dispatchEvent(new Event('change', { bubbles: true })); - return true; - } - element.textContent = String(element.textContent || '') + text; - element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text })); - return true; - })()`; -} - -function buildKeypressScript(selector: string | undefined, key: string): string { - return String.raw`(() => { - const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"}; - if (!element) return false; - element.focus?.(); - const key = ${JSON.stringify(key)}; - const eventInit = { bubbles: true, cancelable: true, key }; - element.dispatchEvent(new KeyboardEvent('keydown', eventInit)); - element.dispatchEvent(new KeyboardEvent('keypress', eventInit)); - element.dispatchEvent(new KeyboardEvent('keyup', eventInit)); - return true; - })()`; -} - -function buildSelectScript(selector: string, value: string): string { - return String.raw`(() => { - const element = document.querySelector(${JSON.stringify(selector)}); - if (!element) return false; + const resolved = ${buildResolveExpression(metadata)}; + if (!resolved.ok) return resolved; + const element = resolved.element; element.scrollIntoView?.({ block: 'center', inline: 'center' }); element.focus?.(); const nextValue = ${JSON.stringify(value)}; @@ -290,205 +339,34 @@ function buildSelectScript(selector: string, value: string): string { element.value = nextValue; element.dispatchEvent(new Event('input', { bubbles: true })); element.dispatchEvent(new Event('change', { bubbles: true })); - return true; + return { ok: true }; } return false; })()`; } -function buildHoverScript(selector: string): string { +function buildRuntimeElementExpression(metadata: BrowserRefMetadata): string { return String.raw`(() => { - const element = document.querySelector(${JSON.stringify(selector)}); - if (!element) return false; - element.scrollIntoView?.({ block: 'center', inline: 'center' }); - const rect = element.getBoundingClientRect(); - const eventInit = { - bubbles: true, - cancelable: true, - clientX: rect.left + rect.width / 2, - clientY: rect.top + rect.height / 2, - screenX: window.screenX + rect.left + rect.width / 2, - screenY: window.screenY + rect.top + rect.height / 2, - view: window, - }; - element.dispatchEvent(new MouseEvent('mouseover', eventInit)); - element.dispatchEvent(new MouseEvent('mouseenter', eventInit)); - element.dispatchEvent(new MouseEvent('mousemove', eventInit)); - return true; + const resolved = ${buildResolveExpression(metadata)}; + if (!resolved.ok) return null; + return resolved.element; })()`; } -function buildDragScript(sourceSelector: string, targetSelector: string): string { - return String.raw`(() => { - const source = document.querySelector(${JSON.stringify(sourceSelector)}); - const target = document.querySelector(${JSON.stringify(targetSelector)}); - if (!source || !target) return false; - source.scrollIntoView?.({ block: 'center', inline: 'center' }); - target.scrollIntoView?.({ block: 'center', inline: 'center' }); - const data = new DataTransfer(); - const sourceRect = source.getBoundingClientRect(); - const targetRect = target.getBoundingClientRect(); - function eventInit(rect) { - return { - bubbles: true, - cancelable: true, - clientX: rect.left + rect.width / 2, - clientY: rect.top + rect.height / 2, - screenX: window.screenX + rect.left + rect.width / 2, - screenY: window.screenY + rect.top + rect.height / 2, - dataTransfer: data, - view: window, - }; - } - source.dispatchEvent(new MouseEvent('mousedown', eventInit(sourceRect))); - source.dispatchEvent(new DragEvent('dragstart', eventInit(sourceRect))); - target.dispatchEvent(new DragEvent('dragenter', eventInit(targetRect))); - target.dispatchEvent(new DragEvent('dragover', eventInit(targetRect))); - target.dispatchEvent(new DragEvent('drop', eventInit(targetRect))); - source.dispatchEvent(new DragEvent('dragend', eventInit(sourceRect))); - target.dispatchEvent(new MouseEvent('mouseup', eventInit(targetRect))); - return true; - })()`; -} - -function parseRawSnapshotElements(value: unknown): RawSnapshotElement[] { - const parsed = typeof value === "string" ? JSON.parse(value) : value; - if (!Array.isArray(parsed)) { - return []; - } - return parsed.flatMap((item): RawSnapshotElement[] => { - if (!item || typeof item !== "object") { - return []; - } - const record = item as Record; - const selector = readString(record.selector); - if (!selector) { - return []; - } - return [ - { - role: readString(record.role) || "generic", - tagName: (readString(record.tagName) || "element").toLowerCase(), - text: readString(record.text) || "", - selector, - attributes: readAttributes(record.attributes), - }, - ]; - }); +function buildResolveExpression(metadata: BrowserRefMetadata): string { + return `window.__PASEO_BROWSER_AUTOMATION__?.resolve(${JSON.stringify(metadata.ref)}, ${JSON.stringify(metadata.fingerprint)}) ?? { ok: false, reason: 'stale_ref' }`; } function readString(value: unknown): string | null { return typeof value === "string" ? value : null; } -function readAttributes(value: unknown): Record { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return {}; - } - const result: Record = {}; - for (const [key, attributeValue] of Object.entries(value)) { - if (typeof attributeValue === "string") { - result[key] = attributeValue; - } - } - return result; +function readStringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((item): item is string => typeof item === "string") + : []; } -const SNAPSHOT_SCRIPT = String.raw`(() => { - const MAX_ELEMENTS = 200; - const CANDIDATE_SELECTOR = [ - 'a[href]', - 'button', - 'input', - 'textarea', - 'select', - 'summary', - '[role]', - '[tabindex]:not([tabindex="-1"])', - '[contenteditable=""]', - '[contenteditable="true"]' - ].join(','); - - function cssEscape(value) { - if (window.CSS && typeof window.CSS.escape === 'function') { - return window.CSS.escape(value); - } - return String(value).replace(/[^a-zA-Z0-9_-]/g, '\\$&'); - } - - function isVisible(element) { - const style = window.getComputedStyle(element); - if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { - return false; - } - const rect = element.getBoundingClientRect(); - return rect.width > 0 && rect.height > 0; - } - - function roleFor(element) { - const explicit = element.getAttribute('role'); - if (explicit) return explicit; - const tag = element.tagName.toLowerCase(); - if (tag === 'a') return 'link'; - if (tag === 'button') return 'button'; - if (tag === 'select') return 'combobox'; - if (tag === 'textarea') return 'textbox'; - if (tag === 'summary') return 'button'; - if (tag === 'input') { - const type = (element.getAttribute('type') || 'text').toLowerCase(); - if (type === 'checkbox') return 'checkbox'; - if (type === 'radio') return 'radio'; - if (type === 'button' || type === 'submit' || type === 'reset') return 'button'; - return 'textbox'; - } - return 'generic'; - } - - function textFor(element) { - const tag = element.tagName.toLowerCase(); - const pieces = [ - element.getAttribute('aria-label'), - element.getAttribute('alt'), - element.getAttribute('title'), - tag === 'input' ? element.getAttribute('placeholder') : null, - tag === 'input' || tag === 'textarea' ? element.value : null, - element.innerText, - element.textContent - ]; - const text = pieces.find((piece) => typeof piece === 'string' && piece.trim().length > 0); - return (text || '').replace(/\s+/g, ' ').trim().slice(0, 300); - } - - function selectorFor(element) { - if (element.id) return '#' + cssEscape(element.id); - const parts = []; - let current = element; - while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) { - const tag = current.tagName.toLowerCase(); - const parent = current.parentElement; - if (!parent) break; - const siblings = Array.from(parent.children).filter((sibling) => sibling.tagName === current.tagName); - const index = siblings.indexOf(current) + 1; - parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag); - current = parent; - } - return parts.length > 0 ? parts.join(' > ') : element.tagName.toLowerCase(); - } - - return JSON.stringify(Array.from(document.querySelectorAll(CANDIDATE_SELECTOR)) - .filter(isVisible) - .slice(0, MAX_ELEMENTS) - .map((element) => ({ - role: roleFor(element), - tagName: element.tagName.toLowerCase(), - text: textFor(element), - selector: selectorFor(element), - attributes: { - ...(element.id ? { id: element.id } : {}), - ...(element.getAttribute('name') ? { name: element.getAttribute('name') } : {}), - ...(element.getAttribute('type') ? { type: element.getAttribute('type') } : {}), - ...(element.getAttribute('href') ? { href: element.getAttribute('href') } : {}), - ...(element.getAttribute('aria-label') ? { 'aria-label': element.getAttribute('aria-label') } : {}) - } - }))); -})()`; +function readNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} diff --git a/packages/desktop/src/features/browser-automation/trusted-input.test.ts b/packages/desktop/src/features/browser-automation/trusted-input.test.ts new file mode 100644 index 000000000..9389eb256 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/trusted-input.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, test } from "vitest"; +import { dispatchTrustedKey } from "./trusted-input.js"; + +describe("trusted browser input", () => { + test("Space dispatches a real space key event", async () => { + const commands: Array<{ command: string; params?: Record }> = []; + + await dispatchTrustedKey(async (command, params) => { + commands.push({ command, ...(params ? { params } : {}) }); + return {}; + }, "Space"); + + expect(commands).toEqual([ + { + command: "Input.dispatchKeyEvent", + params: { + type: "keyDown", + key: " ", + code: "Space", + windowsVirtualKeyCode: 32, + nativeVirtualKeyCode: 32, + text: " ", + unmodifiedText: " ", + }, + }, + { + command: "Input.dispatchKeyEvent", + params: { + type: "keyUp", + key: " ", + code: "Space", + windowsVirtualKeyCode: 32, + nativeVirtualKeyCode: 32, + }, + }, + ]); + }); +}); diff --git a/packages/desktop/src/features/browser-automation/trusted-input.ts b/packages/desktop/src/features/browser-automation/trusted-input.ts new file mode 100644 index 000000000..0391cc25b --- /dev/null +++ b/packages/desktop/src/features/browser-automation/trusted-input.ts @@ -0,0 +1,217 @@ +import type { ActionablePoint } from "./actionability.js"; +import type { CdpCommandSender } from "./cdp-session-queue.js"; + +export type MouseButton = "left" | "right" | "middle"; +export type InputModifier = "Alt" | "Control" | "Meta" | "Shift"; + +export interface ClickInputOptions { + button?: MouseButton; + doubleClick?: boolean; + modifiers?: InputModifier[]; +} + +const MODIFIER_MASKS: Record = { + Alt: 1, + Control: 2, + Meta: 4, + Shift: 8, +}; + +const SPECIAL_KEY_DEFINITIONS: Record< + string, + { key: string; code: string; windowsVirtualKeyCode: number; text?: string } +> = { + Enter: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13, text: "\r" }, + Space: { key: " ", code: "Space", windowsVirtualKeyCode: 32, text: " " }, + Tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9, text: "\t" }, + Escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 }, + Backspace: { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 }, + Delete: { key: "Delete", code: "Delete", windowsVirtualKeyCode: 46 }, + ArrowUp: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 }, + ArrowDown: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 }, + ArrowLeft: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 }, + ArrowRight: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 }, + Home: { key: "Home", code: "Home", windowsVirtualKeyCode: 36 }, + End: { key: "End", code: "End", windowsVirtualKeyCode: 35 }, + PageUp: { key: "PageUp", code: "PageUp", windowsVirtualKeyCode: 33 }, + PageDown: { key: "PageDown", code: "PageDown", windowsVirtualKeyCode: 34 }, +}; + +export async function dispatchTrustedClick( + send: CdpCommandSender, + point: ActionablePoint, + options: ClickInputOptions = {}, +): Promise { + const button = options.button ?? "left"; + const modifiers = modifierMask(options.modifiers); + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + button: "none", + modifiers, + }); + if (options.doubleClick) { + await dispatchTrustedMouseClick(send, point, button, modifiers, 1); + await dispatchTrustedMouseClick(send, point, button, modifiers, 2); + return; + } + await dispatchTrustedMouseClick(send, point, button, modifiers, 1); +} + +async function dispatchTrustedMouseClick( + send: CdpCommandSender, + point: ActionablePoint, + button: MouseButton, + modifiers: number, + clickCount: number, +): Promise { + await send("Input.dispatchMouseEvent", { + type: "mousePressed", + x: point.x, + y: point.y, + button, + buttons: mouseButtonMask(button), + clickCount, + modifiers, + }); + await send("Input.dispatchMouseEvent", { + type: "mouseReleased", + x: point.x, + y: point.y, + button, + buttons: 0, + clickCount, + modifiers, + }); +} + +export async function dispatchTrustedHover( + send: CdpCommandSender, + point: ActionablePoint, +): Promise { + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: point.x, + y: point.y, + button: "none", + }); +} + +export async function dispatchTrustedDrag( + send: CdpCommandSender, + source: ActionablePoint, + target: ActionablePoint, +): Promise { + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: source.x, + y: source.y, + button: "none", + }); + await send("Input.dispatchMouseEvent", { + type: "mousePressed", + x: source.x, + y: source.y, + button: "left", + buttons: 1, + clickCount: 1, + }); + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: (source.x + target.x) / 2, + y: (source.y + target.y) / 2, + button: "left", + buttons: 1, + }); + await send("Input.dispatchMouseEvent", { + type: "mouseMoved", + x: target.x, + y: target.y, + button: "left", + buttons: 1, + }); + await send("Input.dispatchMouseEvent", { + type: "mouseReleased", + x: target.x, + y: target.y, + button: "left", + buttons: 0, + clickCount: 1, + }); +} + +export async function dispatchTrustedScroll( + send: CdpCommandSender, + point: ActionablePoint, + deltaX: number, + deltaY: number, +): Promise { + await send("Input.dispatchMouseEvent", { + type: "mouseWheel", + x: point.x, + y: point.y, + deltaX, + deltaY, + }); +} + +export async function dispatchTrustedText(send: CdpCommandSender, text: string): Promise { + if (text.length === 0) { + return; + } + await send("Input.insertText", { text }); +} + +export async function dispatchTrustedKey(send: CdpCommandSender, key: string): Promise { + const definition = keyDefinition(key); + await send("Input.dispatchKeyEvent", { + type: "keyDown", + key: definition.key, + code: definition.code, + windowsVirtualKeyCode: definition.windowsVirtualKeyCode, + nativeVirtualKeyCode: definition.windowsVirtualKeyCode, + ...(definition.text ? { text: definition.text, unmodifiedText: definition.text } : {}), + }); + await send("Input.dispatchKeyEvent", { + type: "keyUp", + key: definition.key, + code: definition.code, + windowsVirtualKeyCode: definition.windowsVirtualKeyCode, + nativeVirtualKeyCode: definition.windowsVirtualKeyCode, + }); +} + +function keyDefinition(key: string): { + key: string; + code: string; + windowsVirtualKeyCode: number; + text?: string; +} { + const special = SPECIAL_KEY_DEFINITIONS[key]; + if (special) { + return special; + } + const text = key.length === 1 ? key : ""; + const upper = text.toUpperCase(); + return { + key, + code: upper ? `Key${upper}` : key, + windowsVirtualKeyCode: upper ? upper.charCodeAt(0) : 0, + ...(text ? { text, unmodifiedText: text } : {}), + }; +} + +function modifierMask(modifiers: InputModifier[] | undefined): number { + return (modifiers ?? []).reduce((mask, modifier) => mask | MODIFIER_MASKS[modifier], 0); +} + +function mouseButtonMask(button: MouseButton): number { + if (button === "right") { + return 2; + } + if (button === "middle") { + return 4; + } + return 1; +} diff --git a/packages/desktop/src/features/browser-webviews/index.ts b/packages/desktop/src/features/browser-webviews/index.ts index 6efbe8f93..ff5464994 100644 --- a/packages/desktop/src/features/browser-webviews/index.ts +++ b/packages/desktop/src/features/browser-webviews/index.ts @@ -70,6 +70,10 @@ export function registerPaseoBrowserWorkspace(input: BrowserWorkspaceRegistratio browserRegistry.registerWorkspace(input); } +export function unregisterPaseoBrowser(browserId: string): void { + browserRegistry.unregisterBrowser(browserId); +} + export function getPaseoBrowserWorkspaceId(browserId: string): string | null { return browserRegistry.getWorkspaceId(browserId); } diff --git a/packages/desktop/src/features/browser-webviews/registry.ts b/packages/desktop/src/features/browser-webviews/registry.ts index 93f921280..cdadd643c 100644 --- a/packages/desktop/src/features/browser-webviews/registry.ts +++ b/packages/desktop/src/features/browser-webviews/registry.ts @@ -51,6 +51,16 @@ export class PaseoBrowserWebviewRegistry { this.workspaceIdsByBrowserId.set(input.browserId, input.workspaceId); } + public unregisterBrowser(browserId: string): void { + const webContentsId = this.webContentsIdsByBrowserId.get(browserId) ?? null; + if (webContentsId !== null) { + this.browserIdsByWebContentsId.delete(webContentsId); + this.webContentsIdsByBrowserId.delete(browserId); + } + this.workspaceIdsByBrowserId.delete(browserId); + this.deleteActiveBrowserReferences(browserId); + } + public getWorkspaceId(browserId: string): string | null { return this.workspaceIdsByBrowserId.get(browserId) ?? null; } diff --git a/packages/desktop/src/main.ts b/packages/desktop/src/main.ts index fcad58da0..fd4ffb7fd 100644 --- a/packages/desktop/src/main.ts +++ b/packages/desktop/src/main.ts @@ -54,6 +54,7 @@ import { listRegisteredPaseoBrowserIds, readBrowserIdFromWebviewAttach, registerBrowserWebviewNavigationGuards, + unregisterPaseoBrowser, registerPaseoBrowserWorkspace, registerPaseoBrowserWebContents, setWorkspaceActivePaseoBrowserId, @@ -328,6 +329,12 @@ ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: un } }); +ipcMain.handle("paseo:browser:unregister-workspace-browser", (_event, browserId: unknown) => { + if (typeof browserId === "string" && browserId.trim().length > 0) { + unregisterPaseoBrowser(browserId.trim()); + } +}); + ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput: unknown) => { const input = readActiveBrowserInput(rawInput); if (input) { diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index 02fe9dc0e..84e5dc8bd 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -76,6 +76,8 @@ contextBridge.exposeInMainWorld("paseoDesktop", { browser: { registerWorkspaceBrowser: (input: { browserId: string; workspaceId: string }) => ipcRenderer.invoke("paseo:browser:register-workspace-browser", input), + unregisterWorkspaceBrowser: (browserId: string) => + ipcRenderer.invoke("paseo:browser:unregister-workspace-browser", browserId), setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) => ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input), openDevTools: (browserId: string) => diff --git a/packages/protocol/src/browser-automation/rpc-schemas.test.ts b/packages/protocol/src/browser-automation/rpc-schemas.test.ts index 39e9b2ac8..a47f248b8 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.test.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.test.ts @@ -15,7 +15,39 @@ const commandParseCases = [ { name: "click", command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } }, - expected: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } }, + expected: { + command: "click", + args: { + browserId: BROWSER_ID, + ref: "@e1", + button: "left", + doubleClick: false, + modifiers: [], + }, + }, + }, + { + name: "click options", + command: { + command: "click", + args: { + browserId: BROWSER_ID, + ref: "@e1", + button: "right", + doubleClick: true, + modifiers: ["Meta", "Shift"], + }, + }, + expected: { + command: "click", + args: { + browserId: BROWSER_ID, + ref: "@e1", + button: "right", + doubleClick: true, + modifiers: ["Meta", "Shift"], + }, + }, }, { name: "fill", @@ -123,6 +155,72 @@ const commandParseCases = [ command: { command: "logs", args: { browserId: BROWSER_ID } }, expected: { command: "logs", args: { browserId: BROWSER_ID, maxEntries: 50 } }, }, + { + name: "evaluate", + command: { + command: "evaluate", + args: { browserId: BROWSER_ID, function: "() => document.title" }, + }, + expected: { + command: "evaluate", + args: { browserId: BROWSER_ID, function: "() => document.title" }, + }, + }, + { + name: "evaluate with ref", + command: { + command: "evaluate", + args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" }, + }, + expected: { + command: "evaluate", + args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" }, + }, + }, + { + name: "scroll", + command: { + command: "scroll", + args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 }, + }, + expected: { + command: "scroll", + args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 }, + }, + }, + { + name: "scroll with ref", + command: { + command: "scroll", + args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 10, deltaY: -20 }, + }, + expected: { + command: "scroll", + args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 10, deltaY: -20 }, + }, + }, + { + name: "resize", + command: { + command: "resize", + args: { browserId: BROWSER_ID, width: 1024, height: 768 }, + }, + expected: { + command: "resize", + args: { browserId: BROWSER_ID, width: 1024, height: 768 }, + }, + }, + { + name: "close_tab", + command: { + command: "close_tab", + args: { browserId: BROWSER_ID }, + }, + expected: { + command: "close_tab", + args: { browserId: BROWSER_ID }, + }, + }, ] as const; const resultParseCases = [ @@ -134,16 +232,10 @@ const resultParseCases = [ workspaceId: "workspace-1", url: "https://example.com/form", title: "Fixture", - elements: [ - { - ref: "@e1", - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, - }, - ], + format: "aria-yaml", + snapshot: '- document "Fixture"\n - textbox "Name" [ref=@e1]', + truncated: false, + stats: { nodeCount: 2, refCount: 1, textLength: 52, iframeCount: 0, maxDepth: 1 }, }, expected: { command: "snapshot", @@ -151,16 +243,10 @@ const resultParseCases = [ workspaceId: "workspace-1", url: "https://example.com/form", title: "Fixture", - elements: [ - { - ref: "@e1", - role: "textbox", - tagName: "input", - text: "Name", - selector: "#name", - attributes: { id: "name", type: "text" }, - }, - ], + format: "aria-yaml", + snapshot: '- document "Fixture"\n - textbox "Name" [ref=@e1]', + truncated: false, + stats: { nodeCount: 2, refCount: 1, textLength: 52, iframeCount: 0, maxDepth: 1 }, }, }, { @@ -305,6 +391,68 @@ const resultParseCases = [ ], }, }, + { + name: "evaluate", + result: { + command: "evaluate", + browserId: BROWSER_ID, + resultJson: '{"title":"Fixture"}', + truncated: false, + }, + expected: { + command: "evaluate", + browserId: BROWSER_ID, + resultJson: '{"title":"Fixture"}', + truncated: false, + }, + }, + { + name: "scroll", + result: { + command: "scroll", + browserId: BROWSER_ID, + ref: "@e1", + deltaX: 10, + deltaY: 400, + x: 40, + y: 30, + }, + expected: { + command: "scroll", + browserId: BROWSER_ID, + ref: "@e1", + deltaX: 10, + deltaY: 400, + x: 40, + y: 30, + }, + }, + { + name: "resize", + result: { + command: "resize", + browserId: BROWSER_ID, + width: 1024, + height: 768, + }, + expected: { + command: "resize", + browserId: BROWSER_ID, + width: 1024, + height: 768, + }, + }, + { + name: "close_tab", + result: { + command: "close_tab", + browserId: BROWSER_ID, + }, + expected: { + command: "close_tab", + browserId: BROWSER_ID, + }, + }, ] as const; describe("browser automation execute RPC schemas", () => { @@ -526,7 +674,10 @@ describe("browser automation execute RPC schemas", () => { workspaceId: "workspace-1", url: "https://example.com", title: "Example", - elements: [], + format: "aria-yaml", + snapshot: "- document", + truncated: false, + stats: { nodeCount: 1, refCount: 0, textLength: 10 }, }, }, }); @@ -557,6 +708,53 @@ describe("browser automation execute RPC schemas", () => { }, ); + test("success responses can report handled dialogs", () => { + const parsed = BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-click", + ok: true, + result: { command: "click", browserId: BROWSER_ID, ref: "@e1" }, + dialogs: [ + { + type: "alert", + message: "Saved", + action: "accepted", + timestamp: 123, + }, + { + type: "prompt", + message: "Name?", + defaultValue: "Maya", + action: "dismissed", + timestamp: 124, + }, + ], + }, + }); + + expect(parsed.payload).toEqual({ + requestId: "req-click", + ok: true, + result: { command: "click", browserId: BROWSER_ID, ref: "@e1" }, + dialogs: [ + { + type: "alert", + message: "Saved", + action: "accepted", + timestamp: 123, + }, + { + type: "prompt", + message: "Name?", + defaultValue: "Maya", + action: "dismissed", + timestamp: 124, + }, + ], + }); + }); + test("error responses keep stable codes, messages, and retry defaults", () => { const parsed = BrowserAutomationExecuteResponseSchema.parse({ type: "browser.automation.execute.response", @@ -580,4 +778,44 @@ describe("browser automation execute RPC schemas", () => { }, }); }); + + test("failure responses can report handled dialogs", () => { + const parsed = BrowserAutomationExecuteResponseSchema.parse({ + type: "browser.automation.execute.response", + payload: { + requestId: "req-navigate", + ok: false, + error: { + code: "browser_timeout", + message: "Timed out waiting for browser URL: /next", + }, + dialogs: [ + { + type: "beforeunload", + message: "Leave site?", + action: "dismissed", + timestamp: 200, + }, + ], + }, + }); + + expect(parsed.payload).toEqual({ + requestId: "req-navigate", + ok: false, + error: { + code: "browser_timeout", + message: "Timed out waiting for browser URL: /next", + retryable: false, + }, + dialogs: [ + { + type: "beforeunload", + message: "Leave site?", + action: "dismissed", + timestamp: 200, + }, + ], + }); + }); }); diff --git a/packages/protocol/src/browser-automation/rpc-schemas.ts b/packages/protocol/src/browser-automation/rpc-schemas.ts index d3cdedd49..55785dd69 100644 --- a/packages/protocol/src/browser-automation/rpc-schemas.ts +++ b/packages/protocol/src/browser-automation/rpc-schemas.ts @@ -39,6 +39,10 @@ export const BROWSER_AUTOMATION_COMMAND_NAMES = [ "hover", "drag", "logs", + "evaluate", + "scroll", + "resize", + "close_tab", ] as const; export const BrowserAutomationCommandNameSchema = z.enum(BROWSER_AUTOMATION_COMMAND_NAMES); @@ -55,6 +59,8 @@ const BrowserAutomationTabTargetSchema = z .strict(); const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/); +const BrowserAutomationMouseButtonSchema = z.enum(["left", "right", "middle"]); +const BrowserAutomationInputModifierSchema = z.enum(["Alt", "Control", "Meta", "Shift"]); const BrowserAutomationHttpUrlSchema = z .string() .url() @@ -87,6 +93,9 @@ export const BrowserAutomationClickCommandSchema = z.object({ command: z.literal("click"), args: BrowserAutomationTabTargetSchema.extend({ ref: BrowserAutomationRefSchema, + button: BrowserAutomationMouseButtonSchema.default("left"), + doubleClick: z.boolean().default(false), + modifiers: z.array(BrowserAutomationInputModifierSchema).default([]), }), }); @@ -192,6 +201,36 @@ export const BrowserAutomationLogsCommandSchema = z.object({ }), }); +export const BrowserAutomationEvaluateCommandSchema = z.object({ + command: z.literal("evaluate"), + args: BrowserAutomationTabTargetSchema.extend({ + function: z.string().min(1), + ref: BrowserAutomationRefSchema.optional(), + }), +}); + +export const BrowserAutomationScrollCommandSchema = z.object({ + command: z.literal("scroll"), + args: BrowserAutomationTabTargetSchema.extend({ + ref: BrowserAutomationRefSchema.optional(), + deltaX: z.number(), + deltaY: z.number(), + }), +}); + +export const BrowserAutomationResizeCommandSchema = z.object({ + command: z.literal("resize"), + args: BrowserAutomationTabTargetSchema.extend({ + width: z.number().int().positive(), + height: z.number().int().positive(), + }), +}); + +export const BrowserAutomationCloseTabCommandSchema = z.object({ + command: z.literal("close_tab"), + args: BrowserAutomationTabTargetSchema, +}); + export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [ BrowserAutomationListTabsCommandSchema, BrowserAutomationNewTabCommandSchema, @@ -211,6 +250,10 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [ BrowserAutomationHoverCommandSchema, BrowserAutomationDragCommandSchema, BrowserAutomationLogsCommandSchema, + BrowserAutomationEvaluateCommandSchema, + BrowserAutomationScrollCommandSchema, + BrowserAutomationResizeCommandSchema, + BrowserAutomationCloseTabCommandSchema, ]); export const BrowserAutomationTabInfoSchema = z.object({ @@ -236,14 +279,15 @@ export const BrowserAutomationNewTabResultSchema = z.object({ url: z.string().min(1), }); -export const BrowserAutomationSnapshotElementSchema = z.object({ - ref: z.string().regex(/^@e\d+$/), - role: z.string(), - tagName: z.string(), - text: z.string(), - selector: z.string(), - attributes: z.record(z.string(), z.string()).default({}), -}); +export const BrowserAutomationSnapshotStatsSchema = z + .object({ + nodeCount: z.number().int().nonnegative(), + refCount: z.number().int().nonnegative(), + textLength: z.number().int().nonnegative(), + iframeCount: z.number().int().nonnegative().optional(), + maxDepth: z.number().int().nonnegative().optional(), + }) + .strict(); export const BrowserAutomationSnapshotResultSchema = z.object({ command: z.literal("snapshot"), @@ -251,13 +295,18 @@ export const BrowserAutomationSnapshotResultSchema = z.object({ workspaceId: z.string().min(1).optional(), url: z.string(), title: z.string(), - elements: z.array(BrowserAutomationSnapshotElementSchema), + format: z.literal("aria-yaml"), + snapshot: z.string(), + truncated: z.boolean(), + stats: BrowserAutomationSnapshotStatsSchema, }); export const BrowserAutomationClickResultSchema = z.object({ command: z.literal("click"), browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, + x: z.number().optional(), + y: z.number().optional(), }); export const BrowserAutomationFillResultSchema = z.object({ @@ -276,6 +325,8 @@ export const BrowserAutomationTypeResultSchema = z.object({ command: z.literal("type"), browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema.optional(), + x: z.number().optional(), + y: z.number().optional(), }); export const BrowserAutomationKeypressResultSchema = z.object({ @@ -283,6 +334,8 @@ export const BrowserAutomationKeypressResultSchema = z.object({ browserId: BrowserAutomationBrowserIdSchema, key: z.string().min(1), ref: BrowserAutomationRefSchema.optional(), + x: z.number().optional(), + y: z.number().optional(), }); export const BrowserAutomationNavigateResultSchema = z.object({ @@ -333,6 +386,8 @@ export const BrowserAutomationHoverResultSchema = z.object({ command: z.literal("hover"), browserId: BrowserAutomationBrowserIdSchema, ref: BrowserAutomationRefSchema, + x: z.number().optional(), + y: z.number().optional(), }); export const BrowserAutomationDragResultSchema = z.object({ @@ -340,6 +395,10 @@ export const BrowserAutomationDragResultSchema = z.object({ browserId: BrowserAutomationBrowserIdSchema, sourceRef: BrowserAutomationRefSchema, targetRef: BrowserAutomationRefSchema, + sourceX: z.number().optional(), + sourceY: z.number().optional(), + targetX: z.number().optional(), + targetY: z.number().optional(), }); export const BrowserAutomationConsoleLogEntrySchema = z.object({ @@ -367,6 +426,35 @@ export const BrowserAutomationLogsResultSchema = z.object({ network: z.array(BrowserAutomationNetworkLogEntrySchema), }); +export const BrowserAutomationEvaluateResultSchema = z.object({ + command: z.literal("evaluate"), + browserId: BrowserAutomationBrowserIdSchema, + resultJson: z.string(), + truncated: z.boolean(), +}); + +export const BrowserAutomationScrollResultSchema = z.object({ + command: z.literal("scroll"), + browserId: BrowserAutomationBrowserIdSchema, + ref: BrowserAutomationRefSchema.optional(), + deltaX: z.number(), + deltaY: z.number(), + x: z.number().optional(), + y: z.number().optional(), +}); + +export const BrowserAutomationResizeResultSchema = z.object({ + command: z.literal("resize"), + browserId: BrowserAutomationBrowserIdSchema, + width: z.number().int().positive(), + height: z.number().int().positive(), +}); + +export const BrowserAutomationCloseTabResultSchema = z.object({ + command: z.literal("close_tab"), + browserId: BrowserAutomationBrowserIdSchema, +}); + export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [ BrowserAutomationListTabsResultSchema, BrowserAutomationNewTabResultSchema, @@ -386,6 +474,10 @@ export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [ BrowserAutomationHoverResultSchema, BrowserAutomationDragResultSchema, BrowserAutomationLogsResultSchema, + BrowserAutomationEvaluateResultSchema, + BrowserAutomationScrollResultSchema, + BrowserAutomationResizeResultSchema, + BrowserAutomationCloseTabResultSchema, ]); export const BrowserAutomationErrorSchema = z.object({ @@ -394,6 +486,15 @@ export const BrowserAutomationErrorSchema = z.object({ retryable: z.boolean().default(false), }); +export const BrowserAutomationDialogEventSchema = z.object({ + type: z.enum(["alert", "confirm", "prompt", "beforeunload"]), + message: z.string(), + defaultValue: z.string().optional(), + action: z.enum(["accepted", "dismissed"]), + promptText: z.string().optional(), + timestamp: z.number(), +}); + export const BrowserAutomationExecuteRequestSchema = z .object({ type: z.literal("browser.automation.execute.request"), @@ -412,11 +513,13 @@ export const BrowserAutomationExecuteResponseSchema = z.object({ requestId: z.string().min(1), ok: z.literal(true), result: BrowserAutomationResultSchema, + dialogs: z.array(BrowserAutomationDialogEventSchema).optional(), }), z.object({ requestId: z.string().min(1), ok: z.literal(false), error: BrowserAutomationErrorSchema, + dialogs: z.array(BrowserAutomationDialogEventSchema).optional(), }), ]), }); @@ -431,6 +534,7 @@ export type BrowserAutomationConsoleLogEntry = z.infer< export type BrowserAutomationNetworkLogEntry = z.infer< typeof BrowserAutomationNetworkLogEntrySchema >; +export type BrowserAutomationDialogEvent = z.infer; export type BrowserAutomationExecuteRequest = z.infer; export type BrowserAutomationExecuteResponse = z.infer< typeof BrowserAutomationExecuteResponseSchema diff --git a/packages/protocol/src/messages.browser-automation.test.ts b/packages/protocol/src/messages.browser-automation.test.ts index 7fb255dd1..3e5805b69 100644 --- a/packages/protocol/src/messages.browser-automation.test.ts +++ b/packages/protocol/src/messages.browser-automation.test.ts @@ -100,6 +100,28 @@ describe("browser automation protocol integration", () => { }); }); + test("browser host capability accepts new tool commands as supported commands", () => { + expect( + WSHelloMessageSchema.parse({ + type: "hello", + clientId: "client-1", + clientType: "mobile", + protocolVersion: 1, + capabilities: { + [CLIENT_CAPS.browserHost]: { + supportedCommands: ["evaluate", "scroll", "resize", "close_tab"], + hostKind: "desktop app", + }, + }, + }).capabilities, + ).toMatchObject({ + [CLIENT_CAPS.browserHost]: { + supportedCommands: ["evaluate", "scroll", "resize", "close_tab"], + hostKind: "desktop app", + }, + }); + }); + test("hello remains valid when no browser host capability is advertised", () => { expect( WSHelloMessageSchema.parse({ diff --git a/packages/server/src/server/browser-tools/broker.test.ts b/packages/server/src/server/browser-tools/broker.test.ts index d73f63e84..b7de56e2f 100644 --- a/packages/server/src/server/browser-tools/broker.test.ts +++ b/packages/server/src/server/browser-tools/broker.test.ts @@ -221,7 +221,10 @@ describe("BrowserToolsBroker", () => { workspaceId: "workspace-1", url: "https://example.com", title: "Example", - elements: [], + format: "aria-yaml", + snapshot: "- document", + truncated: false, + stats: { nodeCount: 1, refCount: 0, textLength: 10 }, }, }); @@ -234,7 +237,10 @@ describe("BrowserToolsBroker", () => { workspaceId: "workspace-1", url: "https://example.com", title: "Example", - elements: [], + format: "aria-yaml", + snapshot: "- document", + truncated: false, + stats: { nodeCount: 1, refCount: 0, textLength: 10 }, }, }); }); @@ -299,7 +305,10 @@ describe("BrowserToolsBroker", () => { workspaceId: "workspace-1", url: "https://example.com", title: "Example", - elements: [], + format: "aria-yaml", + snapshot: "- document", + truncated: false, + stats: { nodeCount: 1, refCount: 0, textLength: 10 }, }, }); @@ -423,7 +432,10 @@ describe("BrowserToolsBroker", () => { workspaceId: "workspace-1", url: "https://one.example", title: "One", - elements: [], + format: "aria-yaml", + snapshot: "- document", + truncated: false, + stats: { nodeCount: 1, refCount: 0, textLength: 10 }, }, }); secondHost.resolveLatestWith(broker, { @@ -435,7 +447,10 @@ describe("BrowserToolsBroker", () => { workspaceId: "workspace-1", url: "https://two.example", title: "Two", - elements: [], + format: "aria-yaml", + snapshot: "- document", + truncated: false, + stats: { nodeCount: 1, refCount: 0, textLength: 10 }, }, }); @@ -449,6 +464,127 @@ describe("BrowserToolsBroker", () => { }); }); + test.each([ + { + name: "scroll", + command: { command: "scroll", args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 } }, + result: { command: "scroll", browserId: BROWSER_ID, deltaX: 0, deltaY: 400 }, + }, + { + name: "resize", + command: { command: "resize", args: { browserId: BROWSER_ID, width: 1024, height: 768 } }, + result: { command: "resize", browserId: BROWSER_ID, width: 1024, height: 768 }, + }, + { + name: "close_tab", + command: { command: "close_tab", args: { browserId: BROWSER_ID } }, + result: { command: "close_tab", browserId: BROWSER_ID }, + }, + ] satisfies Array<{ + name: string; + command: BrowserAutomationCommand; + result: BrowserAutomationExecuteResponse["payload"] extends infer Payload + ? Payload extends { ok: true } + ? Payload["result"] + : never + : never; + }>)("routes $name to the host that owns the browser id", async ({ command, result }) => { + const broker = createBroker({ enabled: true }); + const other = new FakeBrowserHostClient("host-1"); + const owner = new FakeBrowserHostClient("host-2"); + broker.registerClient(other); + broker.registerClient(owner); + + const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } }); + owner.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "new_tab", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://one.example", + }, + }); + await newTabPromise; + + const resultPromise = broker.execute({ command, requestId: "req-command" }); + + expect(owner.receivedRequests.at(-1)).toEqual({ + type: "browser.automation.execute.request", + requestId: "req-command", + command, + }); + expect(other.receivedRequests).toEqual([]); + + owner.resolveLatestWith(broker, { + requestId: "req-command", + ok: true, + result, + }); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-command", + ok: true, + result, + }); + }); + + test("successful close_tab clears browser id host affinity", async () => { + const broker = createBroker({ enabled: true }); + const other = new FakeBrowserHostClient("host-1"); + const owner = new FakeBrowserHostClient("host-2"); + broker.registerClient(other); + broker.registerClient(owner); + + const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } }); + owner.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "new_tab", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://one.example", + }, + }); + await newTabPromise; + + const closePromise = broker.execute({ + command: { command: "close_tab", args: { browserId: BROWSER_ID } }, + requestId: "req-close", + }); + owner.resolveLatestWith(broker, { + requestId: "req-close", + ok: true, + result: { command: "close_tab", browserId: BROWSER_ID }, + }); + await expect(closePromise).resolves.toMatchObject({ + ok: true, + result: { command: "close_tab", browserId: BROWSER_ID }, + }); + + await expect( + broker.execute({ + command: { command: "snapshot", args: { browserId: BROWSER_ID } }, + requestId: "req-after-close", + }), + ).resolves.toEqual({ + requestId: "req-after-close", + ok: false, + error: { + code: "browser_tab_not_found", + message: `Browser tab ${BROWSER_ID} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`, + retryable: false, + }, + }); + expect(owner.receivedRequests.at(-1)?.command).toEqual({ + command: "close_tab", + args: { browserId: BROWSER_ID }, + }); + expect(other.receivedRequests).toEqual([]); + }); + test("failed list tabs aggregation does not seed browser id affinity", async () => { const broker = createBroker({ enabled: true }); const firstHost = new FakeBrowserHostClient("host-1"); @@ -531,6 +667,35 @@ describe("BrowserToolsBroker", () => { retryable: false, }, }); + await expect( + broker.execute({ + command: { + command: "evaluate", + args: { browserId: BROWSER_ID, function: "() => document.title" }, + }, + }), + ).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_unsupported", + message: 'Browser automation command "evaluate" is not supported by the desktop app.', + retryable: false, + }, + }); + await expect( + broker.execute({ + command: { command: "scroll", args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 } }, + }), + ).resolves.toEqual({ + requestId: "req-1", + ok: false, + error: { + code: "browser_unsupported", + message: 'Browser automation command "scroll" is not supported by the desktop app.', + retryable: false, + }, + }); expect(client.receivedRequests).toEqual([]); }); @@ -623,7 +788,10 @@ describe("BrowserToolsBroker", () => { workspaceId: "workspace-1", url: "https://example.com", title: "Example", - elements: [], + format: "aria-yaml", + snapshot: "- document", + truncated: false, + stats: { nodeCount: 1, refCount: 0, textLength: 10 }, }, }); diff --git a/packages/server/src/server/browser-tools/broker.ts b/packages/server/src/server/browser-tools/broker.ts index f938403ff..a362aad00 100644 --- a/packages/server/src/server/browser-tools/broker.ts +++ b/packages/server/src/server/browser-tools/broker.ts @@ -401,6 +401,12 @@ export class BrowserToolsBroker { return; } + if (payload.result.command === "close_tab") { + this.browserHostByBrowserId.delete(payload.result.browserId); + this.strandedBrowserHostByBrowserId.delete(payload.result.browserId); + return; + } + if ("browserId" in payload.result) { this.browserHostByBrowserId.set(payload.result.browserId, clientId); this.strandedBrowserHostByBrowserId.delete(payload.result.browserId); @@ -462,28 +468,10 @@ export class BrowserToolsBroker { } function getBrowserIdForCommand(command: BrowserAutomationCommand): string | null { - switch (command.command) { - case "list_tabs": - case "new_tab": - return null; - case "snapshot": - case "click": - case "fill": - case "wait": - case "type": - case "keypress": - case "navigate": - case "back": - case "forward": - case "reload": - case "screenshot": - case "upload": - case "select": - case "hover": - case "drag": - case "logs": - return command.args.browserId; + if (command.command === "list_tabs" || command.command === "new_tab") { + return null; } + return command.args.browserId; } function describeBrowserHost(host: RegisteredBrowserHost): string { diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts index d0848c46e..3ee89d967 100644 --- a/packages/server/src/server/browser-tools/tools.test.ts +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -133,7 +133,10 @@ function snapshotPayload(): Extract { workspaceId: "wks_workspace_a", url: "https://example.com", title: "Example", - elements: [], + format: "aria-yaml", + snapshot: '- document "Example"\n - button "Save" [ref=@e1]', + truncated: false, + stats: { nodeCount: 2, refCount: 1, textLength: 50 }, }, }; } @@ -158,7 +161,43 @@ const routedToolCases = [ name: "click", toolName: "browser_click", input: { browserId: BROWSER_ID, ref: "@e2" }, - command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e2" } }, + command: { + command: "click", + args: { + browserId: BROWSER_ID, + ref: "@e2", + button: "left", + doubleClick: false, + modifiers: [], + }, + }, + payload: { + requestId: "req-click", + ok: true, + result: { command: "click", browserId: BROWSER_ID, ref: "@e2" }, + }, + content: [{ type: "text", text: "Clicked browser element @e2." }], + }, + { + name: "click options", + toolName: "browser_click", + input: { + browserId: BROWSER_ID, + ref: "@e2", + button: "right", + doubleClick: true, + modifiers: ["Control", "Shift"], + }, + command: { + command: "click", + args: { + browserId: BROWSER_ID, + ref: "@e2", + button: "right", + doubleClick: true, + modifiers: ["Control", "Shift"], + }, + }, payload: { requestId: "req-click", ok: true, @@ -344,6 +383,85 @@ const routedToolCases = [ }, content: [{ type: "text", text: "Dragged browser element @e4 to @e5." }], }, + { + name: "evaluate", + toolName: "browser_evaluate", + input: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" }, + command: { + command: "evaluate", + args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" }, + }, + payload: { + requestId: "req-evaluate", + ok: true, + result: { + command: "evaluate", + browserId: BROWSER_ID, + resultJson: '"Save"', + truncated: false, + }, + }, + content: [{ type: "text", text: 'Browser evaluate returned:\n"Save"' }], + }, + { + name: "scroll", + toolName: "browser_scroll", + input: { browserId: BROWSER_ID, ref: "@e1", deltaX: 0, deltaY: 400 }, + command: { + command: "scroll", + args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 0, deltaY: 400 }, + }, + payload: { + requestId: "req-scroll", + ok: true, + result: { + command: "scroll", + browserId: BROWSER_ID, + ref: "@e1", + deltaX: 0, + deltaY: 400, + }, + }, + content: [{ type: "text", text: "Scrolled browser element @e1 by 0, 400." }], + }, + { + name: "resize", + toolName: "browser_resize", + input: { browserId: BROWSER_ID, width: 1024, height: 768 }, + command: { + command: "resize", + args: { browserId: BROWSER_ID, width: 1024, height: 768 }, + }, + payload: { + requestId: "req-resize", + ok: true, + result: { + command: "resize", + browserId: BROWSER_ID, + width: 1024, + height: 768, + }, + }, + content: [{ type: "text", text: "Resized browser viewport to 1024x768." }], + }, + { + name: "close_tab", + toolName: "browser_close_tab", + input: { browserId: BROWSER_ID }, + command: { + command: "close_tab", + args: { browserId: BROWSER_ID }, + }, + payload: { + requestId: "req-close-tab", + ok: true, + result: { + command: "close_tab", + browserId: BROWSER_ID, + }, + }, + content: [{ type: "text", text: `Closed browser tab ${BROWSER_ID}.` }], + }, ] satisfies Array<{ name: string; toolName: string; @@ -459,6 +577,10 @@ describe("registerBrowserTools", () => { "browser_select", "browser_drag", "browser_logs", + "browser_evaluate", + "browser_scroll", + "browser_resize", + "browser_close_tab", ]); }); @@ -668,7 +790,10 @@ describe("registerBrowserTools", () => { workspaceId: "wks_workspace_a", url: "https://example.com", title: "Example", - elements: [], + format: "aria-yaml", + snapshot: '- document "Example"\n - button "Save" [ref=@e1]', + truncated: false, + stats: { nodeCount: 2, refCount: 1, textLength: 50 }, }, context: { agentId: "agent-1", @@ -726,6 +851,89 @@ describe("registerBrowserTools", () => { }, ); + test("success responses include handled dialog metadata and a text note", async () => { + const harness = new BrowserToolHarness(); + const payload: Extract = { + requestId: "req-click", + ok: true, + result: { command: "click", browserId: BROWSER_ID, ref: "@e1" }, + dialogs: [ + { + type: "confirm", + message: "Delete item?", + action: "dismissed", + timestamp: 123, + }, + ], + }; + harness.broker.setResponse(payload); + + const response = await harness.execute("browser_click", { browserId: BROWSER_ID, ref: "@e1" }); + + expect(response.content).toEqual([ + { + type: "text", + text: 'Clicked browser element @e1.\nHandled browser dialog: dismissed confirm "Delete item?".', + }, + ]); + expect(response.structuredContent).toEqual({ + ok: true, + result: payload.result, + dialogs: payload.dialogs, + context: { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + browserId: BROWSER_ID, + }, + }); + }); + + test("failure responses include handled dialog metadata and a text note", async () => { + const harness = new BrowserToolHarness(); + const payload: Extract = { + requestId: "req-wait", + ok: false, + error: { + code: "browser_timeout", + message: "Timed out waiting for browser URL: /next", + retryable: true, + }, + dialogs: [ + { + type: "beforeunload", + message: "Leave site?", + action: "dismissed", + timestamp: 124, + }, + ], + }; + harness.broker.setResponse(payload); + + const response = await harness.execute("browser_wait", { + browserId: BROWSER_ID, + url: "/next", + }); + + expect(response.content).toEqual([ + { + type: "text", + text: 'The browser did not respond before the timeout. Try again or check the browser host.\nHandled browser dialog: dismissed beforeunload "Leave site?".', + }, + ]); + expect(response.structuredContent).toEqual({ + ok: false, + error: payload.error, + dialogs: payload.dialogs, + context: { + agentId: "agent-1", + cwd: "/repo", + workspaceId: "wks_workspace_a", + browserId: BROWSER_ID, + }, + }); + }); + test("wait rejects calls without a condition", () => { const harness = new BrowserToolHarness(); diff --git a/packages/server/src/server/browser-tools/tools.ts b/packages/server/src/server/browser-tools/tools.ts index 1181bfd56..cb9bf92cb 100644 --- a/packages/server/src/server/browser-tools/tools.ts +++ b/packages/server/src/server/browser-tools/tools.ts @@ -45,6 +45,18 @@ const BrowserToolOutputSchema = { retryable: z.boolean(), }) .optional(), + dialogs: z + .array( + z.object({ + type: z.enum(["alert", "confirm", "prompt", "beforeunload"]), + message: z.string(), + defaultValue: z.string().optional(), + action: z.enum(["accepted", "dismissed"]), + promptText: z.string().optional(), + timestamp: z.number(), + }), + ) + .optional(), context: z .object({ agentId: z.string().optional(), @@ -70,6 +82,8 @@ const BrowserHttpUrlInputSchema = z return normalized; }); const BrowserRefInputSchema = z.string().regex(/^@e\d+$/); +const BrowserClickButtonInputSchema = z.enum(["left", "right", "middle"]); +const BrowserClickModifierInputSchema = z.enum(["Alt", "Control", "Meta", "Shift"]); const BrowserWaitInputSchema = z .object({ text: z.string().min(1).optional(), @@ -178,10 +192,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void inputSchema: { ref: BrowserRefInputSchema, browserId: BrowserAutomationBrowserIdSchema, + button: BrowserClickButtonInputSchema.optional(), + doubleClick: z.boolean().optional(), + modifiers: z.array(BrowserClickModifierInputSchema).optional(), }, outputSchema: BrowserToolOutputSchema, }, - async ({ ref, browserId }) => { + async ({ ref, browserId, button, doubleClick, modifiers }) => { const context = resolveBrowserToolContext(options); const payload = await options.broker.execute({ agentId: context.agentId, @@ -193,6 +210,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void args: { browserId, ref, + button: button ?? "left", + doubleClick: doubleClick ?? false, + modifiers: modifiers ?? [], }, }, }); @@ -605,6 +625,136 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void return browserToolResult({ payload, context: { ...context, browserId } }); }, ); + + options.registerTool( + "browser_evaluate", + { + title: "Evaluate browser JavaScript", + description: + "Evaluate a JavaScript function in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; when ref is provided, refs come from the latest browser_snapshot and the resolved element is passed as the first argument.", + inputSchema: { + function: z.string().min(1), + ref: BrowserRefInputSchema.optional(), + browserId: BrowserAutomationBrowserIdSchema, + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ function: functionSource, ref, browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + + command: { + command: "evaluate", + args: { + browserId, + function: functionSource, + ...(ref ? { ref } : {}), + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_scroll", + { + title: "Scroll browser", + description: + "Scroll a Paseo browser tab by deltaX/deltaY CSS pixels. Use browserId from browser_new_tab or browser_list_tabs; optional ref comes from the latest browser_snapshot and centers the wheel input over that element.", + inputSchema: { + browserId: BrowserAutomationBrowserIdSchema, + ref: BrowserRefInputSchema.optional(), + deltaX: z.number(), + deltaY: z.number(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId, ref, deltaX, deltaY }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + + command: { + command: "scroll", + args: { + browserId, + ...(ref ? { ref } : {}), + deltaX, + deltaY, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_resize", + { + title: "Resize browser viewport", + description: + "Resize a Paseo browser tab's resident webview viewport. Use browserId from browser_new_tab or browser_list_tabs.", + inputSchema: { + browserId: BrowserAutomationBrowserIdSchema, + width: z.number().int().positive(), + height: z.number().int().positive(), + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId, width, height }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + + command: { + command: "resize", + args: { + browserId, + width, + height, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); + + options.registerTool( + "browser_close_tab", + { + title: "Close browser tab", + description: + "Close a Paseo browser tab, remove its resident webview, and unregister it from the browser automation host. Use browserId from browser_new_tab or browser_list_tabs.", + inputSchema: { + browserId: BrowserAutomationBrowserIdSchema, + }, + outputSchema: BrowserToolOutputSchema, + }, + async ({ browserId }) => { + const context = resolveBrowserToolContext(options); + const payload = await options.broker.execute({ + agentId: context.agentId, + cwd: context.cwd, + ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + + command: { + command: "close_tab", + args: { + browserId, + }, + }, + }); + return browserToolResult({ payload, context: { ...context, browserId } }); + }, + ); } function resolveBrowserToolContext(options: RegisterBrowserToolsOptions): { @@ -681,16 +831,23 @@ function browserToolResult(params: { structuredContent: { ok: true, result: browserToolStructuredResult(payload.result), + ...(payload.dialogs ? { dialogs: payload.dialogs } : {}), context, }, }; } return { - content: [{ type: "text", text: summarizeBrowserError(payload.error) }], + content: [ + { + type: "text", + text: appendDialogSummary(summarizeBrowserError(payload.error), payload.dialogs), + }, + ], structuredContent: { ok: false, error: payload.error, + ...(payload.dialogs ? { dialogs: payload.dialogs } : {}), context, }, }; @@ -732,34 +889,35 @@ function browserToolImageContent( function summarizeBrowserSuccess( payload: Extract, ): string { + const withDialogs = (summary: string) => appendDialogSummary(summary, payload.dialogs); const controlSummary = summarizeBrowserControlSuccess(payload.result); if (controlSummary) { - return controlSummary; + return withDialogs(controlSummary); } const refActionSummary = summarizeBrowserRefActionSuccess(payload.result); if (refActionSummary) { - return refActionSummary; + return withDialogs(refActionSummary); } const diagnosticsSummary = summarizeBrowserDiagnosticsSuccess(payload.result); if (diagnosticsSummary) { - return diagnosticsSummary; + return withDialogs(diagnosticsSummary); } const keyboardSummary = summarizeBrowserKeyboardSuccess(payload.result); if (keyboardSummary) { - return keyboardSummary; + return withDialogs(keyboardSummary); } const navigationSummary = summarizeBrowserNavigationSuccess(payload.result); if (navigationSummary) { - return navigationSummary; + return withDialogs(navigationSummary); } const mediaSummary = summarizeBrowserMediaSuccess(payload.result); if (mediaSummary) { - return mediaSummary; + return withDialogs(mediaSummary); } if (payload.result.command === "list_tabs") { @@ -771,26 +929,49 @@ function summarizeBrowserSuccess( const active = tab.isActive ? " active" : ""; return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url}`; }); - return [ - `Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values for tab-scoped browser tools.`, - ...tabLines, - ].join("\n"); + return withDialogs( + [ + `Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values for tab-scoped browser tools.`, + ...tabLines, + ].join("\n"), + ); } if (payload.result.command === "new_tab") { - return `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`; + return withDialogs( + `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`, + ); } if (payload.result.command === "snapshot") { - const count = payload.result.elements.length; - return `Snapshot captured ${count} element${count === 1 ? "" : "s"}.`; + return withDialogs( + [ + `Snapshot captured ${payload.result.stats.nodeCount} node${payload.result.stats.nodeCount === 1 ? "" : "s"} with ${payload.result.stats.refCount} ref${payload.result.stats.refCount === 1 ? "" : "s"}.`, + `Title: ${payload.result.title || "Untitled"}`, + `URL: ${payload.result.url}`, + "", + payload.result.snapshot, + ].join("\n"), + ); } if (payload.result.command === "wait") { - return `Browser wait matched ${payload.result.matched}.`; + return withDialogs(`Browser wait matched ${payload.result.matched}.`); } - return `Browser ${payload.result.command} complete.`; + return withDialogs(`Browser ${payload.result.command} complete.`); +} + +function appendDialogSummary( + summary: string, + dialogs: BrowserToolsResponsePayload["dialogs"], +): string { + if (!dialogs || dialogs.length === 0) { + return summary; + } + return `${summary}\nHandled browser dialog${dialogs.length === 1 ? "" : "s"}: ${dialogs + .map((dialog) => `${dialog.action} ${dialog.type} ${JSON.stringify(dialog.message)}`) + .join("; ")}.`; } function summarizeBrowserMediaSuccess( @@ -841,9 +1022,18 @@ function summarizeBrowserNavigationSuccess( function summarizeBrowserDiagnosticsSuccess( result: Extract["result"], ): string | null { + if (result.command === "evaluate") { + return [ + "Browser evaluate returned:", + result.resultJson, + ...(result.truncated ? ["Result was truncated."] : []), + ].join("\n"); + } + if (result.command !== "logs") { return null; } + const consoleCount = result.console.length; const networkCount = result.network.length; return `Read ${consoleCount} console log${consoleCount === 1 ? "" : "s"} and ${networkCount} network entr${networkCount === 1 ? "y" : "ies"}.`; @@ -878,6 +1068,20 @@ function summarizeBrowserControlSuccess( return `Dragged browser element ${result.sourceRef} to ${result.targetRef}.`; } + if (result.command === "scroll") { + return result.ref + ? `Scrolled browser element ${result.ref} by ${result.deltaX}, ${result.deltaY}.` + : `Scrolled browser by ${result.deltaX}, ${result.deltaY}.`; + } + + if (result.command === "resize") { + return `Resized browser viewport to ${result.width}x${result.height}.`; + } + + if (result.command === "close_tab") { + return `Closed browser tab ${result.browserId}.`; + } + return null; }