mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Browser tools: full accessibility snapshot, trusted input, dialogs, evaluate, and tab controls (#1881)
* feat(browser): replace flat snapshot with ARIA tree and page-side refs * feat(browser): trusted CDP input with actionability waits * feat(browser): auto-handle javascript dialogs * feat(browser): add browser_evaluate * feat(browser): add scroll, resize, and close_tab tools * fix(browser): review fixes for snapshot/input/dialog seams * fix(browser): address review findings on dialog capture, truncation, and keypress targets * fix(harness): drop resize check that cannot run under parked compositor surfaces * fix(browser): second review round — snapshot fidelity, key input, dialog scoping
This commit is contained in:
Mohamed Boudra
@@ -21,6 +21,18 @@ Run it with the repo Electron:
|
||||
npm run capture-harness --workspace=@getpaseo/desktop
|
||||
```
|
||||
|
||||
Run the browser automation fixture with:
|
||||
|
||||
```bash
|
||||
PASEO_CAPTURE_HARNESS_GROUP=automation npm run capture-harness --workspace=@getpaseo/desktop
|
||||
```
|
||||
|
||||
The automation group uses a real guest webview to verify the page-side ref contract:
|
||||
ARIA-like snapshot text includes headings, static text, and controls; refs survive
|
||||
`pushState` when the element still matches; same-URL rerenders stale old refs; and a
|
||||
file-input ref can be resolved to a CDP backend node id for upload. It also verifies
|
||||
page-context evaluation, including passing a resolved ref element as the function argument.
|
||||
|
||||
On macOS the harness process must set `app.setActivationPolicy("accessory")` and
|
||||
hide the Dock icon before creating any window. `showInactive()` only prevents window
|
||||
focus; a normal Electron app launch can still activate the app and steal focus.
|
||||
|
||||
@@ -75,6 +75,8 @@ class FakeBrowserBridge {
|
||||
public readonly executedRequests: BrowserAutomationExecuteRequest[] = [];
|
||||
public readonly registeredWorkspaceBrowsers: Array<{ browserId: string; workspaceId: string }> =
|
||||
[];
|
||||
public readonly unregisteredWorkspaceBrowsers: string[] = [];
|
||||
public readonly clearedPartitions: string[] = [];
|
||||
public readonly activeWorkspaceBrowsers: Array<{
|
||||
browserId: string | null;
|
||||
workspaceId: string;
|
||||
@@ -99,6 +101,14 @@ class FakeBrowserBridge {
|
||||
this.registeredWorkspaceBrowsers.push(input);
|
||||
};
|
||||
|
||||
public unregisterWorkspaceBrowser = async (browserId: string): Promise<void> => {
|
||||
this.unregisteredWorkspaceBrowsers.push(browserId);
|
||||
};
|
||||
|
||||
public clearPartition = async (browserId: string): Promise<void> => {
|
||||
this.clearedPartitions.push(browserId);
|
||||
};
|
||||
|
||||
public setWorkspaceActiveBrowser = async (input: {
|
||||
browserId: string | null;
|
||||
workspaceId: string;
|
||||
@@ -175,6 +185,35 @@ function browserNewTabRequest(): BrowserAutomationExecuteRequest {
|
||||
};
|
||||
}
|
||||
|
||||
function browserResizeRequest(
|
||||
browserId: string,
|
||||
input: { workspaceId?: string } = {},
|
||||
): BrowserAutomationExecuteRequest {
|
||||
return {
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-resize",
|
||||
agentId: "agent-1",
|
||||
workspaceId: input.workspaceId ?? "wks_workspace_a",
|
||||
command: {
|
||||
command: "resize",
|
||||
args: { browserId, width: 1024, height: 768 },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function browserCloseTabRequest(browserId: string): BrowserAutomationExecuteRequest {
|
||||
return {
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-close-tab",
|
||||
agentId: "agent-1",
|
||||
workspaceId: "wks_workspace_a",
|
||||
command: {
|
||||
command: "close_tab",
|
||||
args: { browserId },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function emptyListTabsPayload(requestId = "req-new:list_tabs"): BrowserAutomationResponsePayload {
|
||||
return {
|
||||
requestId,
|
||||
@@ -355,6 +394,106 @@ describe("mountBrowserAutomationHandler", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("browser_resize updates resident webview dimensions", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.mount({ serverId: "server-1" });
|
||||
|
||||
browser.receive(browserNewTabRequest());
|
||||
await flushAsyncWork();
|
||||
const result = newTabResultFrom(browser.client.payloadAt(0));
|
||||
|
||||
browser.receive(browserResizeRequest(result.browserId));
|
||||
await flushAsyncWork();
|
||||
|
||||
expect(browser.client.payloadAt(1)).toEqual({
|
||||
requestId: "req-resize",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "resize",
|
||||
browserId: result.browserId,
|
||||
width: 1024,
|
||||
height: 768,
|
||||
},
|
||||
});
|
||||
expect(browser.browser.executedRequests).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("browser_resize returns not found for a tab outside the request workspace", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.mount({ serverId: "server-1" });
|
||||
|
||||
browser.receive(browserNewTabRequest());
|
||||
await flushAsyncWork();
|
||||
const result = newTabResultFrom(browser.client.payloadAt(0));
|
||||
|
||||
browser.receive(browserResizeRequest(result.browserId, { workspaceId: "wks_workspace_b" }));
|
||||
await flushAsyncWork();
|
||||
|
||||
expect(browser.client.payloadAt(1)).toEqual({
|
||||
requestId: "req-resize",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: `No browser tab found for ID: ${result.browserId}`,
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("browser_close_tab removes the workspace tab, browser record, resident webview, registry entry, and partition", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
const workspaceKey = buildWorkspaceTabPersistenceKey({
|
||||
serverId: "server-1",
|
||||
workspaceId: "wks_workspace_a",
|
||||
});
|
||||
if (!workspaceKey) {
|
||||
throw new Error("Expected workspace key");
|
||||
}
|
||||
browser.mount({ serverId: "server-1" });
|
||||
|
||||
browser.receive(browserNewTabRequest());
|
||||
await flushAsyncWork();
|
||||
const result = newTabResultFrom(browser.client.payloadAt(0));
|
||||
|
||||
browser.receive(browserCloseTabRequest(result.browserId));
|
||||
await flushAsyncWork();
|
||||
|
||||
expect(browser.client.payloadAt(1)).toEqual({
|
||||
requestId: "req-close-tab",
|
||||
ok: true,
|
||||
result: { command: "close_tab", browserId: result.browserId },
|
||||
});
|
||||
expect(workspaceBrowserTabs(workspaceKey, result.browserId)).toEqual([]);
|
||||
expect(useBrowserStore.getState().browsersById[result.browserId]).toBeUndefined();
|
||||
expect(browser.browser.unregisteredWorkspaceBrowsers).toEqual([result.browserId]);
|
||||
expect(browser.browser.clearedPartitions).toEqual([result.browserId]);
|
||||
expect(currentBrowserTabs()).toEqual([]);
|
||||
});
|
||||
|
||||
test("browser_close_tab returns not found after the tab is gone", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.mount({ serverId: "server-1" });
|
||||
|
||||
browser.receive(browserNewTabRequest());
|
||||
await flushAsyncWork();
|
||||
const result = newTabResultFrom(browser.client.payloadAt(0));
|
||||
|
||||
browser.receive(browserCloseTabRequest(result.browserId));
|
||||
await flushAsyncWork();
|
||||
browser.receive(browserCloseTabRequest(result.browserId));
|
||||
await flushAsyncWork();
|
||||
|
||||
expect(browser.client.payloadAt(2)).toEqual({
|
||||
requestId: "req-close-tab",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: `No browser tab found for ID: ${result.browserId}`,
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("non-new-tab requests send the desktop bridge response", async () => {
|
||||
const browser = new BrowserAutomationHandlerHarness();
|
||||
browser.browser.response = {
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
|
||||
import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
|
||||
import { createWorkspaceBrowser } from "@/stores/browser-store";
|
||||
import {
|
||||
ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault,
|
||||
removeResidentBrowserWebview,
|
||||
resizeResidentBrowserWebview,
|
||||
} from "@/components/browser-webview-resident";
|
||||
import { createWorkspaceBrowser, getBrowserRecord, useBrowserStore } from "@/stores/browser-store";
|
||||
import {
|
||||
buildWorkspaceTabPersistenceKey,
|
||||
collectAllTabs,
|
||||
useWorkspaceLayoutStore,
|
||||
} from "@/stores/workspace-layout-store";
|
||||
|
||||
@@ -114,6 +119,33 @@ async function handleBrowserAutomationRequest(params: {
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.command.command === "resize") {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: resizeBrowserTabForRequest({ request, serverId }),
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (request.command.command === "close_tab") {
|
||||
try {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: await closeBrowserTabForRequest({
|
||||
request,
|
||||
serverId,
|
||||
browserHost,
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: normalizeThrownBridgeError(request.requestId, error),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!executeAutomationCommand) {
|
||||
client.sendBrowserAutomationExecuteResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
@@ -140,6 +172,127 @@ async function handleBrowserAutomationRequest(params: {
|
||||
}
|
||||
}
|
||||
|
||||
function resizeBrowserTabForRequest(params: {
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
serverId?: string;
|
||||
}): BrowserAutomationResponsePayload {
|
||||
const { request, serverId } = params;
|
||||
const command = request.command as Extract<
|
||||
BrowserAutomationExecuteRequest["command"],
|
||||
{ command: "resize" }
|
||||
>;
|
||||
const browserId = command.args.browserId;
|
||||
if (!getBrowserRecord(browserId)) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_tab_not_found",
|
||||
message: `No browser tab found for ID: ${browserId}`,
|
||||
});
|
||||
}
|
||||
|
||||
const workspaceId = request.workspaceId;
|
||||
if (serverId && workspaceId && !findWorkspaceBrowserTab({ serverId, workspaceId, browserId })) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_tab_not_found",
|
||||
message: `No browser tab found for ID: ${browserId}`,
|
||||
});
|
||||
}
|
||||
|
||||
const dimensions = resizeResidentBrowserWebview({
|
||||
browserId,
|
||||
width: command.args.width,
|
||||
height: command.args.height,
|
||||
});
|
||||
if (!dimensions) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_tab_not_found",
|
||||
message: `No browser tab found for ID: ${browserId}`,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
result: {
|
||||
command: "resize",
|
||||
browserId,
|
||||
width: dimensions.width,
|
||||
height: dimensions.height,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function closeBrowserTabForRequest(params: {
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
serverId?: string;
|
||||
browserHost: DesktopHostBridge["browser"] | undefined;
|
||||
}): Promise<BrowserAutomationResponsePayload> {
|
||||
const { request, serverId, browserHost } = params;
|
||||
const command = request.command as Extract<
|
||||
BrowserAutomationExecuteRequest["command"],
|
||||
{ command: "close_tab" }
|
||||
>;
|
||||
const browserId = command.args.browserId;
|
||||
const workspaceId = request.workspaceId;
|
||||
const workspaceTab = serverId
|
||||
? findWorkspaceBrowserTab({ serverId, workspaceId, browserId })
|
||||
: null;
|
||||
if (!workspaceTab && (!serverId || !workspaceId)) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_unsupported",
|
||||
message: "Cannot close a browser tab without a workspace context.",
|
||||
});
|
||||
}
|
||||
if (!workspaceTab || !getBrowserRecord(browserId)) {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_tab_not_found",
|
||||
message: `No browser tab found for ID: ${browserId}`,
|
||||
});
|
||||
}
|
||||
|
||||
useWorkspaceLayoutStore.getState().closeTab(workspaceTab.workspaceKey, workspaceTab.tabId);
|
||||
useBrowserStore.getState().removeBrowser(browserId);
|
||||
removeResidentBrowserWebview(browserId);
|
||||
await browserHost?.unregisterWorkspaceBrowser?.(browserId);
|
||||
await browserHost?.clearPartition?.(browserId);
|
||||
|
||||
return {
|
||||
requestId: request.requestId,
|
||||
ok: true,
|
||||
result: { command: "close_tab", browserId },
|
||||
};
|
||||
}
|
||||
|
||||
function findWorkspaceBrowserTab(input: {
|
||||
serverId: string;
|
||||
workspaceId: string | undefined;
|
||||
browserId: string;
|
||||
}): { workspaceKey: string; tabId: string } | null {
|
||||
if (!input.workspaceId) {
|
||||
return null;
|
||||
}
|
||||
const workspaceKey = buildWorkspaceTabPersistenceKey({
|
||||
serverId: input.serverId,
|
||||
workspaceId: input.workspaceId,
|
||||
});
|
||||
if (!workspaceKey) {
|
||||
return null;
|
||||
}
|
||||
const layout = useWorkspaceLayoutStore.getState().layoutByWorkspace[workspaceKey];
|
||||
const tab = layout
|
||||
? collectAllTabs(layout.root).find((candidate) => {
|
||||
return (
|
||||
candidate.target.kind === "browser" && candidate.target.browserId === input.browserId
|
||||
);
|
||||
})
|
||||
: null;
|
||||
return tab ? { workspaceKey, tabId: tab.tabId } : null;
|
||||
}
|
||||
|
||||
async function openBrowserTabForRequest(params: {
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
serverId?: string;
|
||||
|
||||
@@ -4,6 +4,7 @@ const RESIDENT_VIEWPORT_WIDTH = 1280;
|
||||
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
||||
|
||||
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
|
||||
const residentWebviewSizesByBrowserId = new Map<string, { width: number; height: number }>();
|
||||
|
||||
interface BrowserWebviewElement extends HTMLElement {
|
||||
src: string;
|
||||
@@ -66,11 +67,24 @@ function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLEle
|
||||
return null;
|
||||
}
|
||||
|
||||
function applyResidentWebviewStyle(webview: HTMLElement): void {
|
||||
function dimensionsForBrowser(browserId: string | null): { width: number; height: number } {
|
||||
if (!browserId) {
|
||||
return { width: RESIDENT_VIEWPORT_WIDTH, height: RESIDENT_VIEWPORT_HEIGHT };
|
||||
}
|
||||
return (
|
||||
residentWebviewSizesByBrowserId.get(browserId) ?? {
|
||||
width: RESIDENT_VIEWPORT_WIDTH,
|
||||
height: RESIDENT_VIEWPORT_HEIGHT,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function applyResidentWebviewStyle(webview: HTMLElement, browserId: string | null): void {
|
||||
const dimensions = dimensionsForBrowser(browserId);
|
||||
webview.style.display = "inline-flex";
|
||||
webview.style.flex = "0 0 auto";
|
||||
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
webview.style.width = `${dimensions.width}px`;
|
||||
webview.style.height = `${dimensions.height}px`;
|
||||
webview.style.border = "0";
|
||||
webview.style.background = "transparent";
|
||||
webview.style.position = "absolute";
|
||||
@@ -163,10 +177,33 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl
|
||||
}
|
||||
|
||||
residentWebviewsByBrowserId.set(normalizedBrowserId, webview);
|
||||
applyResidentWebviewStyle(webview);
|
||||
applyResidentWebviewStyle(webview, normalizedBrowserId);
|
||||
getResidentBrowserHost(ownerDocument).appendChild(webview);
|
||||
}
|
||||
|
||||
export function resizeResidentBrowserWebview(input: {
|
||||
browserId: string;
|
||||
width: number;
|
||||
height: number;
|
||||
}): { width: number; height: number } | null {
|
||||
const normalizedBrowserId = trimNonEmpty(input.browserId);
|
||||
if (!normalizedBrowserId) {
|
||||
return null;
|
||||
}
|
||||
const width = Math.max(1, Math.round(input.width));
|
||||
const height = Math.max(1, Math.round(input.height));
|
||||
residentWebviewSizesByBrowserId.set(normalizedBrowserId, { width, height });
|
||||
|
||||
const ownerDocument = readDocument();
|
||||
const webview = ownerDocument ? findBrowserWebview(normalizedBrowserId, ownerDocument) : null;
|
||||
if (webview) {
|
||||
webview.style.width = `${width}px`;
|
||||
webview.style.height = `${height}px`;
|
||||
}
|
||||
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
export function removeResidentBrowserWebview(browserId: string): void {
|
||||
const normalizedBrowserId = trimNonEmpty(browserId);
|
||||
if (!normalizedBrowserId) {
|
||||
@@ -175,6 +212,7 @@ export function removeResidentBrowserWebview(browserId: string): void {
|
||||
|
||||
const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null;
|
||||
residentWebviewsByBrowserId.delete(normalizedBrowserId);
|
||||
residentWebviewSizesByBrowserId.delete(normalizedBrowserId);
|
||||
resident?.remove();
|
||||
}
|
||||
|
||||
@@ -183,5 +221,6 @@ export function clearResidentBrowserWebviewsForTests(): void {
|
||||
webview.remove();
|
||||
}
|
||||
residentWebviewsByBrowserId.clear();
|
||||
residentWebviewSizesByBrowserId.clear();
|
||||
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
|
||||
}
|
||||
|
||||
@@ -128,6 +128,7 @@ export interface DesktopBrowserNewTabRequestEvent {
|
||||
|
||||
export interface DesktopBrowserBridge {
|
||||
registerWorkspaceBrowser?: (input: { browserId: string; workspaceId: string }) => Promise<void>;
|
||||
unregisterWorkspaceBrowser?: (browserId: string) => Promise<void>;
|
||||
setWorkspaceActiveBrowser?: (input: {
|
||||
workspaceId: string;
|
||||
browserId: string | null;
|
||||
|
||||
@@ -1095,12 +1095,823 @@ async function runPermanentParkingGroup() {
|
||||
return results;
|
||||
}
|
||||
|
||||
function automationFixtureUrl() {
|
||||
const html = `<!doctype html>
|
||||
<html>
|
||||
<head><meta charset="utf-8"><title>Automation Fixture</title></head>
|
||||
<style>
|
||||
body { min-height: 1800px; }
|
||||
#hover-target { display: none; }
|
||||
#hover-source:hover + #hover-target { display: inline-block; }
|
||||
#moving {
|
||||
animation: slide 320ms linear;
|
||||
}
|
||||
#drag-target {
|
||||
display: inline-block;
|
||||
margin-left: 24px;
|
||||
padding: 12px;
|
||||
border: 1px solid #888;
|
||||
}
|
||||
@keyframes slide {
|
||||
from { transform: translateX(80px); }
|
||||
to { transform: translateX(0); }
|
||||
}
|
||||
</style>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Settings</h1>
|
||||
<section aria-label="Account">
|
||||
<p>Connected as Maya</p>
|
||||
<label for="name">Name</label>
|
||||
<input id="name" value="Maya">
|
||||
<a href="#docs">Read docs</a>
|
||||
<button id="save">Save changes</button>
|
||||
<button id="delayed" disabled>Delayed save</button>
|
||||
<button id="moving">Moving target</button>
|
||||
<button id="hover-source">Reveal actions</button>
|
||||
<button id="hover-target">Revealed action</button>
|
||||
<button id="drag-source">Drag source</button>
|
||||
<span id="drag-target">Drop target</span>
|
||||
<button id="dialog-alert">Open alert</button>
|
||||
<button id="dialog-confirm">Open confirm</button>
|
||||
<button id="dialog-prompt">Open prompt</button>
|
||||
<button id="dialog-beforeunload">Arm beforeunload</button>
|
||||
<input id="upload" type="file" aria-label="Upload receipt">
|
||||
</section>
|
||||
</main>
|
||||
<script>
|
||||
window.fixtureLog = [];
|
||||
document.getElementById("save").addEventListener("click", (event) => {
|
||||
window.fixtureLog.push({
|
||||
event: "click-save",
|
||||
trusted: event.isTrusted,
|
||||
button: event.button,
|
||||
detail: event.detail,
|
||||
ctrlKey: event.ctrlKey,
|
||||
shiftKey: event.shiftKey,
|
||||
});
|
||||
});
|
||||
document.getElementById("save").addEventListener("mousedown", (event) => {
|
||||
window.fixtureLog.push({
|
||||
event: "down-save",
|
||||
trusted: event.isTrusted,
|
||||
button: event.button,
|
||||
detail: event.detail,
|
||||
ctrlKey: event.ctrlKey,
|
||||
shiftKey: event.shiftKey,
|
||||
});
|
||||
});
|
||||
document.getElementById("name").addEventListener("input", (event) => {
|
||||
window.fixtureLog.push({ event: "input-name", trusted: event.isTrusted });
|
||||
});
|
||||
document.getElementById("delayed").addEventListener("click", (event) => {
|
||||
window.fixtureLog.push({ event: "click-delayed", trusted: event.isTrusted });
|
||||
});
|
||||
document.getElementById("moving").addEventListener("click", (event) => {
|
||||
window.fixtureLog.push({ event: "click-moving", trusted: event.isTrusted });
|
||||
});
|
||||
document.getElementById("drag-source").addEventListener("pointerdown", (event) => {
|
||||
window.fixtureLog.push({ event: "drag-down", trusted: event.isTrusted });
|
||||
});
|
||||
document.getElementById("drag-target").addEventListener("pointerup", (event) => {
|
||||
window.fixtureLog.push({ event: "drag-up", trusted: event.isTrusted });
|
||||
});
|
||||
document.getElementById("dialog-alert").addEventListener("click", () => {
|
||||
alert("Alert opened");
|
||||
window.fixtureLog.push({ event: "alert-returned" });
|
||||
});
|
||||
document.getElementById("dialog-confirm").addEventListener("click", () => {
|
||||
window.fixtureLog.push({ event: "confirm-result", value: confirm("Confirm action?") });
|
||||
});
|
||||
document.getElementById("dialog-prompt").addEventListener("click", () => {
|
||||
window.fixtureLog.push({ event: "prompt-result", value: prompt("Prompt value?", "Maya") });
|
||||
});
|
||||
window.beforeUnloadEnabled = false;
|
||||
document.getElementById("dialog-beforeunload").addEventListener("click", () => {
|
||||
window.beforeUnloadEnabled = true;
|
||||
window.fixtureLog.push({ event: "beforeunload-armed" });
|
||||
});
|
||||
window.addEventListener("beforeunload", (event) => {
|
||||
if (!window.beforeUnloadEnabled) return;
|
||||
event.preventDefault();
|
||||
event.returnValue = "Leave fixture?";
|
||||
});
|
||||
setTimeout(() => {
|
||||
document.getElementById("delayed").disabled = false;
|
||||
}, 250);
|
||||
window.pushFixtureState = () => history.pushState({}, "", "#advanced");
|
||||
window.sameUrlRerender = () => {
|
||||
const oldSave = document.getElementById("save");
|
||||
const nextSave = document.createElement("button");
|
||||
nextSave.id = "save";
|
||||
nextSave.textContent = "Save later";
|
||||
oldSave.replaceWith(nextSave);
|
||||
};
|
||||
</script>
|
||||
</body>
|
||||
</html>`;
|
||||
return `data:text/html;charset=utf-8,${encodeURIComponent(html)}`;
|
||||
}
|
||||
|
||||
const AUTOMATION_SNAPSHOT_PROBE = String.raw`(() => {
|
||||
const refs = new Map();
|
||||
const lines = ['- document "Automation Fixture"'];
|
||||
let nextRef = 1;
|
||||
function text(value) {
|
||||
return String(value || '').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
function fingerprint(element, role, name) {
|
||||
return { role, name, tagName: element.tagName.toLowerCase(), type: element.getAttribute('type') || '', ariaLabel: element.getAttribute('aria-label') || '' };
|
||||
}
|
||||
function runtime() {
|
||||
const api = {
|
||||
refs,
|
||||
resolve(ref, expected) {
|
||||
const element = refs.get(ref);
|
||||
if (!element || !element.isConnected) return { ok: false, reason: 'stale_ref' };
|
||||
const role = roleFor(element);
|
||||
const name = nameFor(element, role);
|
||||
const current = fingerprint(element, role, name);
|
||||
return current.role === expected.role && current.name === expected.name && current.tagName === expected.tagName && current.type === expected.type && current.ariaLabel === expected.ariaLabel
|
||||
? { ok: true, element }
|
||||
: { ok: false, reason: 'stale_ref' };
|
||||
}
|
||||
};
|
||||
Object.defineProperty(window, '__PASEO_BROWSER_AUTOMATION__', { configurable: true, value: api });
|
||||
return api;
|
||||
}
|
||||
function roleFor(element) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (/^h[1-6]$/.test(tag)) return 'heading';
|
||||
if (tag === 'input') return element.type === 'file' ? 'button' : 'textbox';
|
||||
if (tag === 'button') return 'button';
|
||||
if (element.id === 'drag-target') return 'button';
|
||||
if (tag === 'a') return 'link';
|
||||
if (tag === 'section') return 'region';
|
||||
return '';
|
||||
}
|
||||
function nameFor(element, role) {
|
||||
if (element.getAttribute('aria-label')) return element.getAttribute('aria-label');
|
||||
if (element.id) {
|
||||
const label = document.querySelector('label[for="' + element.id + '"]');
|
||||
if (label) return text(label.textContent);
|
||||
}
|
||||
return role === 'textbox' ? text(element.value || element.placeholder) : text(element.textContent);
|
||||
}
|
||||
const api = runtime();
|
||||
const heading = document.querySelector('h1');
|
||||
lines.push(' - heading "' + nameFor(heading, 'heading') + '" [level=1]');
|
||||
lines.push(' - text: "Connected as Maya"');
|
||||
for (const element of document.querySelectorAll('input, a, button, #drag-target')) {
|
||||
const role = roleFor(element);
|
||||
const name = nameFor(element, role);
|
||||
const ref = '@e' + nextRef++;
|
||||
const fp = fingerprint(element, role, name);
|
||||
api.refs.set(ref, element);
|
||||
lines.push(' - ' + role + ' "' + name + '" [ref=' + ref + ']');
|
||||
}
|
||||
return {
|
||||
snapshot: lines.join('\n'),
|
||||
refs: Array.from(api.refs.entries()).map(([ref, element]) => {
|
||||
const role = roleFor(element);
|
||||
return { ref, fingerprint: fingerprint(element, role, nameFor(element, role)) };
|
||||
})
|
||||
};
|
||||
})()`;
|
||||
|
||||
async function attachAutomationDebugger(guest) {
|
||||
if (!guest.debugger.isAttached()) {
|
||||
guest.debugger.attach("1.3");
|
||||
}
|
||||
return (command, params = {}) => guest.debugger.sendCommand(command, params);
|
||||
}
|
||||
|
||||
async function automationRefPoint(guest, ref, fingerprint) {
|
||||
const result = await guest.executeJavaScript(
|
||||
String.raw`(async () => {
|
||||
const fingerprint = ${JSON.stringify(fingerprint)};
|
||||
const ref = ${JSON.stringify(ref)};
|
||||
const deadline = performance.now() + 5000;
|
||||
const nextFrame = () => new Promise((resolve) => requestAnimationFrame(resolve));
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const sameRect = (a, b) =>
|
||||
Math.abs(a.x - b.x) < 0.25 &&
|
||||
Math.abs(a.y - b.y) < 0.25 &&
|
||||
Math.abs(a.width - b.width) < 0.25 &&
|
||||
Math.abs(a.height - b.height) < 0.25;
|
||||
const isDisabled = (element) =>
|
||||
Boolean(element.closest?.('[aria-disabled="true"]')) ||
|
||||
Boolean('disabled' in element && element.disabled);
|
||||
const isVisible = (element, rect) => {
|
||||
const style = getComputedStyle(element);
|
||||
return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
|
||||
};
|
||||
while (performance.now() <= deadline) {
|
||||
const resolved = window.__PASEO_BROWSER_AUTOMATION__.resolve(ref, fingerprint);
|
||||
if (!resolved.ok || !resolved.element?.isConnected) return { ok: false, reason: 'stale_ref' };
|
||||
const element = resolved.element;
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (!isVisible(element, rect) || isDisabled(element)) {
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
await nextFrame();
|
||||
const first = element.getBoundingClientRect();
|
||||
await nextFrame();
|
||||
const second = element.getBoundingClientRect();
|
||||
if (!sameRect(first, second)) continue;
|
||||
const x = Math.min(Math.max(second.left + second.width / 2, 0), Math.max(window.innerWidth - 1, 0));
|
||||
const y = Math.min(Math.max(second.top + second.height / 2, 0), Math.max(window.innerHeight - 1, 0));
|
||||
const hit = document.elementFromPoint(x, y);
|
||||
if (hit && (hit === element || element.contains(hit))) return { ok: true, x, y };
|
||||
await sleep(25);
|
||||
}
|
||||
return { ok: false, reason: 'timeout' };
|
||||
})()`,
|
||||
true,
|
||||
);
|
||||
if (!result || result.ok !== true) {
|
||||
fail(`automation ref ${ref} was not actionable: ${JSON.stringify(result)}`);
|
||||
}
|
||||
return { x: result.x, y: result.y };
|
||||
}
|
||||
|
||||
async function automationClick(guest, refEntry, options = {}) {
|
||||
const send = await attachAutomationDebugger(guest);
|
||||
const point = await automationRefPoint(guest, refEntry.ref, refEntry.fingerprint);
|
||||
const button = options.button || "left";
|
||||
const modifiers = automationModifierMask(options.modifiers || []);
|
||||
const clickCount = options.doubleClick ? 2 : 1;
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button: "none",
|
||||
modifiers,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
buttons: automationButtonMask(button),
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
buttons: 0,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
return point;
|
||||
}
|
||||
|
||||
async function automationHover(guest, refEntry) {
|
||||
const send = await attachAutomationDebugger(guest);
|
||||
const point = await automationRefPoint(guest, refEntry.ref, refEntry.fingerprint);
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button: "none",
|
||||
});
|
||||
}
|
||||
|
||||
async function automationScroll(guest, deltaX, deltaY) {
|
||||
const send = await attachAutomationDebugger(guest);
|
||||
const point = await guest.executeJavaScript(
|
||||
"({ x: Math.max(0, (window.innerWidth || 1) / 2), y: Math.max(0, (window.innerHeight || 1) / 2) })",
|
||||
true,
|
||||
);
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseWheel",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
deltaX,
|
||||
deltaY,
|
||||
});
|
||||
}
|
||||
|
||||
async function automationDrag(guest, sourceRef, targetRef) {
|
||||
const send = await attachAutomationDebugger(guest);
|
||||
const source = await automationRefPoint(guest, sourceRef.ref, sourceRef.fingerprint);
|
||||
const target = await automationRefPoint(guest, targetRef.ref, targetRef.fingerprint);
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: source.x,
|
||||
y: source.y,
|
||||
button: "none",
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: source.x,
|
||||
y: source.y,
|
||||
button: "left",
|
||||
buttons: 1,
|
||||
clickCount: 1,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: target.x,
|
||||
y: target.y,
|
||||
button: "left",
|
||||
buttons: 1,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: target.x,
|
||||
y: target.y,
|
||||
button: "left",
|
||||
buttons: 0,
|
||||
clickCount: 1,
|
||||
});
|
||||
}
|
||||
|
||||
async function automationType(guest, refEntry, text) {
|
||||
const send = await attachAutomationDebugger(guest);
|
||||
await automationClick(guest, refEntry);
|
||||
await send("Input.insertText", { text });
|
||||
}
|
||||
|
||||
async function automationEvaluate(guest, functionSource, refEntry) {
|
||||
return guest.executeJavaScript(
|
||||
String.raw`(async () => {
|
||||
const userFunction = (0, eval)(${JSON.stringify(`(${functionSource})`)});
|
||||
const args = [];
|
||||
${
|
||||
refEntry
|
||||
? `const resolved = window.__PASEO_BROWSER_AUTOMATION__.resolve(${JSON.stringify(refEntry.ref)}, ${JSON.stringify(refEntry.fingerprint)});
|
||||
if (!resolved.ok) return { ok: false, reason: "stale_ref" };
|
||||
args.push(resolved.element);`
|
||||
: ""
|
||||
}
|
||||
return JSON.stringify(await userFunction(...args));
|
||||
})()`,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
const AUTOMATION_DIALOG_POLICY = {
|
||||
alert: { action: "accepted", accept: true },
|
||||
confirm: { action: "dismissed", accept: false },
|
||||
prompt: { action: "dismissed", accept: false },
|
||||
beforeunload: { action: "dismissed", accept: false },
|
||||
};
|
||||
|
||||
const AUTOMATION_PROMPT_SHIM_INSTALL = String.raw`(() => {
|
||||
const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__";
|
||||
const state = window[stateKey] || { prompts: [], installed: false };
|
||||
window[stateKey] = state;
|
||||
if (state.installed) return true;
|
||||
window.prompt = (message = "", defaultValue = "") => {
|
||||
state.prompts.push({
|
||||
type: "prompt",
|
||||
message: String(message ?? ""),
|
||||
defaultValue: String(defaultValue ?? ""),
|
||||
action: "dismissed",
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
return null;
|
||||
};
|
||||
state.installed = true;
|
||||
return true;
|
||||
})()`;
|
||||
|
||||
const AUTOMATION_PROMPT_SHIM_DRAIN = String.raw`(() => {
|
||||
const state = window.__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__;
|
||||
if (!state || !Array.isArray(state.prompts)) return [];
|
||||
return state.prompts.splice(0);
|
||||
})()`;
|
||||
|
||||
async function captureAutomationDialogs(guest, action, expectedCount) {
|
||||
const send = await attachAutomationDebugger(guest);
|
||||
await send("Page.enable");
|
||||
await send("Runtime.evaluate", {
|
||||
expression: AUTOMATION_PROMPT_SHIM_INSTALL,
|
||||
returnByValue: true,
|
||||
});
|
||||
const dialogs = [];
|
||||
const listener = (_event, method, params = {}) => {
|
||||
if (method !== "Page.javascriptDialogOpening") return;
|
||||
const type = AUTOMATION_DIALOG_POLICY[params.type] ? params.type : "alert";
|
||||
const policy = AUTOMATION_DIALOG_POLICY[type];
|
||||
dialogs.push({
|
||||
type,
|
||||
message: String(params.message || ""),
|
||||
...(typeof params.defaultPrompt === "string" ? { defaultValue: params.defaultPrompt } : {}),
|
||||
action: policy.action,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
void send("Page.handleJavaScriptDialog", { accept: policy.accept });
|
||||
};
|
||||
guest.debugger.on("message", listener);
|
||||
try {
|
||||
await action();
|
||||
const promptResult = await send("Runtime.evaluate", {
|
||||
expression: AUTOMATION_PROMPT_SHIM_DRAIN,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (Array.isArray(promptResult.result?.value)) {
|
||||
dialogs.push(...promptResult.result.value);
|
||||
}
|
||||
await waitForDialogCount(dialogs, expectedCount);
|
||||
return dialogs;
|
||||
} finally {
|
||||
guest.debugger.removeListener?.("message", listener);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForDialogCount(dialogs, expectedCount) {
|
||||
const deadline = Date.now() + 1000;
|
||||
do {
|
||||
if (dialogs.length >= expectedCount) return;
|
||||
await delay(25);
|
||||
} while (Date.now() < deadline);
|
||||
fail(`automation observed ${dialogs.length}/${expectedCount} dialogs`);
|
||||
}
|
||||
|
||||
function assertAutomationDialog(dialogs, expected) {
|
||||
const dialog = dialogs.find((entry) => entry.type === expected.type);
|
||||
if (!dialog) {
|
||||
fail(`automation did not report ${expected.type} dialog: ${JSON.stringify(dialogs)}`);
|
||||
}
|
||||
for (const [key, value] of Object.entries(expected)) {
|
||||
if (dialog[key] !== value) {
|
||||
fail(`automation ${expected.type} dialog ${key}=${JSON.stringify(dialog[key])}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function automationButtonMask(button) {
|
||||
if (button === "right") return 2;
|
||||
if (button === "middle") return 4;
|
||||
return 1;
|
||||
}
|
||||
|
||||
function automationModifierMask(modifiers) {
|
||||
const masks = { Alt: 1, Control: 2, Meta: 4, Shift: 8 };
|
||||
return modifiers.reduce((mask, modifier) => mask | masks[modifier], 0);
|
||||
}
|
||||
|
||||
function automationRefByName(snapshot, name) {
|
||||
const entry = snapshot.refs.find((ref) => ref.fingerprint.name === name);
|
||||
if (!entry) {
|
||||
fail(`automation ref missing for ${name}`);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
async function automationLog(guest) {
|
||||
return guest.executeJavaScript("window.fixtureLog", true);
|
||||
}
|
||||
|
||||
async function waitForAutomationLog(guest, predicate, label) {
|
||||
const deadline = Date.now() + 1000;
|
||||
do {
|
||||
const log = await automationLog(guest);
|
||||
if (Array.isArray(log) && log.some(predicate)) {
|
||||
return log;
|
||||
}
|
||||
await delay(25);
|
||||
} while (Date.now() < deadline);
|
||||
fail(`automation log never observed ${label}`);
|
||||
}
|
||||
|
||||
async function runAutomationGroup() {
|
||||
const results = [];
|
||||
const handle = createInactiveHarnessWindow({
|
||||
width: 1000,
|
||||
height: 700,
|
||||
backgroundColor: "#202020",
|
||||
webPreferences: {
|
||||
webviewTag: true,
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
sandbox: false,
|
||||
},
|
||||
});
|
||||
const { win } = handle;
|
||||
installHarnessWebviewGuards(win);
|
||||
const tracker = trackAttachedGuests(win, { disableGuestBackgroundThrottlingAtAttach: true });
|
||||
try {
|
||||
await withTimeout(
|
||||
win.loadFile(path.join(ROOT, "index.html"), {
|
||||
query: {
|
||||
webviewCount: "0",
|
||||
permanentParkingState: "p1-overflow-1x1",
|
||||
targetUrl: automationFixtureUrl(),
|
||||
},
|
||||
}),
|
||||
"automation harness window loadFile",
|
||||
);
|
||||
await waitForInactiveReveal(handle, "automation harness window");
|
||||
const { guest } = await appendPermanentWebview({
|
||||
win,
|
||||
tracker,
|
||||
state: { id: "p1-overflow-1x1" },
|
||||
sourceUrl: automationFixtureUrl(),
|
||||
});
|
||||
await waitForGuestLoad(guest);
|
||||
|
||||
const first = await guest.executeJavaScript(AUTOMATION_SNAPSHOT_PROBE, true);
|
||||
assertAutomationSnapshot(first);
|
||||
pass("automation snapshot renders headings static text controls and refs");
|
||||
results.push({ group: "automation", check: "snapshot", pass: true });
|
||||
|
||||
const saveRef = first.refs.find((ref) => ref.fingerprint.name === "Save changes");
|
||||
if (!saveRef) {
|
||||
fail("automation save ref missing");
|
||||
}
|
||||
await guest.executeJavaScript("window.pushFixtureState()", true);
|
||||
const pushStateResult = await guest.executeJavaScript(
|
||||
`window.__PASEO_BROWSER_AUTOMATION__.resolve(${JSON.stringify(saveRef.ref)}, ${JSON.stringify(saveRef.fingerprint)}).ok`,
|
||||
true,
|
||||
);
|
||||
if (pushStateResult !== true) {
|
||||
fail("automation ref did not survive pushState");
|
||||
}
|
||||
pass("automation ref resolves after pushState");
|
||||
results.push({ group: "automation", check: "pushState-ref", pass: true });
|
||||
|
||||
const pageEvaluate = await automationEvaluate(guest, "() => ({ title: document.title })");
|
||||
const refEvaluate = await automationEvaluate(
|
||||
guest,
|
||||
"(element) => ({ text: element.textContent.trim(), tag: element.tagName.toLowerCase() })",
|
||||
saveRef,
|
||||
);
|
||||
if (
|
||||
pageEvaluate !== '{"title":"Automation Fixture"}' ||
|
||||
refEvaluate !== '{"text":"Save changes","tag":"button"}'
|
||||
) {
|
||||
fail(`automation evaluate returned page=${pageEvaluate} ref=${refEvaluate}`);
|
||||
}
|
||||
pass("automation evaluate runs in page context and receives ref element");
|
||||
results.push({ group: "automation", check: "evaluate", pass: true });
|
||||
|
||||
await guest.executeJavaScript("window.scrollTo(0, 0)", true);
|
||||
await automationScroll(guest, 0, 500);
|
||||
await delay(100);
|
||||
const scrollY = await guest.executeJavaScript("window.scrollY", true);
|
||||
if (!(scrollY > 0)) {
|
||||
fail(`automation scroll did not change window.scrollY: ${scrollY}`);
|
||||
}
|
||||
pass("automation scroll changes the viewport scroll position");
|
||||
results.push({ group: "automation", check: "scroll", pass: true });
|
||||
|
||||
const nameRef = automationRefByName(first, "Name");
|
||||
await automationType(guest, nameRef, " Ada");
|
||||
await automationClick(guest, saveRef);
|
||||
await waitForAutomationLog(
|
||||
guest,
|
||||
(entry) => entry.event === "input-name" && entry.trusted === true,
|
||||
"trusted input event",
|
||||
);
|
||||
await waitForAutomationLog(
|
||||
guest,
|
||||
(entry) => entry.event === "click-save" && entry.trusted === true,
|
||||
"trusted click event",
|
||||
);
|
||||
pass("automation trusted click and text input events reach the page");
|
||||
results.push({ group: "automation", check: "trusted-input", pass: true });
|
||||
|
||||
const hoverRef = automationRefByName(first, "Reveal actions");
|
||||
await automationHover(guest, hoverRef);
|
||||
const hoverVisible = await guest.executeJavaScript(
|
||||
"getComputedStyle(document.getElementById('hover-target')).display !== 'none'",
|
||||
true,
|
||||
);
|
||||
if (hoverVisible !== true) {
|
||||
fail("automation hover did not reveal the hover target");
|
||||
}
|
||||
pass("automation hover reveals CSS :hover content");
|
||||
results.push({ group: "automation", check: "hover-reveal", pass: true });
|
||||
|
||||
const delayedRef = automationRefByName(first, "Delayed save");
|
||||
await automationClick(guest, delayedRef);
|
||||
const delayedLog = await automationLog(guest);
|
||||
if (!delayedLog.some((entry) => entry.event === "click-delayed" && entry.trusted === true)) {
|
||||
fail("automation delayed enabled button did not receive trusted click");
|
||||
}
|
||||
pass("automation action waits for delayed enabled state");
|
||||
results.push({ group: "automation", check: "delayed-enable", pass: true });
|
||||
|
||||
const movingRef = automationRefByName(first, "Moving target");
|
||||
await automationClick(guest, movingRef);
|
||||
const movingLog = await automationLog(guest);
|
||||
if (!movingLog.some((entry) => entry.event === "click-moving" && entry.trusted === true)) {
|
||||
fail("automation moving button did not receive trusted click");
|
||||
}
|
||||
pass("automation action waits for moving element stabilization");
|
||||
results.push({ group: "automation", check: "moving-stable", pass: true });
|
||||
|
||||
const dragSourceRef = automationRefByName(first, "Drag source");
|
||||
const dragTargetRef = automationRefByName(first, "Drop target");
|
||||
await automationDrag(guest, dragSourceRef, dragTargetRef);
|
||||
const dragLog = await automationLog(guest);
|
||||
if (
|
||||
!dragLog.some((entry) => entry.event === "drag-down" && entry.trusted === true) ||
|
||||
!dragLog.some((entry) => entry.event === "drag-up" && entry.trusted === true)
|
||||
) {
|
||||
fail("automation pointer drag did not produce trusted pointer events");
|
||||
}
|
||||
pass("automation pointer-event drag reaches source and target");
|
||||
results.push({ group: "automation", check: "pointer-drag", pass: true });
|
||||
|
||||
await automationClick(guest, saveRef, {
|
||||
button: "right",
|
||||
doubleClick: true,
|
||||
modifiers: ["Control", "Shift"],
|
||||
});
|
||||
const optionsLog = await automationLog(guest);
|
||||
if (
|
||||
!optionsLog.some(
|
||||
(entry) =>
|
||||
entry.event === "down-save" &&
|
||||
entry.trusted === true &&
|
||||
entry.button === 2 &&
|
||||
entry.detail === 2 &&
|
||||
entry.ctrlKey === true &&
|
||||
entry.shiftKey === true,
|
||||
)
|
||||
) {
|
||||
fail("automation click options did not affect button/modifiers/double-click count");
|
||||
}
|
||||
pass("automation click options affect button modifiers and double-click count");
|
||||
results.push({ group: "automation", check: "click-options", pass: true });
|
||||
|
||||
const alertRef = automationRefByName(first, "Open alert");
|
||||
const alertDialogs = await captureAutomationDialogs(
|
||||
guest,
|
||||
() => automationClick(guest, alertRef),
|
||||
1,
|
||||
);
|
||||
assertAutomationDialog(alertDialogs, {
|
||||
type: "alert",
|
||||
message: "Alert opened",
|
||||
action: "accepted",
|
||||
});
|
||||
await waitForAutomationLog(guest, (entry) => entry.event === "alert-returned", "alert return");
|
||||
pass("automation alert dialogs are accepted and reported");
|
||||
results.push({ group: "automation", check: "dialog-alert", pass: true });
|
||||
|
||||
const confirmRef = automationRefByName(first, "Open confirm");
|
||||
const confirmDialogs = await captureAutomationDialogs(
|
||||
guest,
|
||||
() => automationClick(guest, confirmRef),
|
||||
1,
|
||||
);
|
||||
assertAutomationDialog(confirmDialogs, {
|
||||
type: "confirm",
|
||||
message: "Confirm action?",
|
||||
action: "dismissed",
|
||||
});
|
||||
await waitForAutomationLog(
|
||||
guest,
|
||||
(entry) => entry.event === "confirm-result" && entry.value === false,
|
||||
"confirm dismissal",
|
||||
);
|
||||
pass("automation confirm dialogs are dismissed and reported");
|
||||
results.push({ group: "automation", check: "dialog-confirm", pass: true });
|
||||
|
||||
const promptRef = automationRefByName(first, "Open prompt");
|
||||
const promptDialogs = await captureAutomationDialogs(
|
||||
guest,
|
||||
() => automationClick(guest, promptRef),
|
||||
1,
|
||||
);
|
||||
assertAutomationDialog(promptDialogs, {
|
||||
type: "prompt",
|
||||
message: "Prompt value?",
|
||||
defaultValue: "Maya",
|
||||
action: "dismissed",
|
||||
});
|
||||
await waitForAutomationLog(
|
||||
guest,
|
||||
(entry) => entry.event === "prompt-result" && entry.value === null,
|
||||
"prompt dismissal",
|
||||
);
|
||||
pass("automation prompt dialogs are dismissed and reported");
|
||||
results.push({ group: "automation", check: "dialog-prompt", pass: true });
|
||||
|
||||
const beforeUnloadRef = automationRefByName(first, "Arm beforeunload");
|
||||
await automationClick(guest, beforeUnloadRef);
|
||||
const beforeUrl = guest.getURL();
|
||||
const beforeUnloadDialogs = await captureAutomationDialogs(
|
||||
guest,
|
||||
async () => {
|
||||
await guest.loadURL(automationFixtureUrl()).catch(() => {});
|
||||
},
|
||||
1,
|
||||
);
|
||||
assertAutomationDialog(beforeUnloadDialogs, {
|
||||
type: "beforeunload",
|
||||
action: "dismissed",
|
||||
});
|
||||
if (guest.getURL() !== beforeUrl) {
|
||||
fail("automation beforeunload dismissal did not cancel navigation");
|
||||
}
|
||||
pass("automation beforeunload dialogs are dismissed and reported");
|
||||
results.push({ group: "automation", check: "dialog-beforeunload", pass: true });
|
||||
|
||||
await guest.executeJavaScript("window.sameUrlRerender()", true);
|
||||
const rerenderResult = await guest.executeJavaScript(
|
||||
`window.__PASEO_BROWSER_AUTOMATION__.resolve(${JSON.stringify(saveRef.ref)}, ${JSON.stringify(saveRef.fingerprint)}).reason`,
|
||||
true,
|
||||
);
|
||||
if (rerenderResult !== "stale_ref") {
|
||||
fail(`automation same-url rerender returned ${rerenderResult}`);
|
||||
}
|
||||
pass("automation same-url rerender stales old ref");
|
||||
results.push({ group: "automation", check: "same-url-stale-ref", pass: true });
|
||||
|
||||
const second = await guest.executeJavaScript(AUTOMATION_SNAPSHOT_PROBE, true);
|
||||
const uploadRef = second.refs.find((ref) => ref.fingerprint.name === "Upload receipt");
|
||||
if (!uploadRef) {
|
||||
fail("automation upload ref missing");
|
||||
}
|
||||
if (!guest.debugger.isAttached()) {
|
||||
guest.debugger.attach("1.3");
|
||||
}
|
||||
const evaluated = await guest.debugger.sendCommand("Runtime.evaluate", {
|
||||
expression: `(() => window.__PASEO_BROWSER_AUTOMATION__.resolve(${JSON.stringify(uploadRef.ref)}, ${JSON.stringify(uploadRef.fingerprint)}).element)()`,
|
||||
objectGroup: "paseo-browser-automation",
|
||||
returnByValue: false,
|
||||
});
|
||||
const described = await guest.debugger.sendCommand("DOM.describeNode", {
|
||||
objectId: evaluated.result.objectId,
|
||||
});
|
||||
if (!described.node || typeof described.node.backendNodeId !== "number") {
|
||||
fail("automation upload ref did not resolve to backendNodeId");
|
||||
}
|
||||
pass("automation upload ref resolves to backendNodeId");
|
||||
results.push({ group: "automation", check: "upload-backend-node", pass: true });
|
||||
|
||||
// Resize is not harness-testable: the harness hosts webviews in the parked
|
||||
// 1px resident host, and Electron does not propagate CSS-box resizes to a
|
||||
// parked guest's capture surface (see docs/browser-capture-harness.md).
|
||||
// The production resize path is app-owned webview sizing, covered by
|
||||
// packages/app/src/browser-automation/handler.test.ts.
|
||||
|
||||
await fsp.writeFile(
|
||||
path.join(OUT_DIR, "automation-results.json"),
|
||||
`${JSON.stringify({ generatedAt: new Date().toISOString(), results }, null, 2)}\n`,
|
||||
);
|
||||
return results;
|
||||
} finally {
|
||||
await closeHarnessWindow(win);
|
||||
}
|
||||
}
|
||||
|
||||
function assertAutomationSnapshot(snapshot) {
|
||||
const text = snapshot && snapshot.snapshot;
|
||||
if (typeof text !== "string") {
|
||||
fail("automation snapshot returned no text");
|
||||
}
|
||||
for (const expected of [
|
||||
'heading "Settings"',
|
||||
'text: "Connected as Maya"',
|
||||
'textbox "Name" [ref=@e1]',
|
||||
'link "Read docs"',
|
||||
'button "Save changes"',
|
||||
'button "Upload receipt"',
|
||||
]) {
|
||||
if (!text.includes(expected)) {
|
||||
fail(`automation snapshot missing ${expected}`);
|
||||
}
|
||||
}
|
||||
if (text.includes("selector")) {
|
||||
fail("automation snapshot exposed selector text");
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
ensureDirSync(OUT_DIR);
|
||||
if (!["all", "existing", "permanent-parking"].includes(HARNESS_GROUP)) {
|
||||
if (!["all", "existing", "permanent-parking", "automation"].includes(HARNESS_GROUP)) {
|
||||
fail(`unknown harness group ${HARNESS_GROUP}`);
|
||||
}
|
||||
|
||||
if (HARNESS_GROUP === "automation") {
|
||||
const automationResults = await runAutomationGroup();
|
||||
await fsp.writeFile(
|
||||
path.join(OUT_DIR, "results.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
generatedAt: new Date().toISOString(),
|
||||
automationResults,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
pass(`capture harness automation complete output=${OUT_DIR}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (HARNESS_GROUP === "permanent-parking") {
|
||||
const permanentParkingResults = await runPermanentParkingGroup();
|
||||
await fsp.writeFile(
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
import type { SnapshotPage } from "./snapshot-engine.js";
|
||||
|
||||
export interface ActionablePoint {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface ActionableTarget {
|
||||
point: ActionablePoint;
|
||||
rect: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
}
|
||||
|
||||
export type ActionabilityResult =
|
||||
| { ok: true; target: ActionableTarget }
|
||||
| { ok: false; reason: "stale_ref" | "timeout"; detail?: string };
|
||||
|
||||
const DEFAULT_ACTIONABILITY_TIMEOUT_MS = 5_000;
|
||||
|
||||
export async function waitForActionableTarget(input: {
|
||||
page: SnapshotPage;
|
||||
elementExpression: string;
|
||||
editable?: boolean;
|
||||
timeoutMs?: number;
|
||||
}): Promise<ActionabilityResult> {
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildActionabilityScript({
|
||||
elementExpression: input.elementExpression,
|
||||
editable: input.editable === true,
|
||||
timeoutMs: input.timeoutMs ?? DEFAULT_ACTIONABILITY_TIMEOUT_MS,
|
||||
}),
|
||||
);
|
||||
return readActionabilityResult(result);
|
||||
}
|
||||
|
||||
function readActionabilityResult(value: unknown): ActionabilityResult {
|
||||
if (!value || typeof value !== "object") {
|
||||
return { ok: false, reason: "timeout" };
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
if (record.ok === true && isActionableTarget(record.target)) {
|
||||
return { ok: true, target: record.target };
|
||||
}
|
||||
if (record.ok === false) {
|
||||
const reason = record.reason;
|
||||
if (reason === "stale_ref" || reason === "timeout") {
|
||||
return {
|
||||
ok: false,
|
||||
reason,
|
||||
...(typeof record.detail === "string" ? { detail: record.detail } : {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
return { ok: false, reason: "timeout" };
|
||||
}
|
||||
|
||||
function isActionableTarget(value: unknown): value is ActionableTarget {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return isPoint(record.point) && isRect(record.rect);
|
||||
}
|
||||
|
||||
function isPoint(value: unknown): value is ActionablePoint {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return isFiniteNumber(record.x) && isFiniteNumber(record.y);
|
||||
}
|
||||
|
||||
function isRect(value: unknown): value is ActionableTarget["rect"] {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return (
|
||||
isFiniteNumber(record.x) &&
|
||||
isFiniteNumber(record.y) &&
|
||||
isFiniteNumber(record.width) &&
|
||||
isFiniteNumber(record.height)
|
||||
);
|
||||
}
|
||||
|
||||
function isFiniteNumber(value: unknown): value is number {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
function buildActionabilityScript(input: {
|
||||
elementExpression: string;
|
||||
editable: boolean;
|
||||
timeoutMs: number;
|
||||
}): string {
|
||||
return String.raw`(async () => {
|
||||
const deadline = performance.now() + ${JSON.stringify(input.timeoutMs)};
|
||||
const requiresEditable = ${JSON.stringify(input.editable)};
|
||||
|
||||
const nextFrame = () => new Promise((resolve) => requestAnimationFrame(resolve));
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const nearlyEqual = (a, b) => Math.abs(a - b) < 0.25;
|
||||
const sameRect = (a, b) =>
|
||||
nearlyEqual(a.x, b.x) &&
|
||||
nearlyEqual(a.y, b.y) &&
|
||||
nearlyEqual(a.width, b.width) &&
|
||||
nearlyEqual(a.height, b.height);
|
||||
const rectPayload = (rect) => ({
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
});
|
||||
const centerPoint = (rect) => ({
|
||||
x: Math.min(Math.max(rect.left + rect.width / 2, 0), Math.max(window.innerWidth - 1, 0)),
|
||||
y: Math.min(Math.max(rect.top + rect.height / 2, 0), Math.max(window.innerHeight - 1, 0)),
|
||||
});
|
||||
const isDisabled = (element) => {
|
||||
if (element.closest?.('[aria-disabled="true"]')) return true;
|
||||
if ('disabled' in element && element.disabled) return true;
|
||||
const fieldset = element.closest?.('fieldset[disabled]');
|
||||
return Boolean(fieldset);
|
||||
};
|
||||
const isEditable = (element) => {
|
||||
if (element.isContentEditable) return true;
|
||||
const tag = element.tagName?.toLowerCase();
|
||||
if (tag === 'textarea' || tag === 'select') return !element.readOnly && !isDisabled(element);
|
||||
if (tag !== 'input') return false;
|
||||
const type = (element.getAttribute('type') || 'text').toLowerCase();
|
||||
if (['button', 'checkbox', 'color', 'file', 'hidden', 'image', 'radio', 'range', 'reset', 'submit'].includes(type)) return false;
|
||||
return !element.readOnly && !isDisabled(element);
|
||||
};
|
||||
const isVisible = (element, rect) => {
|
||||
const style = getComputedStyle(element);
|
||||
return (
|
||||
rect.width > 0 &&
|
||||
rect.height > 0 &&
|
||||
style.visibility !== 'hidden' &&
|
||||
style.display !== 'none' &&
|
||||
Number(style.opacity || '1') !== 0
|
||||
);
|
||||
};
|
||||
const hitTargetReceivesEvents = (element, point) => {
|
||||
const hit = document.elementFromPoint(point.x, point.y);
|
||||
return Boolean(hit && (hit === element || element.contains(hit)));
|
||||
};
|
||||
const resolveElement = () => (${input.elementExpression});
|
||||
|
||||
let detail = 'not actionable';
|
||||
while (performance.now() <= deadline) {
|
||||
const element = resolveElement();
|
||||
if (!element || !element.isConnected) {
|
||||
return { ok: false, reason: 'stale_ref', detail: 'ref no longer resolves' };
|
||||
}
|
||||
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (!isVisible(element, rect)) {
|
||||
detail = 'not visible';
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
if (isDisabled(element)) {
|
||||
detail = 'disabled';
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
if (requiresEditable && !isEditable(element)) {
|
||||
detail = 'not editable';
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
await nextFrame();
|
||||
const firstRect = element.getBoundingClientRect();
|
||||
await nextFrame();
|
||||
const secondRect = element.getBoundingClientRect();
|
||||
if (!sameRect(firstRect, secondRect)) {
|
||||
detail = 'moving';
|
||||
continue;
|
||||
}
|
||||
|
||||
const point = centerPoint(secondRect);
|
||||
if (!hitTargetReceivesEvents(element, point)) {
|
||||
detail = 'covered';
|
||||
await sleep(25);
|
||||
continue;
|
||||
}
|
||||
|
||||
return { ok: true, target: { point, rect: rectPayload(secondRect) } };
|
||||
}
|
||||
|
||||
return { ok: false, reason: 'timeout', detail };
|
||||
})()`;
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
export const ARIA_SNAPSHOT_SCRIPT_MARKER = "__PASEO_ARIA_SNAPSHOT__";
|
||||
|
||||
// Adapted from Playwright's injected ARIA snapshot model.
|
||||
// Copyright (c) Microsoft Corporation. Licensed under the Apache License, Version 2.0.
|
||||
export const ARIA_SNAPSHOT_SCRIPT = String.raw`(() => {
|
||||
const MARKER = ${JSON.stringify(ARIA_SNAPSHOT_SCRIPT_MARKER)};
|
||||
const MAX_NODES = 1500;
|
||||
const MAX_REFS = 500;
|
||||
const MAX_TEXT_LENGTH = 80000;
|
||||
const ACTIONABLE_ROLES = new Set([
|
||||
'button',
|
||||
'checkbox',
|
||||
'combobox',
|
||||
'link',
|
||||
'menuitem',
|
||||
'option',
|
||||
'radio',
|
||||
'searchbox',
|
||||
'slider',
|
||||
'spinbutton',
|
||||
'switch',
|
||||
'tab',
|
||||
'textbox',
|
||||
'treeitem'
|
||||
]);
|
||||
|
||||
function normalizeText(value) {
|
||||
return String(value || '').replace(/[\u200b\u00ad]/g, '').replace(/[\r\n\s\t]+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function visibilityFor(element) {
|
||||
if (!(element instanceof Element)) return false;
|
||||
const style = window.getComputedStyle(element);
|
||||
if (style.display === 'none' || style.visibility === 'hidden' || Number(style.opacity) === 0) return false;
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0 ? 'box' : 'boxless';
|
||||
}
|
||||
|
||||
function explicitRole(element) {
|
||||
const role = element.getAttribute('role');
|
||||
if (!role || role === 'presentation' || role === 'none') return null;
|
||||
return role.split(/\s+/)[0].toLowerCase();
|
||||
}
|
||||
|
||||
function implicitRole(element) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (/^h[1-6]$/.test(tag)) return 'heading';
|
||||
if (tag === 'a' && element.hasAttribute('href')) return 'link';
|
||||
if (tag === 'button') return 'button';
|
||||
if (tag === 'select') return 'combobox';
|
||||
if (tag === 'textarea') return 'textbox';
|
||||
if (element instanceof HTMLElement && element.isContentEditable) return 'textbox';
|
||||
if (tag === 'summary') return 'button';
|
||||
if (tag === 'main') return 'main';
|
||||
if (tag === 'nav') return 'navigation';
|
||||
if (tag === 'header') return 'banner';
|
||||
if (tag === 'footer') return 'contentinfo';
|
||||
if (tag === 'section') return element.getAttribute('aria-label') ? 'region' : null;
|
||||
if (tag === 'ul' || tag === 'ol') return 'list';
|
||||
if (tag === 'li') return 'listitem';
|
||||
if (tag === 'table') return 'table';
|
||||
if (tag === 'tr') return 'row';
|
||||
if (tag === 'th') return 'columnheader';
|
||||
if (tag === 'td') return 'cell';
|
||||
if (tag === 'iframe') return 'iframe';
|
||||
if (tag === 'input') {
|
||||
const type = (element.getAttribute('type') || 'text').toLowerCase();
|
||||
if (type === 'checkbox') return 'checkbox';
|
||||
if (type === 'radio') return 'radio';
|
||||
if (type === 'button' || type === 'submit' || type === 'reset') return 'button';
|
||||
if (type === 'range') return 'slider';
|
||||
if (type === 'number') return 'spinbutton';
|
||||
if (type === 'search') return 'searchbox';
|
||||
if (type === 'hidden') return null;
|
||||
return 'textbox';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function roleFor(element) {
|
||||
return explicitRole(element) || implicitRole(element);
|
||||
}
|
||||
|
||||
function labelText(element) {
|
||||
if (!(element instanceof HTMLElement)) return '';
|
||||
if (element.id) {
|
||||
const escapedId = window.CSS && typeof window.CSS.escape === 'function'
|
||||
? window.CSS.escape(element.id)
|
||||
: String(element.id).replace(/"/g, '\\"');
|
||||
const label = document.querySelector('label[for="' + escapedId + '"]');
|
||||
if (label) return normalizeText(label.textContent);
|
||||
}
|
||||
const closestLabel = element.closest('label');
|
||||
return closestLabel ? normalizeText(closestLabel.textContent) : '';
|
||||
}
|
||||
|
||||
function nameFor(element, role) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const labelledBy = element.getAttribute('aria-labelledby');
|
||||
if (labelledBy) {
|
||||
const text = labelledBy.split(/\s+/).map((id) => document.getElementById(id)?.textContent || '').join(' ');
|
||||
const normalized = normalizeText(text);
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
const pieces = [
|
||||
element.getAttribute('aria-label'),
|
||||
labelText(element),
|
||||
element.getAttribute('alt'),
|
||||
element.getAttribute('title'),
|
||||
tag === 'input' || tag === 'textarea' ? element.getAttribute('placeholder') : null,
|
||||
tag === 'input' || tag === 'textarea' ? element.value : null,
|
||||
role === 'button' || role === 'link' || role === 'heading' || ACTIONABLE_ROLES.has(role)
|
||||
? element.textContent
|
||||
: null
|
||||
];
|
||||
return normalizeText(pieces.find((piece) => normalizeText(piece).length > 0) || '');
|
||||
}
|
||||
|
||||
function fingerprintNameFor(element, role, name) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (tag !== 'input' && tag !== 'textarea') return name;
|
||||
const mutableValue = normalizeText(element.value);
|
||||
if (!mutableValue || name !== mutableValue) return name;
|
||||
return '';
|
||||
}
|
||||
|
||||
function inheritedDisabled(element) {
|
||||
if (!(element instanceof Element)) return false;
|
||||
if (element.hasAttribute('disabled') || element.getAttribute('aria-disabled') === 'true') return true;
|
||||
if (element.closest('fieldset[disabled]')) return true;
|
||||
return element.closest('[aria-disabled="true"]') !== null;
|
||||
}
|
||||
|
||||
function isActionable(element, role) {
|
||||
if (!role || !ACTIONABLE_ROLES.has(role)) return false;
|
||||
if (visibilityFor(element) !== 'box') return false;
|
||||
if (inheritedDisabled(element)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function fingerprintFor(element, role, name) {
|
||||
return {
|
||||
role,
|
||||
name: fingerprintNameFor(element, role, name),
|
||||
tagName: element.tagName.toLowerCase(),
|
||||
type: element.getAttribute('type') || '',
|
||||
ariaLabel: element.getAttribute('aria-label') || ''
|
||||
};
|
||||
}
|
||||
|
||||
function fingerprintMatches(element, fingerprint) {
|
||||
const role = roleFor(element) || 'generic';
|
||||
const name = nameFor(element, role);
|
||||
const current = fingerprintFor(element, role, name);
|
||||
return current.role === fingerprint.role &&
|
||||
current.name === fingerprint.name &&
|
||||
current.tagName === fingerprint.tagName &&
|
||||
current.type === fingerprint.type &&
|
||||
current.ariaLabel === fingerprint.ariaLabel;
|
||||
}
|
||||
|
||||
function ensureRuntime() {
|
||||
const runtime = {
|
||||
refs: new Map(),
|
||||
resolve(ref, fingerprint) {
|
||||
const element = this.refs.get(ref);
|
||||
if (!element || !element.isConnected || !fingerprintMatches(element, fingerprint)) {
|
||||
return { ok: false, reason: 'stale_ref' };
|
||||
}
|
||||
return { ok: true, element };
|
||||
}
|
||||
};
|
||||
Object.defineProperty(window, '__PASEO_BROWSER_AUTOMATION__', {
|
||||
configurable: true,
|
||||
enumerable: false,
|
||||
value: runtime
|
||||
});
|
||||
return runtime;
|
||||
}
|
||||
|
||||
function stateAttributes(element, role) {
|
||||
const attrs = [];
|
||||
if (role === 'heading') {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const level = /^h[1-6]$/.test(tag) ? Number(tag.slice(1)) : Number(element.getAttribute('aria-level') || 0);
|
||||
if (level) attrs.push('level=' + level);
|
||||
}
|
||||
if (element.matches?.('input[type="checkbox"], input[type="radio"]')) {
|
||||
attrs.push('checked=' + (element.checked ? 'true' : 'false'));
|
||||
}
|
||||
if (element.getAttribute('aria-checked')) attrs.push('checked=' + element.getAttribute('aria-checked'));
|
||||
if (element.getAttribute('aria-expanded')) attrs.push('expanded=' + element.getAttribute('aria-expanded'));
|
||||
if (element.getAttribute('aria-pressed')) attrs.push('pressed=' + element.getAttribute('aria-pressed'));
|
||||
if (element.getAttribute('aria-selected')) attrs.push('selected=' + element.getAttribute('aria-selected'));
|
||||
return attrs;
|
||||
}
|
||||
|
||||
function textNode(text) {
|
||||
return { kind: 'text', text };
|
||||
}
|
||||
|
||||
function elementNode(element, role, name) {
|
||||
return {
|
||||
kind: 'role',
|
||||
role: role || 'generic',
|
||||
name,
|
||||
tagName: element.tagName.toLowerCase(),
|
||||
attributes: stateAttributes(element, role),
|
||||
children: []
|
||||
};
|
||||
}
|
||||
|
||||
let nodeCount = 0;
|
||||
let refCount = 0;
|
||||
let iframeCount = 0;
|
||||
let maxDepth = 0;
|
||||
let truncated = false;
|
||||
let textBudget = MAX_TEXT_LENGTH;
|
||||
const runtime = ensureRuntime();
|
||||
|
||||
function countNode(depth) {
|
||||
if (nodeCount >= MAX_NODES) {
|
||||
truncated = true;
|
||||
return false;
|
||||
}
|
||||
nodeCount += 1;
|
||||
maxDepth = Math.max(maxDepth, depth);
|
||||
return true;
|
||||
}
|
||||
|
||||
function cappedText(text) {
|
||||
if (text.length <= textBudget) {
|
||||
textBudget -= text.length;
|
||||
return text;
|
||||
}
|
||||
truncated = true;
|
||||
const capped = text.slice(0, Math.max(0, textBudget));
|
||||
textBudget = 0;
|
||||
return capped;
|
||||
}
|
||||
|
||||
function visitNode(domNode, depth) {
|
||||
if (!countNode(depth)) return null;
|
||||
if (domNode.nodeType === Node.TEXT_NODE) {
|
||||
const text = cappedText(normalizeText(domNode.textContent));
|
||||
if (!text) return null;
|
||||
return textNode(text);
|
||||
}
|
||||
if (!(domNode instanceof Element)) return null;
|
||||
const visibility = visibilityFor(domNode);
|
||||
if (!visibility) return null;
|
||||
if (domNode.getAttribute('aria-hidden') === 'true') return null;
|
||||
|
||||
const role = roleFor(domNode);
|
||||
const name = role ? nameFor(domNode, role) : '';
|
||||
const children = [];
|
||||
for (const child of Array.from(domNode.childNodes)) {
|
||||
const childSnapshot = visitNode(child, depth + 1);
|
||||
if (childSnapshot) children.push(childSnapshot);
|
||||
if (truncated) break;
|
||||
}
|
||||
|
||||
if (domNode.tagName.toLowerCase() === 'iframe') {
|
||||
iframeCount += 1;
|
||||
}
|
||||
|
||||
if (visibility === 'boxless') {
|
||||
return children.length > 0 ? { kind: 'group', children } : null;
|
||||
}
|
||||
if (!role && children.length === 0) return null;
|
||||
const snapshotNode = role ? elementNode(domNode, role, name) : { kind: 'group', children: [] };
|
||||
snapshotNode.children = children;
|
||||
if (role && isActionable(domNode, role) && refCount < MAX_REFS) {
|
||||
const ref = '@e' + (refCount + 1);
|
||||
const fingerprint = fingerprintFor(domNode, role, name);
|
||||
refCount += 1;
|
||||
runtime.refs.set(ref, domNode);
|
||||
snapshotNode.ref = ref;
|
||||
snapshotNode.fingerprint = fingerprint;
|
||||
} else if (role && isActionable(domNode, role)) {
|
||||
truncated = true;
|
||||
}
|
||||
return snapshotNode;
|
||||
}
|
||||
|
||||
const root = {
|
||||
kind: 'role',
|
||||
role: 'document',
|
||||
name: normalizeText(document.title),
|
||||
tagName: 'document',
|
||||
attributes: [],
|
||||
children: []
|
||||
};
|
||||
for (const child of Array.from(document.body ? document.body.childNodes : document.documentElement.childNodes)) {
|
||||
const childSnapshot = visitNode(child, 1);
|
||||
if (childSnapshot) root.children.push(childSnapshot);
|
||||
if (truncated) break;
|
||||
}
|
||||
|
||||
return JSON.stringify({
|
||||
marker: MARKER,
|
||||
root,
|
||||
refs: Array.from(runtime.refs.entries()).map(([ref, element]) => {
|
||||
const role = roleFor(element) || 'generic';
|
||||
const name = nameFor(element, role);
|
||||
return { ref, fingerprint: fingerprintFor(element, role, name) };
|
||||
}),
|
||||
truncated,
|
||||
stats: { nodeCount, refCount, textLength: 0, iframeCount, maxDepth }
|
||||
});
|
||||
})()`;
|
||||
@@ -0,0 +1,28 @@
|
||||
export type CdpCommandSender = (
|
||||
command: string,
|
||||
params?: Record<string, unknown>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export class CdpSessionQueue {
|
||||
private queue: Promise<void> = Promise.resolve();
|
||||
|
||||
public async run<T>(task: () => Promise<T>): Promise<T> {
|
||||
const previous = this.queue;
|
||||
let releaseCurrent = () => {};
|
||||
const current = new Promise<void>((resolve) => {
|
||||
releaseCurrent = resolve;
|
||||
});
|
||||
const tail = previous.catch(() => {}).then(() => current);
|
||||
this.queue = tail;
|
||||
|
||||
await previous.catch(() => {});
|
||||
try {
|
||||
return await task();
|
||||
} finally {
|
||||
releaseCurrent();
|
||||
if (this.queue === tail) {
|
||||
this.queue = Promise.resolve();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import type { BrowserAutomationDialogEvent } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
|
||||
export const MAX_DIALOGS_PER_COMMAND = 20;
|
||||
|
||||
export interface JavaScriptDialogOpening {
|
||||
type?: unknown;
|
||||
message?: unknown;
|
||||
defaultPrompt?: unknown;
|
||||
}
|
||||
|
||||
interface DialogPolicy {
|
||||
readonly action: BrowserAutomationDialogEvent["action"];
|
||||
readonly accept: boolean;
|
||||
}
|
||||
|
||||
const DIALOG_POLICIES: Record<BrowserAutomationDialogEvent["type"], DialogPolicy> = {
|
||||
alert: { action: "accepted", accept: true },
|
||||
confirm: { action: "dismissed", accept: false },
|
||||
prompt: { action: "dismissed", accept: false },
|
||||
beforeunload: { action: "dismissed", accept: false },
|
||||
};
|
||||
|
||||
export function handledDialogEvent(opening: JavaScriptDialogOpening): BrowserAutomationDialogEvent {
|
||||
const type = normalizeDialogType(opening.type);
|
||||
const policy = DIALOG_POLICIES[type];
|
||||
return {
|
||||
type,
|
||||
message: typeof opening.message === "string" ? opening.message : String(opening.message ?? ""),
|
||||
...(typeof opening.defaultPrompt === "string" ? { defaultValue: opening.defaultPrompt } : {}),
|
||||
action: policy.action,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
export function dialogAcceptValue(type: BrowserAutomationDialogEvent["type"]): boolean {
|
||||
return DIALOG_POLICIES[type].accept;
|
||||
}
|
||||
|
||||
export function promptShimInstallScript(): string {
|
||||
return String.raw`(() => {
|
||||
const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__";
|
||||
const state = window[stateKey] || { prompts: [], installed: false };
|
||||
window[stateKey] = state;
|
||||
if (state.installed) return true;
|
||||
const originalPrompt = window.prompt;
|
||||
Object.defineProperty(state, "originalPrompt", { configurable: true, value: originalPrompt });
|
||||
const promptShim = (message = "", defaultValue = "") => {
|
||||
state.prompts.push({
|
||||
type: "prompt",
|
||||
message: String(message ?? ""),
|
||||
defaultValue: String(defaultValue ?? ""),
|
||||
action: "dismissed",
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
return null;
|
||||
};
|
||||
Object.defineProperty(state, "promptShim", { configurable: true, value: promptShim });
|
||||
window.prompt = promptShim;
|
||||
state.installed = true;
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function promptShimDrainScript(): string {
|
||||
return String.raw`(() => {
|
||||
const state = window.__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__;
|
||||
if (!state || !Array.isArray(state.prompts)) return [];
|
||||
return state.prompts.splice(0);
|
||||
})()`;
|
||||
}
|
||||
|
||||
export function promptShimRestoreScript(): string {
|
||||
return String.raw`(() => {
|
||||
const stateKey = "__PASEO_BROWSER_AUTOMATION_DIALOG_STATE__";
|
||||
const state = window[stateKey];
|
||||
if (!state || !state.installed) return true;
|
||||
if (window.prompt === state.promptShim && typeof state.originalPrompt === "function") {
|
||||
window.prompt = state.originalPrompt;
|
||||
}
|
||||
delete window[stateKey];
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function normalizeDialogType(value: unknown): BrowserAutomationDialogEvent["type"] {
|
||||
return value === "alert" || value === "confirm" || value === "prompt" || value === "beforeunload"
|
||||
? value
|
||||
: "alert";
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
import type { TabImage } from "./service.js";
|
||||
import { adaptWebContents } from "./ipc.js";
|
||||
|
||||
@@ -16,6 +16,15 @@ class FakeImage implements TabImage {
|
||||
class FakeDebugger {
|
||||
public attachedProtocolVersions: string[] = [];
|
||||
public commands: Array<{ command: string; params: Record<string, unknown> }> = [];
|
||||
public blockCommands = false;
|
||||
public readonly blockedCommandNames = new Set<string>();
|
||||
public readonly failedCommandNames = new Set<string>();
|
||||
public readonly promptDialogs: unknown[] = [];
|
||||
public failPromptDrain = false;
|
||||
private messageListener:
|
||||
| ((event: unknown, method: string, params?: Record<string, unknown>) => void)
|
||||
| null = null;
|
||||
private readonly blockedCommands: Array<() => void> = [];
|
||||
|
||||
public isAttached(): boolean {
|
||||
return this.attachedProtocolVersions.length > 0;
|
||||
@@ -27,8 +36,48 @@ class FakeDebugger {
|
||||
|
||||
public async sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown> {
|
||||
this.commands.push({ command, params: params ?? {} });
|
||||
if (this.failedCommandNames.has(command)) {
|
||||
throw new Error(`${command} failed`);
|
||||
}
|
||||
if (this.blockCommands || this.blockedCommandNames.has(command)) {
|
||||
await new Promise<void>((resolve) => {
|
||||
this.blockedCommands.push(resolve);
|
||||
});
|
||||
}
|
||||
if (command === "Runtime.evaluate" && typeof params?.expression === "string") {
|
||||
if (params.expression.includes("state.prompts.splice(0)")) {
|
||||
if (this.failPromptDrain) {
|
||||
throw new Error("execution context destroyed");
|
||||
}
|
||||
return { result: { value: this.promptDialogs.splice(0) } };
|
||||
}
|
||||
return { result: { value: true } };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
public on(
|
||||
event: "message",
|
||||
listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
|
||||
): void {
|
||||
expect(event).toBe("message");
|
||||
this.messageListener = listener;
|
||||
}
|
||||
|
||||
public emitMessage(method: string, params?: Record<string, unknown>): void {
|
||||
if (!this.messageListener) {
|
||||
throw new Error("Debugger message listener was not registered");
|
||||
}
|
||||
this.messageListener({}, method, params);
|
||||
}
|
||||
|
||||
public finishNextCommand(): void {
|
||||
const resolve = this.blockedCommands.shift();
|
||||
if (!resolve) {
|
||||
throw new Error("No command is blocked");
|
||||
}
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
|
||||
type ConsoleMessageListener = (
|
||||
@@ -181,4 +230,323 @@ describe("browser automation IPC adapter", () => {
|
||||
{ command: "Page.captureScreenshot", params: { format: "png" } },
|
||||
]);
|
||||
});
|
||||
|
||||
test("serializes CDP commands per guest contents", async () => {
|
||||
const contents = new FakeWebContents(23);
|
||||
contents.debugger.blockCommands = true;
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
const first = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mouseMoved" });
|
||||
const second = tab.sendDebugCommand?.("Page.captureScreenshot", { format: "png" });
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mouseMoved" } },
|
||||
]);
|
||||
|
||||
contents.debugger.finishNextCommand();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mouseMoved" } },
|
||||
{ command: "Page.captureScreenshot", params: { format: "png" } },
|
||||
]);
|
||||
|
||||
contents.debugger.finishNextCommand();
|
||||
await expect(first).resolves.toEqual({ ok: true });
|
||||
await expect(second).resolves.toEqual({ ok: true });
|
||||
});
|
||||
|
||||
test("handles JavaScript dialogs through the per-tab CDP queue", async () => {
|
||||
const contents = new FakeWebContents(24);
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
const captured = tab.captureDialogs?.(async () => {
|
||||
contents.debugger.blockCommands = true;
|
||||
const input = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mouseReleased" });
|
||||
await flushMicrotasks();
|
||||
|
||||
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "Delete item?",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Page.enable", params: {} },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mouseReleased" } },
|
||||
{ command: "Page.handleJavaScriptDialog", params: { accept: false } },
|
||||
]);
|
||||
|
||||
contents.debugger.blockCommands = false;
|
||||
contents.debugger.finishNextCommand();
|
||||
await input;
|
||||
await flushMicrotasks();
|
||||
return "done";
|
||||
});
|
||||
|
||||
await expect(captured).resolves.toEqual({
|
||||
result: "done",
|
||||
dialogs: [
|
||||
{
|
||||
type: "confirm",
|
||||
message: "Delete item?",
|
||||
action: "dismissed",
|
||||
timestamp: expect.any(Number),
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Page.enable", params: {} },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mouseReleased" } },
|
||||
{ command: "Page.handleJavaScriptDialog", params: { accept: false } },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("handles JavaScript dialogs while the triggering CDP input command is still in flight", async () => {
|
||||
const contents = new FakeWebContents(25);
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
const captured = tab.captureDialogs?.(async () => {
|
||||
contents.debugger.blockedCommandNames.add("Input.dispatchMouseEvent");
|
||||
const input = tab.sendDebugCommand?.("Input.dispatchMouseEvent", { type: "mousePressed" });
|
||||
await flushMicrotasks();
|
||||
|
||||
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
|
||||
type: "alert",
|
||||
message: "Saved",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Page.enable", params: {} },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{ command: "Input.dispatchMouseEvent", params: { type: "mousePressed" } },
|
||||
{ command: "Page.handleJavaScriptDialog", params: { accept: true } },
|
||||
]);
|
||||
|
||||
contents.debugger.blockedCommandNames.clear();
|
||||
contents.debugger.finishNextCommand();
|
||||
await input;
|
||||
return "done";
|
||||
});
|
||||
|
||||
await expect(captured).resolves.toEqual({
|
||||
result: "done",
|
||||
dialogs: [
|
||||
{
|
||||
type: "alert",
|
||||
message: "Saved",
|
||||
action: "accepted",
|
||||
timestamp: expect.any(Number),
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(contents.debugger.commands.at(-1)).toEqual({
|
||||
command: "Runtime.evaluate",
|
||||
params: {
|
||||
expression: expect.stringContaining("delete window[stateKey]"),
|
||||
returnByValue: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("keeps the prompt shim installed until overlapping captures finish", async () => {
|
||||
const contents = new FakeWebContents(26);
|
||||
const tab = adaptWebContents(contents);
|
||||
const firstStarted = deferred<void>();
|
||||
const secondStarted = deferred<void>();
|
||||
const finishFirst = deferred<void>();
|
||||
const finishSecond = deferred<void>();
|
||||
|
||||
const first = tab.captureDialogs?.(async () => {
|
||||
firstStarted.resolve();
|
||||
await finishFirst.promise;
|
||||
return "first";
|
||||
});
|
||||
await firstStarted.promise;
|
||||
|
||||
const second = tab.captureDialogs?.(async () => {
|
||||
secondStarted.resolve();
|
||||
await finishSecond.promise;
|
||||
return "second";
|
||||
});
|
||||
await secondStarted.promise;
|
||||
|
||||
contents.debugger.promptDialogs.push(
|
||||
{
|
||||
type: "prompt",
|
||||
message: "First?",
|
||||
defaultValue: "one",
|
||||
action: "dismissed",
|
||||
timestamp: 1,
|
||||
},
|
||||
{
|
||||
type: "prompt",
|
||||
message: "Second?",
|
||||
defaultValue: "two",
|
||||
action: "dismissed",
|
||||
timestamp: 2,
|
||||
},
|
||||
);
|
||||
|
||||
finishFirst.resolve();
|
||||
await expect(first).resolves.toEqual({
|
||||
result: "first",
|
||||
dialogs: [
|
||||
{
|
||||
type: "prompt",
|
||||
message: "First?",
|
||||
defaultValue: "one",
|
||||
action: "dismissed",
|
||||
timestamp: 1,
|
||||
},
|
||||
{
|
||||
type: "prompt",
|
||||
message: "Second?",
|
||||
defaultValue: "two",
|
||||
action: "dismissed",
|
||||
timestamp: 2,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(
|
||||
contents.debugger.commands.some(
|
||||
(entry) =>
|
||||
entry.command === "Runtime.evaluate" &&
|
||||
typeof entry.params.expression === "string" &&
|
||||
entry.params.expression.includes("delete window[stateKey]"),
|
||||
),
|
||||
).toBe(false);
|
||||
|
||||
finishSecond.resolve();
|
||||
await expect(second).resolves.toEqual({
|
||||
result: "second",
|
||||
dialogs: [
|
||||
{
|
||||
type: "prompt",
|
||||
message: "First?",
|
||||
defaultValue: "one",
|
||||
action: "dismissed",
|
||||
timestamp: 1,
|
||||
},
|
||||
{
|
||||
type: "prompt",
|
||||
message: "Second?",
|
||||
defaultValue: "two",
|
||||
action: "dismissed",
|
||||
timestamp: 2,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(contents.debugger.commands.at(-1)).toEqual({
|
||||
command: "Runtime.evaluate",
|
||||
params: {
|
||||
expression: expect.stringContaining("delete window[stateKey]"),
|
||||
returnByValue: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("leaves JavaScript dialogs alone when no capture is active", async () => {
|
||||
const contents = new FakeWebContents(28);
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
await expect(tab.captureDialogs?.(async () => "done")).resolves.toEqual({
|
||||
result: "done",
|
||||
dialogs: [],
|
||||
});
|
||||
contents.debugger.emitMessage("Page.javascriptDialogOpening", {
|
||||
type: "confirm",
|
||||
message: "Unsaved changes?",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(contents.debugger.commands).not.toContainEqual({
|
||||
command: "Page.handleJavaScriptDialog",
|
||||
params: { accept: false },
|
||||
});
|
||||
});
|
||||
|
||||
test("treats prompt shim drain failures after navigation as no dialogs", async () => {
|
||||
const contents = new FakeWebContents(29);
|
||||
contents.debugger.failPromptDrain = true;
|
||||
const tab = adaptWebContents(contents);
|
||||
|
||||
await expect(tab.captureDialogs?.(async () => "navigated")).resolves.toEqual({
|
||||
result: "navigated",
|
||||
dialogs: [],
|
||||
});
|
||||
|
||||
expect(contents.debugger.commands).toEqual([
|
||||
{ command: "Page.enable", params: {} },
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: { expression: expect.any(String), returnByValue: true },
|
||||
},
|
||||
{
|
||||
command: "Runtime.evaluate",
|
||||
params: {
|
||||
expression: expect.stringContaining("delete window[stateKey]"),
|
||||
returnByValue: true,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("runs the command without dialog capture when CDP setup fails", async () => {
|
||||
const contents = new FakeWebContents(30);
|
||||
contents.debugger.failedCommandNames.add("Page.enable");
|
||||
const tab = adaptWebContents(contents);
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
||||
|
||||
await expect(tab.captureDialogs?.(async () => "done")).resolves.toEqual({
|
||||
result: "done",
|
||||
dialogs: [],
|
||||
});
|
||||
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
"[browser-automation] Dialog capture unavailable; running command without it",
|
||||
{ contentsId: 30, error: expect.any(Error) },
|
||||
);
|
||||
warn.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
async function flushMicrotasks(): Promise<void> {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T | PromiseLike<T>) => void;
|
||||
const promise = new Promise<T>((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
@@ -1,8 +1,20 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import { ipcMain } from "electron";
|
||||
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type {
|
||||
BrowserAutomationConsoleLogEntry,
|
||||
BrowserAutomationDialogEvent,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
|
||||
import { CdpSessionQueue } from "./cdp-session-queue.js";
|
||||
import {
|
||||
dialogAcceptValue,
|
||||
handledDialogEvent,
|
||||
MAX_DIALOGS_PER_COMMAND,
|
||||
promptShimDrainScript,
|
||||
promptShimInstallScript,
|
||||
promptShimRestoreScript,
|
||||
} from "./dialog-handling.js";
|
||||
import { executeAutomationCommand } from "./service.js";
|
||||
import {
|
||||
listRegisteredPaseoBrowserIds,
|
||||
@@ -14,6 +26,8 @@ import {
|
||||
|
||||
const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
|
||||
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
|
||||
const cdpQueuesByContentsId = new Map<number, CdpSessionQueue>();
|
||||
const dialogMonitorsByContentsId = new Map<number, DialogMonitor>();
|
||||
const observedContentsIds = new Set<number>();
|
||||
|
||||
interface IpcHandlerRegistry {
|
||||
@@ -24,6 +38,10 @@ interface WebContentsDebugger {
|
||||
isAttached(): boolean;
|
||||
attach(protocolVersion?: string): void;
|
||||
sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown>;
|
||||
on?(
|
||||
event: "message",
|
||||
listener: (event: unknown, method: string, params?: Record<string, unknown>) => void,
|
||||
): void;
|
||||
}
|
||||
|
||||
interface ConsoleMessageEmitter {
|
||||
@@ -60,6 +78,8 @@ interface BrowserAutomationWebContents extends ConsoleMessageEmitter {
|
||||
|
||||
export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents {
|
||||
observeConsoleMessages(contents);
|
||||
const cdpQueue = getCdpQueue(contents.id);
|
||||
const dialogMonitor = getDialogMonitor(contents, cdpQueue);
|
||||
return {
|
||||
id: contents.id,
|
||||
getURL: () => contents.getURL(),
|
||||
@@ -76,15 +96,27 @@ export function adaptWebContents(contents: BrowserAutomationWebContents): TabCon
|
||||
capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
|
||||
invalidate: () => contents.invalidate(),
|
||||
getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
|
||||
sendDebugCommand: async (command: string, params?: Record<string, unknown>) => {
|
||||
if (!contents.debugger.isAttached()) {
|
||||
contents.debugger.attach("1.3");
|
||||
}
|
||||
return contents.debugger.sendCommand(command, params ?? {});
|
||||
},
|
||||
captureDialogs: (task) => dialogMonitor.capture(task),
|
||||
sendDebugCommand: (command: string, params?: Record<string, unknown>) =>
|
||||
cdpQueue.run(async () => {
|
||||
if (!contents.debugger.isAttached()) {
|
||||
contents.debugger.attach("1.3");
|
||||
}
|
||||
return contents.debugger.sendCommand(command, params ?? {});
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function getCdpQueue(contentsId: number): CdpSessionQueue {
|
||||
const existing = cdpQueuesByContentsId.get(contentsId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const queue = new CdpSessionQueue();
|
||||
cdpQueuesByContentsId.set(contentsId, queue);
|
||||
return queue;
|
||||
}
|
||||
|
||||
function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
|
||||
if (observedContentsIds.has(contents.id)) {
|
||||
return;
|
||||
@@ -99,6 +131,192 @@ function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
|
||||
contents.once("destroyed", () => {
|
||||
observedContentsIds.delete(contents.id);
|
||||
consoleMessagesByContentsId.delete(contents.id);
|
||||
cdpQueuesByContentsId.delete(contents.id);
|
||||
dialogMonitorsByContentsId.delete(contents.id);
|
||||
});
|
||||
}
|
||||
|
||||
function getDialogMonitor(
|
||||
contents: BrowserAutomationWebContents,
|
||||
cdpQueue: CdpSessionQueue,
|
||||
): DialogMonitor {
|
||||
const existing = dialogMonitorsByContentsId.get(contents.id);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const monitor = new DialogMonitor(contents, cdpQueue);
|
||||
dialogMonitorsByContentsId.set(contents.id, monitor);
|
||||
return monitor;
|
||||
}
|
||||
|
||||
class DialogMonitor {
|
||||
private enabled = false;
|
||||
private listenerRegistered = false;
|
||||
private readonly activeCollectors: DialogCollector[] = [];
|
||||
|
||||
public constructor(
|
||||
private readonly contents: BrowserAutomationWebContents,
|
||||
private readonly cdpQueue: CdpSessionQueue,
|
||||
) {}
|
||||
|
||||
public async capture<T>(
|
||||
task: () => Promise<T>,
|
||||
): Promise<{ result: T; dialogs: BrowserAutomationDialogEvent[] }> {
|
||||
const collector: DialogCollector = { dialogs: [] };
|
||||
try {
|
||||
await this.enable();
|
||||
await this.installPromptShim();
|
||||
} catch (error) {
|
||||
console.warn("[browser-automation] Dialog capture unavailable; running command without it", {
|
||||
contentsId: this.contents.id,
|
||||
error,
|
||||
});
|
||||
return { result: await task(), dialogs: [] };
|
||||
}
|
||||
this.activeCollectors.push(collector);
|
||||
try {
|
||||
const result = await task();
|
||||
this.recordPromptShimDialogs(await this.drainPromptShim());
|
||||
return { result, dialogs: collector.dialogs };
|
||||
} finally {
|
||||
const index = this.activeCollectors.indexOf(collector);
|
||||
if (index >= 0) {
|
||||
this.activeCollectors.splice(index, 1);
|
||||
}
|
||||
if (this.activeCollectors.length === 0) {
|
||||
await this.restorePromptShim();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async enable(): Promise<void> {
|
||||
if (this.enabled) {
|
||||
return;
|
||||
}
|
||||
if (!this.contents.debugger.on) {
|
||||
return;
|
||||
}
|
||||
if (!this.listenerRegistered) {
|
||||
this.listenerRegistered = true;
|
||||
this.contents.debugger.on("message", (_event, method, params) => {
|
||||
if (method !== "Page.javascriptDialogOpening") {
|
||||
return;
|
||||
}
|
||||
if (this.activeCollectors.length === 0) {
|
||||
return;
|
||||
}
|
||||
void this.handleOpening(params ?? {});
|
||||
});
|
||||
}
|
||||
await this.sendDebugCommand("Page.enable");
|
||||
this.enabled = true;
|
||||
}
|
||||
|
||||
private async handleOpening(params: Record<string, unknown>): Promise<void> {
|
||||
const event = handledDialogEvent(params);
|
||||
for (const collector of this.activeCollectors) {
|
||||
this.recordDialogs(collector, [event]);
|
||||
}
|
||||
await this.sendDialogResponseCommand("Page.handleJavaScriptDialog", {
|
||||
accept: dialogAcceptValue(event.type),
|
||||
});
|
||||
}
|
||||
|
||||
private async installPromptShim(): Promise<void> {
|
||||
await this.sendDebugCommand("Runtime.evaluate", {
|
||||
expression: promptShimInstallScript(),
|
||||
returnByValue: true,
|
||||
});
|
||||
}
|
||||
|
||||
private async drainPromptShim(): Promise<BrowserAutomationDialogEvent[]> {
|
||||
try {
|
||||
const result = (await this.sendDebugCommand("Runtime.evaluate", {
|
||||
expression: promptShimDrainScript(),
|
||||
returnByValue: true,
|
||||
})) as { result?: { value?: unknown } };
|
||||
return parsePromptShimDialogs(result.result?.value);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private async restorePromptShim(): Promise<void> {
|
||||
try {
|
||||
await this.sendDebugCommand("Runtime.evaluate", {
|
||||
expression: promptShimRestoreScript(),
|
||||
returnByValue: true,
|
||||
});
|
||||
} catch {
|
||||
// Navigation can destroy the execution context before cleanup runs; the next page has no shim.
|
||||
}
|
||||
}
|
||||
|
||||
private recordDialogs(collector: DialogCollector, dialogs: BrowserAutomationDialogEvent[]): void {
|
||||
for (const dialog of dialogs) {
|
||||
if (collector.dialogs.length >= MAX_DIALOGS_PER_COMMAND) {
|
||||
return;
|
||||
}
|
||||
collector.dialogs.push(dialog);
|
||||
}
|
||||
}
|
||||
|
||||
private recordPromptShimDialogs(dialogs: BrowserAutomationDialogEvent[]): void {
|
||||
for (const collector of this.activeCollectors) {
|
||||
this.recordDialogs(collector, dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
private async sendDebugCommand(
|
||||
command: string,
|
||||
params?: Record<string, unknown>,
|
||||
): Promise<unknown> {
|
||||
return this.cdpQueue.run(async () => {
|
||||
if (!this.contents.debugger.isAttached()) {
|
||||
this.contents.debugger.attach("1.3");
|
||||
}
|
||||
return this.contents.debugger.sendCommand(command, params ?? {});
|
||||
});
|
||||
}
|
||||
|
||||
private async sendDialogResponseCommand(
|
||||
command: string,
|
||||
params?: Record<string, unknown>,
|
||||
): Promise<unknown> {
|
||||
// Dialogs can block the CDP command that opened them, so the unblocker must not wait behind
|
||||
// the per-tab command queue.
|
||||
if (!this.contents.debugger.isAttached()) {
|
||||
this.contents.debugger.attach("1.3");
|
||||
}
|
||||
return this.contents.debugger.sendCommand(command, params ?? {});
|
||||
}
|
||||
}
|
||||
|
||||
interface DialogCollector {
|
||||
dialogs: BrowserAutomationDialogEvent[];
|
||||
}
|
||||
|
||||
function parsePromptShimDialogs(value: unknown): BrowserAutomationDialogEvent[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.flatMap((entry): BrowserAutomationDialogEvent[] => {
|
||||
if (!entry || typeof entry !== "object") {
|
||||
return [];
|
||||
}
|
||||
const record = entry as Record<string, unknown>;
|
||||
if (record.type !== "prompt" || record.action !== "dismissed") {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
type: "prompt",
|
||||
message: typeof record.message === "string" ? record.message : "",
|
||||
...(typeof record.defaultValue === "string" ? { defaultValue: record.defaultValue } : {}),
|
||||
action: "dismissed",
|
||||
timestamp: typeof record.timestamp === "number" ? record.timestamp : Date.now(),
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -3,79 +3,162 @@ import { BrowserSnapshotEngine, type SnapshotPage } from "./snapshot-engine.js";
|
||||
|
||||
class SnapshotFixture implements SnapshotPage {
|
||||
public currentUrl = "https://example.com/form";
|
||||
public actionResult: unknown = true;
|
||||
public actionResult: unknown = { ok: true };
|
||||
public alreadyTruncated = false;
|
||||
public snapshotNodes: unknown[] = [
|
||||
{
|
||||
kind: "role",
|
||||
role: "heading",
|
||||
name: "Settings",
|
||||
tagName: "h1",
|
||||
attributes: ["level=1"],
|
||||
children: [],
|
||||
},
|
||||
{ kind: "text", text: "Connected as Maya" },
|
||||
{
|
||||
kind: "role",
|
||||
role: "button",
|
||||
name: "Save changes",
|
||||
tagName: "button",
|
||||
attributes: [],
|
||||
ref: "@e1",
|
||||
fingerprint: {
|
||||
role: "button",
|
||||
name: "Save changes",
|
||||
tagName: "button",
|
||||
type: "",
|
||||
ariaLabel: "",
|
||||
},
|
||||
children: [],
|
||||
},
|
||||
];
|
||||
|
||||
public getURL(): string {
|
||||
return this.currentUrl;
|
||||
}
|
||||
|
||||
public async executeJavaScript(code: string): Promise<unknown> {
|
||||
if (code.includes("CANDIDATE_SELECTOR")) {
|
||||
return JSON.stringify([
|
||||
{
|
||||
role: "textbox",
|
||||
tagName: "input",
|
||||
text: "Name",
|
||||
selector: "#name",
|
||||
attributes: { id: "name", type: "text" },
|
||||
if (code.includes("__PASEO_ARIA_SNAPSHOT__")) {
|
||||
return JSON.stringify({
|
||||
marker: "__PASEO_ARIA_SNAPSHOT__",
|
||||
root: {
|
||||
kind: "role",
|
||||
role: "document",
|
||||
name: "Fixture",
|
||||
tagName: "document",
|
||||
attributes: [],
|
||||
children: this.snapshotNodes,
|
||||
},
|
||||
{
|
||||
role: "button",
|
||||
tagName: "button",
|
||||
text: "Drop",
|
||||
selector: "#drop",
|
||||
attributes: { id: "drop" },
|
||||
},
|
||||
]);
|
||||
refs: [
|
||||
{
|
||||
ref: "@e1",
|
||||
fingerprint: {
|
||||
role: "button",
|
||||
name: "Save changes",
|
||||
tagName: "button",
|
||||
type: "",
|
||||
ariaLabel: "",
|
||||
},
|
||||
},
|
||||
],
|
||||
truncated: this.alreadyTruncated,
|
||||
stats: { nodeCount: 4, refCount: 1, textLength: 0, iframeCount: 0, maxDepth: 1 },
|
||||
});
|
||||
}
|
||||
return this.actionResult;
|
||||
}
|
||||
}
|
||||
|
||||
describe("BrowserSnapshotEngine", () => {
|
||||
it("treats a false result from a ref action script as a stale ref", async () => {
|
||||
it("renders a hierarchical ARIA YAML snapshot with static text and actionable refs", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
page.actionResult = false;
|
||||
|
||||
await expect(engine.click({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: "stale_ref",
|
||||
});
|
||||
await expect(
|
||||
engine.select({ browserId: "browser-1", page, ref: "@e1", value: "us" }),
|
||||
).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: "stale_ref",
|
||||
await expect(engine.snapshot({ browserId: "browser-1", page })).resolves.toEqual({
|
||||
format: "aria-yaml",
|
||||
snapshot: [
|
||||
'- document "Fixture"',
|
||||
' - heading "Settings" [level=1]',
|
||||
' - text: "Connected as Maya"',
|
||||
' - button "Save changes" [ref=@e1]',
|
||||
].join("\n"),
|
||||
truncated: false,
|
||||
stats: { nodeCount: 4, refCount: 1, textLength: 119, iframeCount: 0, maxDepth: 1 },
|
||||
});
|
||||
});
|
||||
|
||||
it("treats a false result from optional ref text/key actions as a stale ref", async () => {
|
||||
it("builds a runtime ref expression with the snapshot fingerprint", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
page.actionResult = false;
|
||||
page.currentUrl = "https://example.com/form?panel=advanced";
|
||||
|
||||
await expect(
|
||||
engine.typeText({ browserId: "browser-1", page, ref: "@e1", text: "Ada" }),
|
||||
).resolves.toEqual({ ok: false, reason: "stale_ref" });
|
||||
await expect(
|
||||
engine.keypress({ browserId: "browser-1", page, ref: "@e1", key: "Enter" }),
|
||||
).resolves.toEqual({ ok: false, reason: "stale_ref" });
|
||||
expect(engine.runtimeElementExpression({ browserId: "browser-1", ref: "@e1" })).toContain(
|
||||
'"name":"Save changes"',
|
||||
);
|
||||
});
|
||||
|
||||
it("treats a false result from drag as a stale ref", async () => {
|
||||
it("treats missing host-side ref metadata as a stale ref", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
page.actionResult = false;
|
||||
expect(engine.runtimeElementExpression({ browserId: "browser-1", ref: "@e2" })).toEqual({
|
||||
ok: false,
|
||||
reason: "missing_ref",
|
||||
});
|
||||
});
|
||||
|
||||
await expect(
|
||||
engine.drag({ browserId: "browser-1", page, sourceRef: "@e1", targetRef: "@e2" }),
|
||||
).resolves.toEqual({ ok: false, reason: "stale_ref" });
|
||||
it("marks rendered output truncation explicitly and deterministically", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
page.snapshotNodes = [
|
||||
{
|
||||
kind: "text",
|
||||
text: "A".repeat(81_000),
|
||||
},
|
||||
];
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
|
||||
const snapshot = await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
expect(snapshot.truncated).toBe(true);
|
||||
expect(snapshot.snapshot.endsWith('- text: "Snapshot truncated."')).toBe(true);
|
||||
expect(snapshot.stats.textLength).toBeLessThanOrEqual(80_000);
|
||||
});
|
||||
|
||||
it("keeps the last rendered node when a short snapshot was capped by node count", async () => {
|
||||
const page = new SnapshotFixture();
|
||||
page.alreadyTruncated = true;
|
||||
page.snapshotNodes = [
|
||||
{
|
||||
kind: "role",
|
||||
role: "button",
|
||||
name: "Final action",
|
||||
tagName: "button",
|
||||
attributes: [],
|
||||
ref: "@e1",
|
||||
fingerprint: {
|
||||
role: "button",
|
||||
name: "Final action",
|
||||
tagName: "button",
|
||||
type: "",
|
||||
ariaLabel: "",
|
||||
},
|
||||
children: [],
|
||||
},
|
||||
];
|
||||
const engine = new BrowserSnapshotEngine();
|
||||
|
||||
const snapshot = await engine.snapshot({ browserId: "browser-1", page });
|
||||
|
||||
expect(snapshot.truncated).toBe(true);
|
||||
expect(snapshot.snapshot).toBe(
|
||||
[
|
||||
'- document "Fixture"',
|
||||
' - button "Final action" [ref=@e1]',
|
||||
'- text: "Snapshot truncated."',
|
||||
].join("\n"),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,24 +1,60 @@
|
||||
import { ARIA_SNAPSHOT_SCRIPT, ARIA_SNAPSHOT_SCRIPT_MARKER } from "./aria-snapshot-script.js";
|
||||
|
||||
export interface SnapshotPage {
|
||||
getURL(): string;
|
||||
executeJavaScript(code: string): Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface BrowserSnapshotElement extends RawSnapshotElement {
|
||||
ref: string;
|
||||
export interface BrowserAriaSnapshot {
|
||||
format: "aria-yaml";
|
||||
snapshot: string;
|
||||
truncated: boolean;
|
||||
stats: BrowserAriaSnapshotStats;
|
||||
}
|
||||
|
||||
interface RawSnapshotElement {
|
||||
export interface BrowserAriaSnapshotStats {
|
||||
nodeCount: number;
|
||||
refCount: number;
|
||||
textLength: number;
|
||||
iframeCount?: number;
|
||||
maxDepth?: number;
|
||||
}
|
||||
|
||||
interface SnapshotNode {
|
||||
kind: "role" | "text" | "group";
|
||||
role?: string;
|
||||
name?: string;
|
||||
text?: string;
|
||||
tagName?: string;
|
||||
attributes?: string[];
|
||||
ref?: string;
|
||||
fingerprint?: BrowserRefFingerprint;
|
||||
children?: SnapshotNode[];
|
||||
}
|
||||
|
||||
interface BrowserRefFingerprint {
|
||||
role: string;
|
||||
name: string;
|
||||
tagName: string;
|
||||
text: string;
|
||||
selector: string;
|
||||
attributes: Record<string, string>;
|
||||
type: string;
|
||||
ariaLabel: string;
|
||||
}
|
||||
|
||||
interface RawAriaSnapshot {
|
||||
marker: string;
|
||||
root: SnapshotNode;
|
||||
refs: BrowserRefMetadata[];
|
||||
truncated: boolean;
|
||||
stats: BrowserAriaSnapshotStats;
|
||||
}
|
||||
|
||||
interface BrowserRefMetadata {
|
||||
ref: string;
|
||||
fingerprint: BrowserRefFingerprint;
|
||||
}
|
||||
|
||||
interface BrowserRefState {
|
||||
nextRefNumber: number;
|
||||
url: string;
|
||||
refs: Map<string, RawSnapshotElement>;
|
||||
refs: Map<string, BrowserRefMetadata>;
|
||||
}
|
||||
|
||||
export type BrowserRefActionResult =
|
||||
@@ -26,45 +62,31 @@ export type BrowserRefActionResult =
|
||||
| { ok: false; reason: "stale_ref" | "missing_ref" };
|
||||
type BrowserRefFailure = Extract<BrowserRefActionResult, { ok: false }>;
|
||||
|
||||
type BrowserRefResolveResult = { ok: true; element: RawSnapshotElement } | BrowserRefFailure;
|
||||
type BrowserRefResolveResult = { ok: true; metadata: BrowserRefMetadata } | BrowserRefFailure;
|
||||
|
||||
const TRUNCATION_MARKER = '- text: "Snapshot truncated."';
|
||||
const MAX_RENDERED_TEXT_LENGTH = 80_000;
|
||||
|
||||
export class BrowserSnapshotEngine {
|
||||
private readonly statesByBrowserId = new Map<string, BrowserRefState>();
|
||||
|
||||
async snapshot(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
}): Promise<BrowserSnapshotElement[]> {
|
||||
const rawElements = parseRawSnapshotElements(
|
||||
await input.page.executeJavaScript(SNAPSHOT_SCRIPT),
|
||||
);
|
||||
const state = {
|
||||
nextRefNumber: 1,
|
||||
url: input.page.getURL(),
|
||||
refs: new Map<string, RawSnapshotElement>(),
|
||||
};
|
||||
const elements = rawElements.map((element) => {
|
||||
const ref = `@e${state.nextRefNumber++}`;
|
||||
state.refs.set(ref, element);
|
||||
return {
|
||||
ref,
|
||||
role: element.role,
|
||||
tagName: element.tagName,
|
||||
text: element.text,
|
||||
selector: element.selector,
|
||||
attributes: element.attributes,
|
||||
};
|
||||
async snapshot(input: { browserId: string; page: SnapshotPage }): Promise<BrowserAriaSnapshot> {
|
||||
const rawSnapshot = parseAriaSnapshot(await input.page.executeJavaScript(ARIA_SNAPSHOT_SCRIPT));
|
||||
const rendered = renderSnapshot(rawSnapshot.root);
|
||||
const capped = capRenderedSnapshot(rendered, rawSnapshot.truncated);
|
||||
this.statesByBrowserId.set(input.browserId, {
|
||||
refs: new Map(rawSnapshot.refs.map((ref) => [ref.ref, ref])),
|
||||
});
|
||||
this.statesByBrowserId.set(input.browserId, state);
|
||||
return elements;
|
||||
}
|
||||
|
||||
async click(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildClickScript(selector));
|
||||
return {
|
||||
format: "aria-yaml",
|
||||
snapshot: capped.snapshot,
|
||||
truncated: capped.truncated,
|
||||
stats: {
|
||||
...rawSnapshot.stats,
|
||||
refCount: rawSnapshot.refs.length,
|
||||
textLength: capped.snapshot.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async fill(input: {
|
||||
@@ -73,39 +95,7 @@ export class BrowserSnapshotEngine {
|
||||
ref: string;
|
||||
value: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildFillScript(selector, input.value));
|
||||
}
|
||||
|
||||
async typeText(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
text: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const selector = this.resolveOptionalRef(input);
|
||||
if (!selector.ok) {
|
||||
return selector;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildTypeScript(selector.selector, input.text),
|
||||
);
|
||||
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
}
|
||||
|
||||
async keypress(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
key: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const selector = this.resolveOptionalRef(input);
|
||||
if (!selector.ok) {
|
||||
return selector;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildKeypressScript(selector.selector, input.key),
|
||||
);
|
||||
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
return this.runRefScript(input, (ref) => buildFillScript(ref, input.value));
|
||||
}
|
||||
|
||||
async select(input: {
|
||||
@@ -114,123 +104,214 @@ export class BrowserSnapshotEngine {
|
||||
ref: string;
|
||||
value: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildSelectScript(selector, input.value));
|
||||
}
|
||||
|
||||
async hover(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
return this.runRefScript(input, (selector) => buildHoverScript(selector));
|
||||
}
|
||||
|
||||
async drag(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
sourceRef: string;
|
||||
targetRef: string;
|
||||
}): Promise<BrowserRefActionResult> {
|
||||
const source = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.sourceRef,
|
||||
});
|
||||
if (!source.ok) {
|
||||
return source;
|
||||
}
|
||||
const target = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.targetRef,
|
||||
});
|
||||
if (!target.ok) {
|
||||
return target;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(
|
||||
buildDragScript(source.element.selector, target.element.selector),
|
||||
);
|
||||
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
return this.runRefScript(input, (ref) => buildSelectScript(ref, input.value));
|
||||
}
|
||||
|
||||
clearBrowser(browserId: string): void {
|
||||
this.statesByBrowserId.delete(browserId);
|
||||
}
|
||||
|
||||
selectorForRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): { ok: true; selector: string } | BrowserRefFailure {
|
||||
runtimeElementExpression(input: { browserId: string; ref: string }): string | BrowserRefFailure {
|
||||
const resolved = this.resolveRef(input);
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
return { ok: true, selector: resolved.element.selector };
|
||||
return buildRuntimeElementExpression(resolved.metadata);
|
||||
}
|
||||
|
||||
private async runRefScript(
|
||||
input: { browserId: string; page: SnapshotPage; ref: string },
|
||||
buildScript: (selector: string) => string,
|
||||
buildScript: (ref: BrowserRefMetadata) => string,
|
||||
): Promise<BrowserRefActionResult> {
|
||||
const resolved = this.resolveRef(input);
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
const result = await input.page.executeJavaScript(buildScript(resolved.element.selector));
|
||||
return result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
|
||||
const result = await input.page.executeJavaScript(buildScript(resolved.metadata));
|
||||
return readActionResult(result, true);
|
||||
}
|
||||
|
||||
private resolveRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref: string;
|
||||
}): BrowserRefResolveResult {
|
||||
private resolveRef(input: { browserId: string; ref: string }): BrowserRefResolveResult {
|
||||
const state = this.statesByBrowserId.get(input.browserId);
|
||||
if (!state || state.url !== input.page.getURL()) {
|
||||
if (!state) {
|
||||
return { ok: false, reason: "stale_ref" };
|
||||
}
|
||||
const element = state.refs.get(input.ref);
|
||||
if (!element) {
|
||||
const metadata = state.refs.get(input.ref);
|
||||
if (!metadata) {
|
||||
return { ok: false, reason: "missing_ref" };
|
||||
}
|
||||
return { ok: true, element };
|
||||
}
|
||||
|
||||
private resolveOptionalRef(input: {
|
||||
browserId: string;
|
||||
page: SnapshotPage;
|
||||
ref?: string;
|
||||
}): { ok: true; selector: string | undefined } | BrowserRefFailure {
|
||||
if (!input.ref) {
|
||||
return { ok: true, selector: undefined };
|
||||
}
|
||||
const resolved = this.resolveRef({
|
||||
browserId: input.browserId,
|
||||
page: input.page,
|
||||
ref: input.ref,
|
||||
});
|
||||
if (!resolved.ok) {
|
||||
return resolved;
|
||||
}
|
||||
return { ok: true, selector: resolved.element.selector };
|
||||
return { ok: true, metadata };
|
||||
}
|
||||
}
|
||||
|
||||
function buildClickScript(selector: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.click();
|
||||
return true;
|
||||
})()`;
|
||||
function readActionResult(value: unknown, staleWhenFalse: boolean): BrowserRefActionResult {
|
||||
if (value === false && staleWhenFalse) {
|
||||
return { ok: false, reason: "stale_ref" };
|
||||
}
|
||||
if (isActionFailure(value)) {
|
||||
return { ok: false, reason: value.reason };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
function buildFillScript(selector: string, value: string): string {
|
||||
function isActionFailure(value: unknown): value is BrowserRefFailure {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
const reason = (value as Record<string, unknown>).reason;
|
||||
return reason === "stale_ref" || reason === "missing_ref";
|
||||
}
|
||||
|
||||
function parseAriaSnapshot(value: unknown): RawAriaSnapshot {
|
||||
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
||||
if (!parsed || typeof parsed !== "object") {
|
||||
return emptySnapshot();
|
||||
}
|
||||
const record = parsed as Record<string, unknown>;
|
||||
if (record.marker !== ARIA_SNAPSHOT_SCRIPT_MARKER) {
|
||||
return emptySnapshot();
|
||||
}
|
||||
const root = parseSnapshotNode(record.root);
|
||||
return {
|
||||
marker: ARIA_SNAPSHOT_SCRIPT_MARKER,
|
||||
root: root ?? emptySnapshot().root,
|
||||
refs: parseRefs(record.refs),
|
||||
truncated: record.truncated === true,
|
||||
stats: parseStats(record.stats),
|
||||
};
|
||||
}
|
||||
|
||||
function emptySnapshot(): RawAriaSnapshot {
|
||||
return {
|
||||
marker: ARIA_SNAPSHOT_SCRIPT_MARKER,
|
||||
root: { kind: "role", role: "document", name: "", tagName: "document", children: [] },
|
||||
refs: [],
|
||||
truncated: false,
|
||||
stats: { nodeCount: 0, refCount: 0, textLength: 0, iframeCount: 0, maxDepth: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
function parseSnapshotNode(value: unknown): SnapshotNode | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const kind = record.kind;
|
||||
if (kind !== "role" && kind !== "text" && kind !== "group") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
kind,
|
||||
...(readString(record.role) ? { role: readString(record.role) ?? undefined } : {}),
|
||||
...(readString(record.name) ? { name: readString(record.name) ?? undefined } : {}),
|
||||
...(readString(record.text) ? { text: readString(record.text) ?? undefined } : {}),
|
||||
...(readString(record.tagName) ? { tagName: readString(record.tagName) ?? undefined } : {}),
|
||||
...(readString(record.ref) ? { ref: readString(record.ref) ?? undefined } : {}),
|
||||
...(parseFingerprint(record.fingerprint)
|
||||
? { fingerprint: parseFingerprint(record.fingerprint) ?? undefined }
|
||||
: {}),
|
||||
attributes: readStringArray(record.attributes),
|
||||
children: Array.isArray(record.children)
|
||||
? record.children.flatMap((child): SnapshotNode[] => {
|
||||
const parsed = parseSnapshotNode(child);
|
||||
return parsed ? [parsed] : [];
|
||||
})
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
function parseRefs(value: unknown): BrowserRefMetadata[] {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return value.flatMap((item): BrowserRefMetadata[] => {
|
||||
if (!item || typeof item !== "object") {
|
||||
return [];
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const ref = readString(record.ref);
|
||||
const fingerprint = parseFingerprint(record.fingerprint);
|
||||
return ref && fingerprint ? [{ ref, fingerprint }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
function parseFingerprint(value: unknown): BrowserRefFingerprint | null {
|
||||
if (!value || typeof value !== "object") {
|
||||
return null;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const role = readString(record.role);
|
||||
const name = readString(record.name);
|
||||
const tagName = readString(record.tagName);
|
||||
const type = readString(record.type);
|
||||
const ariaLabel = readString(record.ariaLabel);
|
||||
if (role === null || name === null || tagName === null || type === null || ariaLabel === null) {
|
||||
return null;
|
||||
}
|
||||
return { role, name, tagName, type, ariaLabel };
|
||||
}
|
||||
|
||||
function parseStats(value: unknown): BrowserAriaSnapshotStats {
|
||||
if (!value || typeof value !== "object") {
|
||||
return { nodeCount: 0, refCount: 0, textLength: 0 };
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return {
|
||||
nodeCount: readNumber(record.nodeCount) ?? 0,
|
||||
refCount: readNumber(record.refCount) ?? 0,
|
||||
textLength: readNumber(record.textLength) ?? 0,
|
||||
...(readNumber(record.iframeCount) !== null
|
||||
? { iframeCount: readNumber(record.iframeCount) ?? undefined }
|
||||
: {}),
|
||||
...(readNumber(record.maxDepth) !== null
|
||||
? { maxDepth: readNumber(record.maxDepth) ?? undefined }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function renderSnapshot(root: SnapshotNode): string {
|
||||
const lines = renderNode(root, 0);
|
||||
return lines.length > 0 ? lines.join("\n") : "- document";
|
||||
}
|
||||
|
||||
function renderNode(node: SnapshotNode, depth: number): string[] {
|
||||
if (node.kind === "group") {
|
||||
return (node.children ?? []).flatMap((child) => renderNode(child, depth));
|
||||
}
|
||||
const indent = " ".repeat(depth);
|
||||
if (node.kind === "text") {
|
||||
return [`${indent}- text: ${JSON.stringify(node.text ?? "")}`];
|
||||
}
|
||||
const attrs = [...(node.attributes ?? [])];
|
||||
if (node.ref) {
|
||||
attrs.push(`ref=${node.ref}`);
|
||||
}
|
||||
const suffix = attrs.length > 0 ? ` [${attrs.join(" ")}]` : "";
|
||||
const ownLine = `${indent}- ${node.role ?? "generic"}${node.name ? ` ${JSON.stringify(node.name)}` : ""}${suffix}`;
|
||||
const childLines = (node.children ?? []).flatMap((child) => renderNode(child, depth + 1));
|
||||
return [ownLine, ...childLines];
|
||||
}
|
||||
|
||||
function capRenderedSnapshot(
|
||||
snapshot: string,
|
||||
alreadyTruncated: boolean,
|
||||
): { snapshot: string; truncated: boolean } {
|
||||
if (snapshot.length <= MAX_RENDERED_TEXT_LENGTH && !alreadyTruncated) {
|
||||
return { snapshot, truncated: false };
|
||||
}
|
||||
if (snapshot.length + 1 + TRUNCATION_MARKER.length <= MAX_RENDERED_TEXT_LENGTH) {
|
||||
return { snapshot: `${snapshot}\n${TRUNCATION_MARKER}`, truncated: true };
|
||||
}
|
||||
const availableLength = MAX_RENDERED_TEXT_LENGTH - TRUNCATION_MARKER.length - 1;
|
||||
const capped = snapshot.slice(0, Math.max(0, availableLength)).replace(/\n[^\n]*$/, "");
|
||||
return { snapshot: `${capped}\n${TRUNCATION_MARKER}`, truncated: true };
|
||||
}
|
||||
|
||||
function buildFillScript(metadata: BrowserRefMetadata, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return resolved;
|
||||
const element = resolved.element;
|
||||
element.scrollIntoView({ block: 'center', inline: 'center' });
|
||||
element.focus();
|
||||
const nextValue = ${JSON.stringify(value)};
|
||||
@@ -238,51 +319,19 @@ function buildFillScript(selector: string, value: string): string {
|
||||
element.value = nextValue;
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
}
|
||||
element.textContent = nextValue;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: nextValue }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildTypeScript(selector: string | undefined, text: string): string {
|
||||
function buildSelectScript(metadata: BrowserRefMetadata, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
const text = ${JSON.stringify(text)};
|
||||
if ('value' in element) {
|
||||
element.value = String(element.value || '') + text;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
}
|
||||
element.textContent = String(element.textContent || '') + text;
|
||||
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'insertText', data: text }));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildKeypressScript(selector: string | undefined, key: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : "document.activeElement"};
|
||||
if (!element) return false;
|
||||
element.focus?.();
|
||||
const key = ${JSON.stringify(key)};
|
||||
const eventInit = { bubbles: true, cancelable: true, key };
|
||||
element.dispatchEvent(new KeyboardEvent('keydown', eventInit));
|
||||
element.dispatchEvent(new KeyboardEvent('keypress', eventInit));
|
||||
element.dispatchEvent(new KeyboardEvent('keyup', eventInit));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildSelectScript(selector: string, value: string): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return resolved;
|
||||
const element = resolved.element;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
element.focus?.();
|
||||
const nextValue = ${JSON.stringify(value)};
|
||||
@@ -290,205 +339,34 @@ function buildSelectScript(selector: string, value: string): string {
|
||||
element.value = nextValue;
|
||||
element.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
element.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return true;
|
||||
return { ok: true };
|
||||
}
|
||||
return false;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildHoverScript(selector: string): string {
|
||||
function buildRuntimeElementExpression(metadata: BrowserRefMetadata): string {
|
||||
return String.raw`(() => {
|
||||
const element = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!element) return false;
|
||||
element.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
const rect = element.getBoundingClientRect();
|
||||
const eventInit = {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: rect.left + rect.width / 2,
|
||||
clientY: rect.top + rect.height / 2,
|
||||
screenX: window.screenX + rect.left + rect.width / 2,
|
||||
screenY: window.screenY + rect.top + rect.height / 2,
|
||||
view: window,
|
||||
};
|
||||
element.dispatchEvent(new MouseEvent('mouseover', eventInit));
|
||||
element.dispatchEvent(new MouseEvent('mouseenter', eventInit));
|
||||
element.dispatchEvent(new MouseEvent('mousemove', eventInit));
|
||||
return true;
|
||||
const resolved = ${buildResolveExpression(metadata)};
|
||||
if (!resolved.ok) return null;
|
||||
return resolved.element;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function buildDragScript(sourceSelector: string, targetSelector: string): string {
|
||||
return String.raw`(() => {
|
||||
const source = document.querySelector(${JSON.stringify(sourceSelector)});
|
||||
const target = document.querySelector(${JSON.stringify(targetSelector)});
|
||||
if (!source || !target) return false;
|
||||
source.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
target.scrollIntoView?.({ block: 'center', inline: 'center' });
|
||||
const data = new DataTransfer();
|
||||
const sourceRect = source.getBoundingClientRect();
|
||||
const targetRect = target.getBoundingClientRect();
|
||||
function eventInit(rect) {
|
||||
return {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: rect.left + rect.width / 2,
|
||||
clientY: rect.top + rect.height / 2,
|
||||
screenX: window.screenX + rect.left + rect.width / 2,
|
||||
screenY: window.screenY + rect.top + rect.height / 2,
|
||||
dataTransfer: data,
|
||||
view: window,
|
||||
};
|
||||
}
|
||||
source.dispatchEvent(new MouseEvent('mousedown', eventInit(sourceRect)));
|
||||
source.dispatchEvent(new DragEvent('dragstart', eventInit(sourceRect)));
|
||||
target.dispatchEvent(new DragEvent('dragenter', eventInit(targetRect)));
|
||||
target.dispatchEvent(new DragEvent('dragover', eventInit(targetRect)));
|
||||
target.dispatchEvent(new DragEvent('drop', eventInit(targetRect)));
|
||||
source.dispatchEvent(new DragEvent('dragend', eventInit(sourceRect)));
|
||||
target.dispatchEvent(new MouseEvent('mouseup', eventInit(targetRect)));
|
||||
return true;
|
||||
})()`;
|
||||
}
|
||||
|
||||
function parseRawSnapshotElements(value: unknown): RawSnapshotElement[] {
|
||||
const parsed = typeof value === "string" ? JSON.parse(value) : value;
|
||||
if (!Array.isArray(parsed)) {
|
||||
return [];
|
||||
}
|
||||
return parsed.flatMap((item): RawSnapshotElement[] => {
|
||||
if (!item || typeof item !== "object") {
|
||||
return [];
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const selector = readString(record.selector);
|
||||
if (!selector) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
role: readString(record.role) || "generic",
|
||||
tagName: (readString(record.tagName) || "element").toLowerCase(),
|
||||
text: readString(record.text) || "",
|
||||
selector,
|
||||
attributes: readAttributes(record.attributes),
|
||||
},
|
||||
];
|
||||
});
|
||||
function buildResolveExpression(metadata: BrowserRefMetadata): string {
|
||||
return `window.__PASEO_BROWSER_AUTOMATION__?.resolve(${JSON.stringify(metadata.ref)}, ${JSON.stringify(metadata.fingerprint)}) ?? { ok: false, reason: 'stale_ref' }`;
|
||||
}
|
||||
|
||||
function readString(value: unknown): string | null {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
function readAttributes(value: unknown): Record<string, string> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return {};
|
||||
}
|
||||
const result: Record<string, string> = {};
|
||||
for (const [key, attributeValue] of Object.entries(value)) {
|
||||
if (typeof attributeValue === "string") {
|
||||
result[key] = attributeValue;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
function readStringArray(value: unknown): string[] {
|
||||
return Array.isArray(value)
|
||||
? value.filter((item): item is string => typeof item === "string")
|
||||
: [];
|
||||
}
|
||||
|
||||
const SNAPSHOT_SCRIPT = String.raw`(() => {
|
||||
const MAX_ELEMENTS = 200;
|
||||
const CANDIDATE_SELECTOR = [
|
||||
'a[href]',
|
||||
'button',
|
||||
'input',
|
||||
'textarea',
|
||||
'select',
|
||||
'summary',
|
||||
'[role]',
|
||||
'[tabindex]:not([tabindex="-1"])',
|
||||
'[contenteditable=""]',
|
||||
'[contenteditable="true"]'
|
||||
].join(',');
|
||||
|
||||
function cssEscape(value) {
|
||||
if (window.CSS && typeof window.CSS.escape === 'function') {
|
||||
return window.CSS.escape(value);
|
||||
}
|
||||
return String(value).replace(/[^a-zA-Z0-9_-]/g, '\\$&');
|
||||
}
|
||||
|
||||
function isVisible(element) {
|
||||
const style = window.getComputedStyle(element);
|
||||
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
|
||||
return false;
|
||||
}
|
||||
const rect = element.getBoundingClientRect();
|
||||
return rect.width > 0 && rect.height > 0;
|
||||
}
|
||||
|
||||
function roleFor(element) {
|
||||
const explicit = element.getAttribute('role');
|
||||
if (explicit) return explicit;
|
||||
const tag = element.tagName.toLowerCase();
|
||||
if (tag === 'a') return 'link';
|
||||
if (tag === 'button') return 'button';
|
||||
if (tag === 'select') return 'combobox';
|
||||
if (tag === 'textarea') return 'textbox';
|
||||
if (tag === 'summary') return 'button';
|
||||
if (tag === 'input') {
|
||||
const type = (element.getAttribute('type') || 'text').toLowerCase();
|
||||
if (type === 'checkbox') return 'checkbox';
|
||||
if (type === 'radio') return 'radio';
|
||||
if (type === 'button' || type === 'submit' || type === 'reset') return 'button';
|
||||
return 'textbox';
|
||||
}
|
||||
return 'generic';
|
||||
}
|
||||
|
||||
function textFor(element) {
|
||||
const tag = element.tagName.toLowerCase();
|
||||
const pieces = [
|
||||
element.getAttribute('aria-label'),
|
||||
element.getAttribute('alt'),
|
||||
element.getAttribute('title'),
|
||||
tag === 'input' ? element.getAttribute('placeholder') : null,
|
||||
tag === 'input' || tag === 'textarea' ? element.value : null,
|
||||
element.innerText,
|
||||
element.textContent
|
||||
];
|
||||
const text = pieces.find((piece) => typeof piece === 'string' && piece.trim().length > 0);
|
||||
return (text || '').replace(/\s+/g, ' ').trim().slice(0, 300);
|
||||
}
|
||||
|
||||
function selectorFor(element) {
|
||||
if (element.id) return '#' + cssEscape(element.id);
|
||||
const parts = [];
|
||||
let current = element;
|
||||
while (current && current.nodeType === Node.ELEMENT_NODE && current !== document.body) {
|
||||
const tag = current.tagName.toLowerCase();
|
||||
const parent = current.parentElement;
|
||||
if (!parent) break;
|
||||
const siblings = Array.from(parent.children).filter((sibling) => sibling.tagName === current.tagName);
|
||||
const index = siblings.indexOf(current) + 1;
|
||||
parts.unshift(siblings.length > 1 ? tag + ':nth-of-type(' + index + ')' : tag);
|
||||
current = parent;
|
||||
}
|
||||
return parts.length > 0 ? parts.join(' > ') : element.tagName.toLowerCase();
|
||||
}
|
||||
|
||||
return JSON.stringify(Array.from(document.querySelectorAll(CANDIDATE_SELECTOR))
|
||||
.filter(isVisible)
|
||||
.slice(0, MAX_ELEMENTS)
|
||||
.map((element) => ({
|
||||
role: roleFor(element),
|
||||
tagName: element.tagName.toLowerCase(),
|
||||
text: textFor(element),
|
||||
selector: selectorFor(element),
|
||||
attributes: {
|
||||
...(element.id ? { id: element.id } : {}),
|
||||
...(element.getAttribute('name') ? { name: element.getAttribute('name') } : {}),
|
||||
...(element.getAttribute('type') ? { type: element.getAttribute('type') } : {}),
|
||||
...(element.getAttribute('href') ? { href: element.getAttribute('href') } : {}),
|
||||
...(element.getAttribute('aria-label') ? { 'aria-label': element.getAttribute('aria-label') } : {})
|
||||
}
|
||||
})));
|
||||
})()`;
|
||||
function readNumber(value: unknown): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { dispatchTrustedKey } from "./trusted-input.js";
|
||||
|
||||
describe("trusted browser input", () => {
|
||||
test("Space dispatches a real space key event", async () => {
|
||||
const commands: Array<{ command: string; params?: Record<string, unknown> }> = [];
|
||||
|
||||
await dispatchTrustedKey(async (command, params) => {
|
||||
commands.push({ command, ...(params ? { params } : {}) });
|
||||
return {};
|
||||
}, "Space");
|
||||
|
||||
expect(commands).toEqual([
|
||||
{
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyDown",
|
||||
key: " ",
|
||||
code: "Space",
|
||||
windowsVirtualKeyCode: 32,
|
||||
nativeVirtualKeyCode: 32,
|
||||
text: " ",
|
||||
unmodifiedText: " ",
|
||||
},
|
||||
},
|
||||
{
|
||||
command: "Input.dispatchKeyEvent",
|
||||
params: {
|
||||
type: "keyUp",
|
||||
key: " ",
|
||||
code: "Space",
|
||||
windowsVirtualKeyCode: 32,
|
||||
nativeVirtualKeyCode: 32,
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,217 @@
|
||||
import type { ActionablePoint } from "./actionability.js";
|
||||
import type { CdpCommandSender } from "./cdp-session-queue.js";
|
||||
|
||||
export type MouseButton = "left" | "right" | "middle";
|
||||
export type InputModifier = "Alt" | "Control" | "Meta" | "Shift";
|
||||
|
||||
export interface ClickInputOptions {
|
||||
button?: MouseButton;
|
||||
doubleClick?: boolean;
|
||||
modifiers?: InputModifier[];
|
||||
}
|
||||
|
||||
const MODIFIER_MASKS: Record<InputModifier, number> = {
|
||||
Alt: 1,
|
||||
Control: 2,
|
||||
Meta: 4,
|
||||
Shift: 8,
|
||||
};
|
||||
|
||||
const SPECIAL_KEY_DEFINITIONS: Record<
|
||||
string,
|
||||
{ key: string; code: string; windowsVirtualKeyCode: number; text?: string }
|
||||
> = {
|
||||
Enter: { key: "Enter", code: "Enter", windowsVirtualKeyCode: 13, text: "\r" },
|
||||
Space: { key: " ", code: "Space", windowsVirtualKeyCode: 32, text: " " },
|
||||
Tab: { key: "Tab", code: "Tab", windowsVirtualKeyCode: 9, text: "\t" },
|
||||
Escape: { key: "Escape", code: "Escape", windowsVirtualKeyCode: 27 },
|
||||
Backspace: { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 },
|
||||
Delete: { key: "Delete", code: "Delete", windowsVirtualKeyCode: 46 },
|
||||
ArrowUp: { key: "ArrowUp", code: "ArrowUp", windowsVirtualKeyCode: 38 },
|
||||
ArrowDown: { key: "ArrowDown", code: "ArrowDown", windowsVirtualKeyCode: 40 },
|
||||
ArrowLeft: { key: "ArrowLeft", code: "ArrowLeft", windowsVirtualKeyCode: 37 },
|
||||
ArrowRight: { key: "ArrowRight", code: "ArrowRight", windowsVirtualKeyCode: 39 },
|
||||
Home: { key: "Home", code: "Home", windowsVirtualKeyCode: 36 },
|
||||
End: { key: "End", code: "End", windowsVirtualKeyCode: 35 },
|
||||
PageUp: { key: "PageUp", code: "PageUp", windowsVirtualKeyCode: 33 },
|
||||
PageDown: { key: "PageDown", code: "PageDown", windowsVirtualKeyCode: 34 },
|
||||
};
|
||||
|
||||
export async function dispatchTrustedClick(
|
||||
send: CdpCommandSender,
|
||||
point: ActionablePoint,
|
||||
options: ClickInputOptions = {},
|
||||
): Promise<void> {
|
||||
const button = options.button ?? "left";
|
||||
const modifiers = modifierMask(options.modifiers);
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button: "none",
|
||||
modifiers,
|
||||
});
|
||||
if (options.doubleClick) {
|
||||
await dispatchTrustedMouseClick(send, point, button, modifiers, 1);
|
||||
await dispatchTrustedMouseClick(send, point, button, modifiers, 2);
|
||||
return;
|
||||
}
|
||||
await dispatchTrustedMouseClick(send, point, button, modifiers, 1);
|
||||
}
|
||||
|
||||
async function dispatchTrustedMouseClick(
|
||||
send: CdpCommandSender,
|
||||
point: ActionablePoint,
|
||||
button: MouseButton,
|
||||
modifiers: number,
|
||||
clickCount: number,
|
||||
): Promise<void> {
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
buttons: mouseButtonMask(button),
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button,
|
||||
buttons: 0,
|
||||
clickCount,
|
||||
modifiers,
|
||||
});
|
||||
}
|
||||
|
||||
export async function dispatchTrustedHover(
|
||||
send: CdpCommandSender,
|
||||
point: ActionablePoint,
|
||||
): Promise<void> {
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
button: "none",
|
||||
});
|
||||
}
|
||||
|
||||
export async function dispatchTrustedDrag(
|
||||
send: CdpCommandSender,
|
||||
source: ActionablePoint,
|
||||
target: ActionablePoint,
|
||||
): Promise<void> {
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: source.x,
|
||||
y: source.y,
|
||||
button: "none",
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mousePressed",
|
||||
x: source.x,
|
||||
y: source.y,
|
||||
button: "left",
|
||||
buttons: 1,
|
||||
clickCount: 1,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: (source.x + target.x) / 2,
|
||||
y: (source.y + target.y) / 2,
|
||||
button: "left",
|
||||
buttons: 1,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: target.x,
|
||||
y: target.y,
|
||||
button: "left",
|
||||
buttons: 1,
|
||||
});
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseReleased",
|
||||
x: target.x,
|
||||
y: target.y,
|
||||
button: "left",
|
||||
buttons: 0,
|
||||
clickCount: 1,
|
||||
});
|
||||
}
|
||||
|
||||
export async function dispatchTrustedScroll(
|
||||
send: CdpCommandSender,
|
||||
point: ActionablePoint,
|
||||
deltaX: number,
|
||||
deltaY: number,
|
||||
): Promise<void> {
|
||||
await send("Input.dispatchMouseEvent", {
|
||||
type: "mouseWheel",
|
||||
x: point.x,
|
||||
y: point.y,
|
||||
deltaX,
|
||||
deltaY,
|
||||
});
|
||||
}
|
||||
|
||||
export async function dispatchTrustedText(send: CdpCommandSender, text: string): Promise<void> {
|
||||
if (text.length === 0) {
|
||||
return;
|
||||
}
|
||||
await send("Input.insertText", { text });
|
||||
}
|
||||
|
||||
export async function dispatchTrustedKey(send: CdpCommandSender, key: string): Promise<void> {
|
||||
const definition = keyDefinition(key);
|
||||
await send("Input.dispatchKeyEvent", {
|
||||
type: "keyDown",
|
||||
key: definition.key,
|
||||
code: definition.code,
|
||||
windowsVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
nativeVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
...(definition.text ? { text: definition.text, unmodifiedText: definition.text } : {}),
|
||||
});
|
||||
await send("Input.dispatchKeyEvent", {
|
||||
type: "keyUp",
|
||||
key: definition.key,
|
||||
code: definition.code,
|
||||
windowsVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
nativeVirtualKeyCode: definition.windowsVirtualKeyCode,
|
||||
});
|
||||
}
|
||||
|
||||
function keyDefinition(key: string): {
|
||||
key: string;
|
||||
code: string;
|
||||
windowsVirtualKeyCode: number;
|
||||
text?: string;
|
||||
} {
|
||||
const special = SPECIAL_KEY_DEFINITIONS[key];
|
||||
if (special) {
|
||||
return special;
|
||||
}
|
||||
const text = key.length === 1 ? key : "";
|
||||
const upper = text.toUpperCase();
|
||||
return {
|
||||
key,
|
||||
code: upper ? `Key${upper}` : key,
|
||||
windowsVirtualKeyCode: upper ? upper.charCodeAt(0) : 0,
|
||||
...(text ? { text, unmodifiedText: text } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function modifierMask(modifiers: InputModifier[] | undefined): number {
|
||||
return (modifiers ?? []).reduce((mask, modifier) => mask | MODIFIER_MASKS[modifier], 0);
|
||||
}
|
||||
|
||||
function mouseButtonMask(button: MouseButton): number {
|
||||
if (button === "right") {
|
||||
return 2;
|
||||
}
|
||||
if (button === "middle") {
|
||||
return 4;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
@@ -70,6 +70,10 @@ export function registerPaseoBrowserWorkspace(input: BrowserWorkspaceRegistratio
|
||||
browserRegistry.registerWorkspace(input);
|
||||
}
|
||||
|
||||
export function unregisterPaseoBrowser(browserId: string): void {
|
||||
browserRegistry.unregisterBrowser(browserId);
|
||||
}
|
||||
|
||||
export function getPaseoBrowserWorkspaceId(browserId: string): string | null {
|
||||
return browserRegistry.getWorkspaceId(browserId);
|
||||
}
|
||||
|
||||
@@ -51,6 +51,16 @@ export class PaseoBrowserWebviewRegistry {
|
||||
this.workspaceIdsByBrowserId.set(input.browserId, input.workspaceId);
|
||||
}
|
||||
|
||||
public unregisterBrowser(browserId: string): void {
|
||||
const webContentsId = this.webContentsIdsByBrowserId.get(browserId) ?? null;
|
||||
if (webContentsId !== null) {
|
||||
this.browserIdsByWebContentsId.delete(webContentsId);
|
||||
this.webContentsIdsByBrowserId.delete(browserId);
|
||||
}
|
||||
this.workspaceIdsByBrowserId.delete(browserId);
|
||||
this.deleteActiveBrowserReferences(browserId);
|
||||
}
|
||||
|
||||
public getWorkspaceId(browserId: string): string | null {
|
||||
return this.workspaceIdsByBrowserId.get(browserId) ?? null;
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ import {
|
||||
listRegisteredPaseoBrowserIds,
|
||||
readBrowserIdFromWebviewAttach,
|
||||
registerBrowserWebviewNavigationGuards,
|
||||
unregisterPaseoBrowser,
|
||||
registerPaseoBrowserWorkspace,
|
||||
registerPaseoBrowserWebContents,
|
||||
setWorkspaceActivePaseoBrowserId,
|
||||
@@ -328,6 +329,12 @@ ipcMain.handle("paseo:browser:register-workspace-browser", (_event, rawInput: un
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle("paseo:browser:unregister-workspace-browser", (_event, browserId: unknown) => {
|
||||
if (typeof browserId === "string" && browserId.trim().length > 0) {
|
||||
unregisterPaseoBrowser(browserId.trim());
|
||||
}
|
||||
});
|
||||
|
||||
ipcMain.handle("paseo:browser:set-workspace-active-browser", (_event, rawInput: unknown) => {
|
||||
const input = readActiveBrowserInput(rawInput);
|
||||
if (input) {
|
||||
|
||||
@@ -76,6 +76,8 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
browser: {
|
||||
registerWorkspaceBrowser: (input: { browserId: string; workspaceId: string }) =>
|
||||
ipcRenderer.invoke("paseo:browser:register-workspace-browser", input),
|
||||
unregisterWorkspaceBrowser: (browserId: string) =>
|
||||
ipcRenderer.invoke("paseo:browser:unregister-workspace-browser", browserId),
|
||||
setWorkspaceActiveBrowser: (input: { workspaceId: string; browserId: string | null }) =>
|
||||
ipcRenderer.invoke("paseo:browser:set-workspace-active-browser", input),
|
||||
openDevTools: (browserId: string) =>
|
||||
|
||||
@@ -15,7 +15,39 @@ const commandParseCases = [
|
||||
{
|
||||
name: "click",
|
||||
command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } },
|
||||
expected: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } },
|
||||
expected: {
|
||||
command: "click",
|
||||
args: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e1",
|
||||
button: "left",
|
||||
doubleClick: false,
|
||||
modifiers: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "click options",
|
||||
command: {
|
||||
command: "click",
|
||||
args: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e1",
|
||||
button: "right",
|
||||
doubleClick: true,
|
||||
modifiers: ["Meta", "Shift"],
|
||||
},
|
||||
},
|
||||
expected: {
|
||||
command: "click",
|
||||
args: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e1",
|
||||
button: "right",
|
||||
doubleClick: true,
|
||||
modifiers: ["Meta", "Shift"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fill",
|
||||
@@ -123,6 +155,72 @@ const commandParseCases = [
|
||||
command: { command: "logs", args: { browserId: BROWSER_ID } },
|
||||
expected: { command: "logs", args: { browserId: BROWSER_ID, maxEntries: 50 } },
|
||||
},
|
||||
{
|
||||
name: "evaluate",
|
||||
command: {
|
||||
command: "evaluate",
|
||||
args: { browserId: BROWSER_ID, function: "() => document.title" },
|
||||
},
|
||||
expected: {
|
||||
command: "evaluate",
|
||||
args: { browserId: BROWSER_ID, function: "() => document.title" },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "evaluate with ref",
|
||||
command: {
|
||||
command: "evaluate",
|
||||
args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" },
|
||||
},
|
||||
expected: {
|
||||
command: "evaluate",
|
||||
args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "scroll",
|
||||
command: {
|
||||
command: "scroll",
|
||||
args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 },
|
||||
},
|
||||
expected: {
|
||||
command: "scroll",
|
||||
args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "scroll with ref",
|
||||
command: {
|
||||
command: "scroll",
|
||||
args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 10, deltaY: -20 },
|
||||
},
|
||||
expected: {
|
||||
command: "scroll",
|
||||
args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 10, deltaY: -20 },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "resize",
|
||||
command: {
|
||||
command: "resize",
|
||||
args: { browserId: BROWSER_ID, width: 1024, height: 768 },
|
||||
},
|
||||
expected: {
|
||||
command: "resize",
|
||||
args: { browserId: BROWSER_ID, width: 1024, height: 768 },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "close_tab",
|
||||
command: {
|
||||
command: "close_tab",
|
||||
args: { browserId: BROWSER_ID },
|
||||
},
|
||||
expected: {
|
||||
command: "close_tab",
|
||||
args: { browserId: BROWSER_ID },
|
||||
},
|
||||
},
|
||||
] as const;
|
||||
|
||||
const resultParseCases = [
|
||||
@@ -134,16 +232,10 @@ const resultParseCases = [
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com/form",
|
||||
title: "Fixture",
|
||||
elements: [
|
||||
{
|
||||
ref: "@e1",
|
||||
role: "textbox",
|
||||
tagName: "input",
|
||||
text: "Name",
|
||||
selector: "#name",
|
||||
attributes: { id: "name", type: "text" },
|
||||
},
|
||||
],
|
||||
format: "aria-yaml",
|
||||
snapshot: '- document "Fixture"\n - textbox "Name" [ref=@e1]',
|
||||
truncated: false,
|
||||
stats: { nodeCount: 2, refCount: 1, textLength: 52, iframeCount: 0, maxDepth: 1 },
|
||||
},
|
||||
expected: {
|
||||
command: "snapshot",
|
||||
@@ -151,16 +243,10 @@ const resultParseCases = [
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com/form",
|
||||
title: "Fixture",
|
||||
elements: [
|
||||
{
|
||||
ref: "@e1",
|
||||
role: "textbox",
|
||||
tagName: "input",
|
||||
text: "Name",
|
||||
selector: "#name",
|
||||
attributes: { id: "name", type: "text" },
|
||||
},
|
||||
],
|
||||
format: "aria-yaml",
|
||||
snapshot: '- document "Fixture"\n - textbox "Name" [ref=@e1]',
|
||||
truncated: false,
|
||||
stats: { nodeCount: 2, refCount: 1, textLength: 52, iframeCount: 0, maxDepth: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -305,6 +391,68 @@ const resultParseCases = [
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "evaluate",
|
||||
result: {
|
||||
command: "evaluate",
|
||||
browserId: BROWSER_ID,
|
||||
resultJson: '{"title":"Fixture"}',
|
||||
truncated: false,
|
||||
},
|
||||
expected: {
|
||||
command: "evaluate",
|
||||
browserId: BROWSER_ID,
|
||||
resultJson: '{"title":"Fixture"}',
|
||||
truncated: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "scroll",
|
||||
result: {
|
||||
command: "scroll",
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e1",
|
||||
deltaX: 10,
|
||||
deltaY: 400,
|
||||
x: 40,
|
||||
y: 30,
|
||||
},
|
||||
expected: {
|
||||
command: "scroll",
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e1",
|
||||
deltaX: 10,
|
||||
deltaY: 400,
|
||||
x: 40,
|
||||
y: 30,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "resize",
|
||||
result: {
|
||||
command: "resize",
|
||||
browserId: BROWSER_ID,
|
||||
width: 1024,
|
||||
height: 768,
|
||||
},
|
||||
expected: {
|
||||
command: "resize",
|
||||
browserId: BROWSER_ID,
|
||||
width: 1024,
|
||||
height: 768,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "close_tab",
|
||||
result: {
|
||||
command: "close_tab",
|
||||
browserId: BROWSER_ID,
|
||||
},
|
||||
expected: {
|
||||
command: "close_tab",
|
||||
browserId: BROWSER_ID,
|
||||
},
|
||||
},
|
||||
] as const;
|
||||
|
||||
describe("browser automation execute RPC schemas", () => {
|
||||
@@ -526,7 +674,10 @@ describe("browser automation execute RPC schemas", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -557,6 +708,53 @@ describe("browser automation execute RPC schemas", () => {
|
||||
},
|
||||
);
|
||||
|
||||
test("success responses can report handled dialogs", () => {
|
||||
const parsed = BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-click",
|
||||
ok: true,
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
|
||||
dialogs: [
|
||||
{
|
||||
type: "alert",
|
||||
message: "Saved",
|
||||
action: "accepted",
|
||||
timestamp: 123,
|
||||
},
|
||||
{
|
||||
type: "prompt",
|
||||
message: "Name?",
|
||||
defaultValue: "Maya",
|
||||
action: "dismissed",
|
||||
timestamp: 124,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(parsed.payload).toEqual({
|
||||
requestId: "req-click",
|
||||
ok: true,
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
|
||||
dialogs: [
|
||||
{
|
||||
type: "alert",
|
||||
message: "Saved",
|
||||
action: "accepted",
|
||||
timestamp: 123,
|
||||
},
|
||||
{
|
||||
type: "prompt",
|
||||
message: "Name?",
|
||||
defaultValue: "Maya",
|
||||
action: "dismissed",
|
||||
timestamp: 124,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("error responses keep stable codes, messages, and retry defaults", () => {
|
||||
const parsed = BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
@@ -580,4 +778,44 @@ describe("browser automation execute RPC schemas", () => {
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("failure responses can report handled dialogs", () => {
|
||||
const parsed = BrowserAutomationExecuteResponseSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: "req-navigate",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_timeout",
|
||||
message: "Timed out waiting for browser URL: /next",
|
||||
},
|
||||
dialogs: [
|
||||
{
|
||||
type: "beforeunload",
|
||||
message: "Leave site?",
|
||||
action: "dismissed",
|
||||
timestamp: 200,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(parsed.payload).toEqual({
|
||||
requestId: "req-navigate",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_timeout",
|
||||
message: "Timed out waiting for browser URL: /next",
|
||||
retryable: false,
|
||||
},
|
||||
dialogs: [
|
||||
{
|
||||
type: "beforeunload",
|
||||
message: "Leave site?",
|
||||
action: "dismissed",
|
||||
timestamp: 200,
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -39,6 +39,10 @@ export const BROWSER_AUTOMATION_COMMAND_NAMES = [
|
||||
"hover",
|
||||
"drag",
|
||||
"logs",
|
||||
"evaluate",
|
||||
"scroll",
|
||||
"resize",
|
||||
"close_tab",
|
||||
] as const;
|
||||
|
||||
export const BrowserAutomationCommandNameSchema = z.enum(BROWSER_AUTOMATION_COMMAND_NAMES);
|
||||
@@ -55,6 +59,8 @@ const BrowserAutomationTabTargetSchema = z
|
||||
.strict();
|
||||
|
||||
const BrowserAutomationRefSchema = z.string().regex(/^@e\d+$/);
|
||||
const BrowserAutomationMouseButtonSchema = z.enum(["left", "right", "middle"]);
|
||||
const BrowserAutomationInputModifierSchema = z.enum(["Alt", "Control", "Meta", "Shift"]);
|
||||
const BrowserAutomationHttpUrlSchema = z
|
||||
.string()
|
||||
.url()
|
||||
@@ -87,6 +93,9 @@ export const BrowserAutomationClickCommandSchema = z.object({
|
||||
command: z.literal("click"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema,
|
||||
button: BrowserAutomationMouseButtonSchema.default("left"),
|
||||
doubleClick: z.boolean().default(false),
|
||||
modifiers: z.array(BrowserAutomationInputModifierSchema).default([]),
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -192,6 +201,36 @@ export const BrowserAutomationLogsCommandSchema = z.object({
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationEvaluateCommandSchema = z.object({
|
||||
command: z.literal("evaluate"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
function: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationScrollCommandSchema = z.object({
|
||||
command: z.literal("scroll"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
deltaX: z.number(),
|
||||
deltaY: z.number(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationResizeCommandSchema = z.object({
|
||||
command: z.literal("resize"),
|
||||
args: BrowserAutomationTabTargetSchema.extend({
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const BrowserAutomationCloseTabCommandSchema = z.object({
|
||||
command: z.literal("close_tab"),
|
||||
args: BrowserAutomationTabTargetSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationListTabsCommandSchema,
|
||||
BrowserAutomationNewTabCommandSchema,
|
||||
@@ -211,6 +250,10 @@ export const BrowserAutomationCommandSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationHoverCommandSchema,
|
||||
BrowserAutomationDragCommandSchema,
|
||||
BrowserAutomationLogsCommandSchema,
|
||||
BrowserAutomationEvaluateCommandSchema,
|
||||
BrowserAutomationScrollCommandSchema,
|
||||
BrowserAutomationResizeCommandSchema,
|
||||
BrowserAutomationCloseTabCommandSchema,
|
||||
]);
|
||||
|
||||
export const BrowserAutomationTabInfoSchema = z.object({
|
||||
@@ -236,14 +279,15 @@ export const BrowserAutomationNewTabResultSchema = z.object({
|
||||
url: z.string().min(1),
|
||||
});
|
||||
|
||||
export const BrowserAutomationSnapshotElementSchema = z.object({
|
||||
ref: z.string().regex(/^@e\d+$/),
|
||||
role: z.string(),
|
||||
tagName: z.string(),
|
||||
text: z.string(),
|
||||
selector: z.string(),
|
||||
attributes: z.record(z.string(), z.string()).default({}),
|
||||
});
|
||||
export const BrowserAutomationSnapshotStatsSchema = z
|
||||
.object({
|
||||
nodeCount: z.number().int().nonnegative(),
|
||||
refCount: z.number().int().nonnegative(),
|
||||
textLength: z.number().int().nonnegative(),
|
||||
iframeCount: z.number().int().nonnegative().optional(),
|
||||
maxDepth: z.number().int().nonnegative().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const BrowserAutomationSnapshotResultSchema = z.object({
|
||||
command: z.literal("snapshot"),
|
||||
@@ -251,13 +295,18 @@ export const BrowserAutomationSnapshotResultSchema = z.object({
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
url: z.string(),
|
||||
title: z.string(),
|
||||
elements: z.array(BrowserAutomationSnapshotElementSchema),
|
||||
format: z.literal("aria-yaml"),
|
||||
snapshot: z.string(),
|
||||
truncated: z.boolean(),
|
||||
stats: BrowserAutomationSnapshotStatsSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationClickResultSchema = z.object({
|
||||
command: z.literal("click"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
ref: BrowserAutomationRefSchema,
|
||||
x: z.number().optional(),
|
||||
y: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationFillResultSchema = z.object({
|
||||
@@ -276,6 +325,8 @@ export const BrowserAutomationTypeResultSchema = z.object({
|
||||
command: z.literal("type"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
x: z.number().optional(),
|
||||
y: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationKeypressResultSchema = z.object({
|
||||
@@ -283,6 +334,8 @@ export const BrowserAutomationKeypressResultSchema = z.object({
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
key: z.string().min(1),
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
x: z.number().optional(),
|
||||
y: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationNavigateResultSchema = z.object({
|
||||
@@ -333,6 +386,8 @@ export const BrowserAutomationHoverResultSchema = z.object({
|
||||
command: z.literal("hover"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
ref: BrowserAutomationRefSchema,
|
||||
x: z.number().optional(),
|
||||
y: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationDragResultSchema = z.object({
|
||||
@@ -340,6 +395,10 @@ export const BrowserAutomationDragResultSchema = z.object({
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
sourceRef: BrowserAutomationRefSchema,
|
||||
targetRef: BrowserAutomationRefSchema,
|
||||
sourceX: z.number().optional(),
|
||||
sourceY: z.number().optional(),
|
||||
targetX: z.number().optional(),
|
||||
targetY: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationConsoleLogEntrySchema = z.object({
|
||||
@@ -367,6 +426,35 @@ export const BrowserAutomationLogsResultSchema = z.object({
|
||||
network: z.array(BrowserAutomationNetworkLogEntrySchema),
|
||||
});
|
||||
|
||||
export const BrowserAutomationEvaluateResultSchema = z.object({
|
||||
command: z.literal("evaluate"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
resultJson: z.string(),
|
||||
truncated: z.boolean(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationScrollResultSchema = z.object({
|
||||
command: z.literal("scroll"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
ref: BrowserAutomationRefSchema.optional(),
|
||||
deltaX: z.number(),
|
||||
deltaY: z.number(),
|
||||
x: z.number().optional(),
|
||||
y: z.number().optional(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationResizeResultSchema = z.object({
|
||||
command: z.literal("resize"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationCloseTabResultSchema = z.object({
|
||||
command: z.literal("close_tab"),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
});
|
||||
|
||||
export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationListTabsResultSchema,
|
||||
BrowserAutomationNewTabResultSchema,
|
||||
@@ -386,6 +474,10 @@ export const BrowserAutomationResultSchema = z.discriminatedUnion("command", [
|
||||
BrowserAutomationHoverResultSchema,
|
||||
BrowserAutomationDragResultSchema,
|
||||
BrowserAutomationLogsResultSchema,
|
||||
BrowserAutomationEvaluateResultSchema,
|
||||
BrowserAutomationScrollResultSchema,
|
||||
BrowserAutomationResizeResultSchema,
|
||||
BrowserAutomationCloseTabResultSchema,
|
||||
]);
|
||||
|
||||
export const BrowserAutomationErrorSchema = z.object({
|
||||
@@ -394,6 +486,15 @@ export const BrowserAutomationErrorSchema = z.object({
|
||||
retryable: z.boolean().default(false),
|
||||
});
|
||||
|
||||
export const BrowserAutomationDialogEventSchema = z.object({
|
||||
type: z.enum(["alert", "confirm", "prompt", "beforeunload"]),
|
||||
message: z.string(),
|
||||
defaultValue: z.string().optional(),
|
||||
action: z.enum(["accepted", "dismissed"]),
|
||||
promptText: z.string().optional(),
|
||||
timestamp: z.number(),
|
||||
});
|
||||
|
||||
export const BrowserAutomationExecuteRequestSchema = z
|
||||
.object({
|
||||
type: z.literal("browser.automation.execute.request"),
|
||||
@@ -412,11 +513,13 @@ export const BrowserAutomationExecuteResponseSchema = z.object({
|
||||
requestId: z.string().min(1),
|
||||
ok: z.literal(true),
|
||||
result: BrowserAutomationResultSchema,
|
||||
dialogs: z.array(BrowserAutomationDialogEventSchema).optional(),
|
||||
}),
|
||||
z.object({
|
||||
requestId: z.string().min(1),
|
||||
ok: z.literal(false),
|
||||
error: BrowserAutomationErrorSchema,
|
||||
dialogs: z.array(BrowserAutomationDialogEventSchema).optional(),
|
||||
}),
|
||||
]),
|
||||
});
|
||||
@@ -431,6 +534,7 @@ export type BrowserAutomationConsoleLogEntry = z.infer<
|
||||
export type BrowserAutomationNetworkLogEntry = z.infer<
|
||||
typeof BrowserAutomationNetworkLogEntrySchema
|
||||
>;
|
||||
export type BrowserAutomationDialogEvent = z.infer<typeof BrowserAutomationDialogEventSchema>;
|
||||
export type BrowserAutomationExecuteRequest = z.infer<typeof BrowserAutomationExecuteRequestSchema>;
|
||||
export type BrowserAutomationExecuteResponse = z.infer<
|
||||
typeof BrowserAutomationExecuteResponseSchema
|
||||
|
||||
@@ -100,6 +100,28 @@ describe("browser automation protocol integration", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("browser host capability accepts new tool commands as supported commands", () => {
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
clientId: "client-1",
|
||||
clientType: "mobile",
|
||||
protocolVersion: 1,
|
||||
capabilities: {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: ["evaluate", "scroll", "resize", "close_tab"],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
},
|
||||
}).capabilities,
|
||||
).toMatchObject({
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: ["evaluate", "scroll", "resize", "close_tab"],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("hello remains valid when no browser host capability is advertised", () => {
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
|
||||
@@ -221,7 +221,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -234,7 +237,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -299,7 +305,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -423,7 +432,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
title: "One",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
secondHost.resolveLatestWith(broker, {
|
||||
@@ -435,7 +447,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://two.example",
|
||||
title: "Two",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
|
||||
@@ -449,6 +464,127 @@ describe("BrowserToolsBroker", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test.each([
|
||||
{
|
||||
name: "scroll",
|
||||
command: { command: "scroll", args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 } },
|
||||
result: { command: "scroll", browserId: BROWSER_ID, deltaX: 0, deltaY: 400 },
|
||||
},
|
||||
{
|
||||
name: "resize",
|
||||
command: { command: "resize", args: { browserId: BROWSER_ID, width: 1024, height: 768 } },
|
||||
result: { command: "resize", browserId: BROWSER_ID, width: 1024, height: 768 },
|
||||
},
|
||||
{
|
||||
name: "close_tab",
|
||||
command: { command: "close_tab", args: { browserId: BROWSER_ID } },
|
||||
result: { command: "close_tab", browserId: BROWSER_ID },
|
||||
},
|
||||
] satisfies Array<{
|
||||
name: string;
|
||||
command: BrowserAutomationCommand;
|
||||
result: BrowserAutomationExecuteResponse["payload"] extends infer Payload
|
||||
? Payload extends { ok: true }
|
||||
? Payload["result"]
|
||||
: never
|
||||
: never;
|
||||
}>)("routes $name to the host that owns the browser id", async ({ command, result }) => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const other = new FakeBrowserHostClient("host-1");
|
||||
const owner = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(other);
|
||||
broker.registerClient(owner);
|
||||
|
||||
const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } });
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
},
|
||||
});
|
||||
await newTabPromise;
|
||||
|
||||
const resultPromise = broker.execute({ command, requestId: "req-command" });
|
||||
|
||||
expect(owner.receivedRequests.at(-1)).toEqual({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-command",
|
||||
command,
|
||||
});
|
||||
expect(other.receivedRequests).toEqual([]);
|
||||
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-command",
|
||||
ok: true,
|
||||
result,
|
||||
});
|
||||
|
||||
await expect(resultPromise).resolves.toEqual({
|
||||
requestId: "req-command",
|
||||
ok: true,
|
||||
result,
|
||||
});
|
||||
});
|
||||
|
||||
test("successful close_tab clears browser id host affinity", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const other = new FakeBrowserHostClient("host-1");
|
||||
const owner = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(other);
|
||||
broker.registerClient(owner);
|
||||
|
||||
const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } });
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
},
|
||||
});
|
||||
await newTabPromise;
|
||||
|
||||
const closePromise = broker.execute({
|
||||
command: { command: "close_tab", args: { browserId: BROWSER_ID } },
|
||||
requestId: "req-close",
|
||||
});
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-close",
|
||||
ok: true,
|
||||
result: { command: "close_tab", browserId: BROWSER_ID },
|
||||
});
|
||||
await expect(closePromise).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "close_tab", browserId: BROWSER_ID },
|
||||
});
|
||||
|
||||
await expect(
|
||||
broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
requestId: "req-after-close",
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-after-close",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: `Browser tab ${BROWSER_ID} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`,
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(owner.receivedRequests.at(-1)?.command).toEqual({
|
||||
command: "close_tab",
|
||||
args: { browserId: BROWSER_ID },
|
||||
});
|
||||
expect(other.receivedRequests).toEqual([]);
|
||||
});
|
||||
|
||||
test("failed list tabs aggregation does not seed browser id affinity", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const firstHost = new FakeBrowserHostClient("host-1");
|
||||
@@ -531,6 +667,35 @@ describe("BrowserToolsBroker", () => {
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
broker.execute({
|
||||
command: {
|
||||
command: "evaluate",
|
||||
args: { browserId: BROWSER_ID, function: "() => document.title" },
|
||||
},
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: 'Browser automation command "evaluate" is not supported by the desktop app.',
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
broker.execute({
|
||||
command: { command: "scroll", args: { browserId: BROWSER_ID, deltaX: 0, deltaY: 400 } },
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: 'Browser automation command "scroll" is not supported by the desktop app.',
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(client.receivedRequests).toEqual([]);
|
||||
});
|
||||
|
||||
@@ -623,7 +788,10 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: "- document",
|
||||
truncated: false,
|
||||
stats: { nodeCount: 1, refCount: 0, textLength: 10 },
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -401,6 +401,12 @@ export class BrowserToolsBroker {
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.result.command === "close_tab") {
|
||||
this.browserHostByBrowserId.delete(payload.result.browserId);
|
||||
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
|
||||
return;
|
||||
}
|
||||
|
||||
if ("browserId" in payload.result) {
|
||||
this.browserHostByBrowserId.set(payload.result.browserId, clientId);
|
||||
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
|
||||
@@ -462,28 +468,10 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
|
||||
function getBrowserIdForCommand(command: BrowserAutomationCommand): string | null {
|
||||
switch (command.command) {
|
||||
case "list_tabs":
|
||||
case "new_tab":
|
||||
return null;
|
||||
case "snapshot":
|
||||
case "click":
|
||||
case "fill":
|
||||
case "wait":
|
||||
case "type":
|
||||
case "keypress":
|
||||
case "navigate":
|
||||
case "back":
|
||||
case "forward":
|
||||
case "reload":
|
||||
case "screenshot":
|
||||
case "upload":
|
||||
case "select":
|
||||
case "hover":
|
||||
case "drag":
|
||||
case "logs":
|
||||
return command.args.browserId;
|
||||
if (command.command === "list_tabs" || command.command === "new_tab") {
|
||||
return null;
|
||||
}
|
||||
return command.args.browserId;
|
||||
}
|
||||
|
||||
function describeBrowserHost(host: RegisteredBrowserHost): string {
|
||||
|
||||
@@ -133,7 +133,10 @@ function snapshotPayload(): Extract<BrowserToolsResponsePayload, { ok: true }> {
|
||||
workspaceId: "wks_workspace_a",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: '- document "Example"\n - button "Save" [ref=@e1]',
|
||||
truncated: false,
|
||||
stats: { nodeCount: 2, refCount: 1, textLength: 50 },
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -158,7 +161,43 @@ const routedToolCases = [
|
||||
name: "click",
|
||||
toolName: "browser_click",
|
||||
input: { browserId: BROWSER_ID, ref: "@e2" },
|
||||
command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e2" } },
|
||||
command: {
|
||||
command: "click",
|
||||
args: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e2",
|
||||
button: "left",
|
||||
doubleClick: false,
|
||||
modifiers: [],
|
||||
},
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-click",
|
||||
ok: true,
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e2" },
|
||||
},
|
||||
content: [{ type: "text", text: "Clicked browser element @e2." }],
|
||||
},
|
||||
{
|
||||
name: "click options",
|
||||
toolName: "browser_click",
|
||||
input: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e2",
|
||||
button: "right",
|
||||
doubleClick: true,
|
||||
modifiers: ["Control", "Shift"],
|
||||
},
|
||||
command: {
|
||||
command: "click",
|
||||
args: {
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e2",
|
||||
button: "right",
|
||||
doubleClick: true,
|
||||
modifiers: ["Control", "Shift"],
|
||||
},
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-click",
|
||||
ok: true,
|
||||
@@ -344,6 +383,85 @@ const routedToolCases = [
|
||||
},
|
||||
content: [{ type: "text", text: "Dragged browser element @e4 to @e5." }],
|
||||
},
|
||||
{
|
||||
name: "evaluate",
|
||||
toolName: "browser_evaluate",
|
||||
input: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" },
|
||||
command: {
|
||||
command: "evaluate",
|
||||
args: { browserId: BROWSER_ID, function: "(element) => element.textContent", ref: "@e1" },
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-evaluate",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "evaluate",
|
||||
browserId: BROWSER_ID,
|
||||
resultJson: '"Save"',
|
||||
truncated: false,
|
||||
},
|
||||
},
|
||||
content: [{ type: "text", text: 'Browser evaluate returned:\n"Save"' }],
|
||||
},
|
||||
{
|
||||
name: "scroll",
|
||||
toolName: "browser_scroll",
|
||||
input: { browserId: BROWSER_ID, ref: "@e1", deltaX: 0, deltaY: 400 },
|
||||
command: {
|
||||
command: "scroll",
|
||||
args: { browserId: BROWSER_ID, ref: "@e1", deltaX: 0, deltaY: 400 },
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-scroll",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "scroll",
|
||||
browserId: BROWSER_ID,
|
||||
ref: "@e1",
|
||||
deltaX: 0,
|
||||
deltaY: 400,
|
||||
},
|
||||
},
|
||||
content: [{ type: "text", text: "Scrolled browser element @e1 by 0, 400." }],
|
||||
},
|
||||
{
|
||||
name: "resize",
|
||||
toolName: "browser_resize",
|
||||
input: { browserId: BROWSER_ID, width: 1024, height: 768 },
|
||||
command: {
|
||||
command: "resize",
|
||||
args: { browserId: BROWSER_ID, width: 1024, height: 768 },
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-resize",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "resize",
|
||||
browserId: BROWSER_ID,
|
||||
width: 1024,
|
||||
height: 768,
|
||||
},
|
||||
},
|
||||
content: [{ type: "text", text: "Resized browser viewport to 1024x768." }],
|
||||
},
|
||||
{
|
||||
name: "close_tab",
|
||||
toolName: "browser_close_tab",
|
||||
input: { browserId: BROWSER_ID },
|
||||
command: {
|
||||
command: "close_tab",
|
||||
args: { browserId: BROWSER_ID },
|
||||
},
|
||||
payload: {
|
||||
requestId: "req-close-tab",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "close_tab",
|
||||
browserId: BROWSER_ID,
|
||||
},
|
||||
},
|
||||
content: [{ type: "text", text: `Closed browser tab ${BROWSER_ID}.` }],
|
||||
},
|
||||
] satisfies Array<{
|
||||
name: string;
|
||||
toolName: string;
|
||||
@@ -459,6 +577,10 @@ describe("registerBrowserTools", () => {
|
||||
"browser_select",
|
||||
"browser_drag",
|
||||
"browser_logs",
|
||||
"browser_evaluate",
|
||||
"browser_scroll",
|
||||
"browser_resize",
|
||||
"browser_close_tab",
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -668,7 +790,10 @@ describe("registerBrowserTools", () => {
|
||||
workspaceId: "wks_workspace_a",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
format: "aria-yaml",
|
||||
snapshot: '- document "Example"\n - button "Save" [ref=@e1]',
|
||||
truncated: false,
|
||||
stats: { nodeCount: 2, refCount: 1, textLength: 50 },
|
||||
},
|
||||
context: {
|
||||
agentId: "agent-1",
|
||||
@@ -726,6 +851,89 @@ describe("registerBrowserTools", () => {
|
||||
},
|
||||
);
|
||||
|
||||
test("success responses include handled dialog metadata and a text note", async () => {
|
||||
const harness = new BrowserToolHarness();
|
||||
const payload: Extract<BrowserToolsResponsePayload, { ok: true }> = {
|
||||
requestId: "req-click",
|
||||
ok: true,
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
|
||||
dialogs: [
|
||||
{
|
||||
type: "confirm",
|
||||
message: "Delete item?",
|
||||
action: "dismissed",
|
||||
timestamp: 123,
|
||||
},
|
||||
],
|
||||
};
|
||||
harness.broker.setResponse(payload);
|
||||
|
||||
const response = await harness.execute("browser_click", { browserId: BROWSER_ID, ref: "@e1" });
|
||||
|
||||
expect(response.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: 'Clicked browser element @e1.\nHandled browser dialog: dismissed confirm "Delete item?".',
|
||||
},
|
||||
]);
|
||||
expect(response.structuredContent).toEqual({
|
||||
ok: true,
|
||||
result: payload.result,
|
||||
dialogs: payload.dialogs,
|
||||
context: {
|
||||
agentId: "agent-1",
|
||||
cwd: "/repo",
|
||||
workspaceId: "wks_workspace_a",
|
||||
browserId: BROWSER_ID,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("failure responses include handled dialog metadata and a text note", async () => {
|
||||
const harness = new BrowserToolHarness();
|
||||
const payload: Extract<BrowserToolsResponsePayload, { ok: false }> = {
|
||||
requestId: "req-wait",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_timeout",
|
||||
message: "Timed out waiting for browser URL: /next",
|
||||
retryable: true,
|
||||
},
|
||||
dialogs: [
|
||||
{
|
||||
type: "beforeunload",
|
||||
message: "Leave site?",
|
||||
action: "dismissed",
|
||||
timestamp: 124,
|
||||
},
|
||||
],
|
||||
};
|
||||
harness.broker.setResponse(payload);
|
||||
|
||||
const response = await harness.execute("browser_wait", {
|
||||
browserId: BROWSER_ID,
|
||||
url: "/next",
|
||||
});
|
||||
|
||||
expect(response.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: 'The browser did not respond before the timeout. Try again or check the browser host.\nHandled browser dialog: dismissed beforeunload "Leave site?".',
|
||||
},
|
||||
]);
|
||||
expect(response.structuredContent).toEqual({
|
||||
ok: false,
|
||||
error: payload.error,
|
||||
dialogs: payload.dialogs,
|
||||
context: {
|
||||
agentId: "agent-1",
|
||||
cwd: "/repo",
|
||||
workspaceId: "wks_workspace_a",
|
||||
browserId: BROWSER_ID,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("wait rejects calls without a condition", () => {
|
||||
const harness = new BrowserToolHarness();
|
||||
|
||||
|
||||
@@ -45,6 +45,18 @@ const BrowserToolOutputSchema = {
|
||||
retryable: z.boolean(),
|
||||
})
|
||||
.optional(),
|
||||
dialogs: z
|
||||
.array(
|
||||
z.object({
|
||||
type: z.enum(["alert", "confirm", "prompt", "beforeunload"]),
|
||||
message: z.string(),
|
||||
defaultValue: z.string().optional(),
|
||||
action: z.enum(["accepted", "dismissed"]),
|
||||
promptText: z.string().optional(),
|
||||
timestamp: z.number(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
context: z
|
||||
.object({
|
||||
agentId: z.string().optional(),
|
||||
@@ -70,6 +82,8 @@ const BrowserHttpUrlInputSchema = z
|
||||
return normalized;
|
||||
});
|
||||
const BrowserRefInputSchema = z.string().regex(/^@e\d+$/);
|
||||
const BrowserClickButtonInputSchema = z.enum(["left", "right", "middle"]);
|
||||
const BrowserClickModifierInputSchema = z.enum(["Alt", "Control", "Meta", "Shift"]);
|
||||
const BrowserWaitInputSchema = z
|
||||
.object({
|
||||
text: z.string().min(1).optional(),
|
||||
@@ -178,10 +192,13 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
inputSchema: {
|
||||
ref: BrowserRefInputSchema,
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
button: BrowserClickButtonInputSchema.optional(),
|
||||
doubleClick: z.boolean().optional(),
|
||||
modifiers: z.array(BrowserClickModifierInputSchema).optional(),
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ ref, browserId }) => {
|
||||
async ({ ref, browserId, button, doubleClick, modifiers }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
@@ -193,6 +210,9 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
args: {
|
||||
browserId,
|
||||
ref,
|
||||
button: button ?? "left",
|
||||
doubleClick: doubleClick ?? false,
|
||||
modifiers: modifiers ?? [],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -605,6 +625,136 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_evaluate",
|
||||
{
|
||||
title: "Evaluate browser JavaScript",
|
||||
description:
|
||||
"Evaluate a JavaScript function in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; when ref is provided, refs come from the latest browser_snapshot and the resolved element is passed as the first argument.",
|
||||
inputSchema: {
|
||||
function: z.string().min(1),
|
||||
ref: BrowserRefInputSchema.optional(),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ function: functionSource, ref, browserId }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
|
||||
command: {
|
||||
command: "evaluate",
|
||||
args: {
|
||||
browserId,
|
||||
function: functionSource,
|
||||
...(ref ? { ref } : {}),
|
||||
},
|
||||
},
|
||||
});
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_scroll",
|
||||
{
|
||||
title: "Scroll browser",
|
||||
description:
|
||||
"Scroll a Paseo browser tab by deltaX/deltaY CSS pixels. Use browserId from browser_new_tab or browser_list_tabs; optional ref comes from the latest browser_snapshot and centers the wheel input over that element.",
|
||||
inputSchema: {
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
ref: BrowserRefInputSchema.optional(),
|
||||
deltaX: z.number(),
|
||||
deltaY: z.number(),
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ browserId, ref, deltaX, deltaY }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
|
||||
command: {
|
||||
command: "scroll",
|
||||
args: {
|
||||
browserId,
|
||||
...(ref ? { ref } : {}),
|
||||
deltaX,
|
||||
deltaY,
|
||||
},
|
||||
},
|
||||
});
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_resize",
|
||||
{
|
||||
title: "Resize browser viewport",
|
||||
description:
|
||||
"Resize a Paseo browser tab's resident webview viewport. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
inputSchema: {
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
width: z.number().int().positive(),
|
||||
height: z.number().int().positive(),
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ browserId, width, height }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
|
||||
command: {
|
||||
command: "resize",
|
||||
args: {
|
||||
browserId,
|
||||
width,
|
||||
height,
|
||||
},
|
||||
},
|
||||
});
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_close_tab",
|
||||
{
|
||||
title: "Close browser tab",
|
||||
description:
|
||||
"Close a Paseo browser tab, remove its resident webview, and unregister it from the browser automation host. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
inputSchema: {
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
async ({ browserId }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const payload = await options.broker.execute({
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
|
||||
command: {
|
||||
command: "close_tab",
|
||||
args: {
|
||||
browserId,
|
||||
},
|
||||
},
|
||||
});
|
||||
return browserToolResult({ payload, context: { ...context, browserId } });
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function resolveBrowserToolContext(options: RegisterBrowserToolsOptions): {
|
||||
@@ -681,16 +831,23 @@ function browserToolResult(params: {
|
||||
structuredContent: {
|
||||
ok: true,
|
||||
result: browserToolStructuredResult(payload.result),
|
||||
...(payload.dialogs ? { dialogs: payload.dialogs } : {}),
|
||||
context,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
content: [{ type: "text", text: summarizeBrowserError(payload.error) }],
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: appendDialogSummary(summarizeBrowserError(payload.error), payload.dialogs),
|
||||
},
|
||||
],
|
||||
structuredContent: {
|
||||
ok: false,
|
||||
error: payload.error,
|
||||
...(payload.dialogs ? { dialogs: payload.dialogs } : {}),
|
||||
context,
|
||||
},
|
||||
};
|
||||
@@ -732,34 +889,35 @@ function browserToolImageContent(
|
||||
function summarizeBrowserSuccess(
|
||||
payload: Extract<BrowserToolsResponsePayload, { ok: true }>,
|
||||
): string {
|
||||
const withDialogs = (summary: string) => appendDialogSummary(summary, payload.dialogs);
|
||||
const controlSummary = summarizeBrowserControlSuccess(payload.result);
|
||||
if (controlSummary) {
|
||||
return controlSummary;
|
||||
return withDialogs(controlSummary);
|
||||
}
|
||||
|
||||
const refActionSummary = summarizeBrowserRefActionSuccess(payload.result);
|
||||
if (refActionSummary) {
|
||||
return refActionSummary;
|
||||
return withDialogs(refActionSummary);
|
||||
}
|
||||
|
||||
const diagnosticsSummary = summarizeBrowserDiagnosticsSuccess(payload.result);
|
||||
if (diagnosticsSummary) {
|
||||
return diagnosticsSummary;
|
||||
return withDialogs(diagnosticsSummary);
|
||||
}
|
||||
|
||||
const keyboardSummary = summarizeBrowserKeyboardSuccess(payload.result);
|
||||
if (keyboardSummary) {
|
||||
return keyboardSummary;
|
||||
return withDialogs(keyboardSummary);
|
||||
}
|
||||
|
||||
const navigationSummary = summarizeBrowserNavigationSuccess(payload.result);
|
||||
if (navigationSummary) {
|
||||
return navigationSummary;
|
||||
return withDialogs(navigationSummary);
|
||||
}
|
||||
|
||||
const mediaSummary = summarizeBrowserMediaSuccess(payload.result);
|
||||
if (mediaSummary) {
|
||||
return mediaSummary;
|
||||
return withDialogs(mediaSummary);
|
||||
}
|
||||
|
||||
if (payload.result.command === "list_tabs") {
|
||||
@@ -771,26 +929,49 @@ function summarizeBrowserSuccess(
|
||||
const active = tab.isActive ? " active" : "";
|
||||
return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url}`;
|
||||
});
|
||||
return [
|
||||
`Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values for tab-scoped browser tools.`,
|
||||
...tabLines,
|
||||
].join("\n");
|
||||
return withDialogs(
|
||||
[
|
||||
`Found ${count} Paseo browser tab${count === 1 ? "" : "s"}. Use these browserId values for tab-scoped browser tools.`,
|
||||
...tabLines,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
if (payload.result.command === "new_tab") {
|
||||
return `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`;
|
||||
return withDialogs(
|
||||
`Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (payload.result.command === "snapshot") {
|
||||
const count = payload.result.elements.length;
|
||||
return `Snapshot captured ${count} element${count === 1 ? "" : "s"}.`;
|
||||
return withDialogs(
|
||||
[
|
||||
`Snapshot captured ${payload.result.stats.nodeCount} node${payload.result.stats.nodeCount === 1 ? "" : "s"} with ${payload.result.stats.refCount} ref${payload.result.stats.refCount === 1 ? "" : "s"}.`,
|
||||
`Title: ${payload.result.title || "Untitled"}`,
|
||||
`URL: ${payload.result.url}`,
|
||||
"",
|
||||
payload.result.snapshot,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
if (payload.result.command === "wait") {
|
||||
return `Browser wait matched ${payload.result.matched}.`;
|
||||
return withDialogs(`Browser wait matched ${payload.result.matched}.`);
|
||||
}
|
||||
|
||||
return `Browser ${payload.result.command} complete.`;
|
||||
return withDialogs(`Browser ${payload.result.command} complete.`);
|
||||
}
|
||||
|
||||
function appendDialogSummary(
|
||||
summary: string,
|
||||
dialogs: BrowserToolsResponsePayload["dialogs"],
|
||||
): string {
|
||||
if (!dialogs || dialogs.length === 0) {
|
||||
return summary;
|
||||
}
|
||||
return `${summary}\nHandled browser dialog${dialogs.length === 1 ? "" : "s"}: ${dialogs
|
||||
.map((dialog) => `${dialog.action} ${dialog.type} ${JSON.stringify(dialog.message)}`)
|
||||
.join("; ")}.`;
|
||||
}
|
||||
|
||||
function summarizeBrowserMediaSuccess(
|
||||
@@ -841,9 +1022,18 @@ function summarizeBrowserNavigationSuccess(
|
||||
function summarizeBrowserDiagnosticsSuccess(
|
||||
result: Extract<BrowserToolsResponsePayload, { ok: true }>["result"],
|
||||
): string | null {
|
||||
if (result.command === "evaluate") {
|
||||
return [
|
||||
"Browser evaluate returned:",
|
||||
result.resultJson,
|
||||
...(result.truncated ? ["Result was truncated."] : []),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
if (result.command !== "logs") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const consoleCount = result.console.length;
|
||||
const networkCount = result.network.length;
|
||||
return `Read ${consoleCount} console log${consoleCount === 1 ? "" : "s"} and ${networkCount} network entr${networkCount === 1 ? "y" : "ies"}.`;
|
||||
@@ -878,6 +1068,20 @@ function summarizeBrowserControlSuccess(
|
||||
return `Dragged browser element ${result.sourceRef} to ${result.targetRef}.`;
|
||||
}
|
||||
|
||||
if (result.command === "scroll") {
|
||||
return result.ref
|
||||
? `Scrolled browser element ${result.ref} by ${result.deltaX}, ${result.deltaY}.`
|
||||
: `Scrolled browser by ${result.deltaX}, ${result.deltaY}.`;
|
||||
}
|
||||
|
||||
if (result.command === "resize") {
|
||||
return `Resized browser viewport to ${result.width}x${result.height}.`;
|
||||
}
|
||||
|
||||
if (result.command === "close_tab") {
|
||||
return `Closed browser tab ${result.browserId}.`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user