puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-08-15 04:42:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(browser-tools): cut low-value tools

Browse Source
This commit is contained in:
Mohamed Boudra
2026-07-02 23:56:20 +02:00
parent b2714ccd89
commit 2ba3ad53ab
13 changed files with 119 additions and 1796 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
packages/desktop/src/features/browser-automation/ipc.test.ts
View File

@@ -1,18 +0,0 @@
import { describe, expect, it } from "vitest";
import { sanitizeDownloadFileName } from "./ipc.js";
describe("browser automation IPC", () => {
it("strips directories from agent-supplied download filenames", () => {
expect(
sanitizeDownloadFileName({
url: "https://example.com/fallback.txt",
fileName: "../../.ssh/authorized_keys",
}),
).toBe("authorized_keys");
});
it("falls back to a safe filename when the URL has no basename", () => {
expect(sanitizeDownloadFileName({ url: "https://example.com/" })).toBe("download");
});
});

57
packages/desktop/src/features/browser-automation/ipc.ts
View File

@@ -1,13 +1,7 @@
import { mkdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { basename, join } from "node:path";
import type { WebContents } from "electron";
import { ipcMain } from "electron";
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type {
BrowserAutomationConsoleLogEntry,
BrowserAutomationCookieEntry,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { TabContents, BrowserRegistry } from "./service.js";
import { executeAutomationCommand } from "./service.js";
import {
@@ -46,61 +40,12 @@ function adaptWebContents(contents: WebContents): TabContents {
isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(),
setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed),
getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [],
getCookies: async (url: string) =>
(await contents.session.cookies.get({ url })).map(normalizeCookie),
sendDebugCommand: async (command: string, params?: Record<string, unknown>) => {
if (!contents.debugger.isAttached()) {
contents.debugger.attach("1.3");
}
return contents.debugger.sendCommand(command, params ?? {});
},
printToPDF: async (options?: Record<string, unknown>) => contents.printToPDF(options ?? {}),
downloadURL: (input) => downloadWithContents(contents, input),
};
}
function downloadWithContents(
contents: WebContents,
input: { url: string; fileName?: string },
): Promise<{ filePath: string; totalBytes?: number; state: string }> {
const downloadDir = join(tmpdir(), "paseo-browser-downloads");
mkdirSync(downloadDir, { recursive: true });
const filePath = join(downloadDir, sanitizeDownloadFileName(input));
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
contents.session.off("will-download", onDownload);
reject(new Error(`Timed out waiting for browser download: ${input.url}`));
}, 30_000);
function onDownload(_event: Electron.Event, item: Electron.DownloadItem): void {
if (item.getURL() !== input.url) {
return;
}
clearTimeout(timeout);
contents.session.off("will-download", onDownload);
item.setSavePath(filePath);
item.once("done", (_doneEvent, state) => {
resolve({ filePath, totalBytes: item.getTotalBytes(), state });
});
}
contents.session.on("will-download", onDownload);
contents.downloadURL(input.url);
});
}
export function sanitizeDownloadFileName(input: { url: string; fileName?: string }): string {
const requestedName = input.fileName ?? basename(new URL(input.url).pathname);
return basename(requestedName) || "download";
}
function normalizeCookie(cookie: Electron.Cookie): BrowserAutomationCookieEntry {
return {
name: cookie.name,
value: cookie.value,
...(cookie.domain ? { domain: cookie.domain } : {}),
...(cookie.path ? { path: cookie.path } : {}),
secure: cookie.secure,
httpOnly: cookie.httpOnly,
...(typeof cookie.expirationDate === "number" ? { expirationDate: cookie.expirationDate } : {}),
};
}

263
packages/desktop/src/features/browser-automation/service.test.ts
View File

@@ -4,7 +4,6 @@ import { describe, expect, test, vi } from "vitest";
import type {
BrowserAutomationCommand,
BrowserAutomationConsoleLogEntry,
BrowserAutomationCookieEntry,
BrowserAutomationExecuteRequest,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
@@ -37,25 +36,14 @@ class FakeTab implements TabContents {
public readonly actions: string[] = [];
public readonly capturedViewports: Array<{ stayHidden?: boolean }> = [];
public readonly debugCommands: Array<{ command: string; params?: Record<string, unknown> }> = [];
public readonly pdfOptions: Record<string, unknown>[] = [];
public readonly downloads: Array<{ url: string; fileName?: string }> = [];
public destroyed = false;
public bodyText = "";
public snapshotElements: unknown[] = [];
public actionScriptResult: unknown = true;
public networkEntries: unknown[] = [];
public storageState: unknown = { localStorage: [], sessionStorage: [] };
public viewport = { width: 1024, height: 768, deviceScaleFactor: 2 };
public consoleMessages: BrowserAutomationConsoleLogEntry[] = [];
public cookies: BrowserAutomationCookieEntry[] = [];
public captureNeverPaints = false;
public pdfBytes = new Uint8Array([37, 80, 68, 70]);
public downloadResult = {
filePath: "/workspace/downloads/file.txt",
totalBytes: 42,
state: "completed",
};
public layoutMetrics = {
cssLayoutViewport: { clientWidth: 390, clientHeight: 844 },
cssContentSize: { width: 390, height: 1200 },
@@ -106,12 +94,6 @@ class FakeTab implements TabContents {
if (code.includes("performance.getEntriesByType")) {
return JSON.stringify(this.networkEntries);
}
if (code.includes("localStorage") && code.includes("sessionStorage")) {
return JSON.stringify(this.storageState);
}
if (code.includes("window.innerWidth")) {
return JSON.stringify(this.viewport);
}
return this.actionScriptResult;
}
@@ -157,10 +139,6 @@ class FakeTab implements TabContents {
return this.consoleMessages;
}
public async getCookies(_url: string): Promise<BrowserAutomationCookieEntry[]> {
return this.cookies;
}
public async sendDebugCommand(
command: string,
params?: Record<string, unknown>,
@@ -180,20 +158,6 @@ class FakeTab implements TabContents {
}
return {};
}
public async printToPDF(options?: Record<string, unknown>): Promise<Uint8Array> {
this.pdfOptions.push(options ?? {});
return this.pdfBytes;
}
public async downloadURL(input: { url: string; fileName?: string }): Promise<{
filePath: string;
totalBytes?: number;
state: string;
}> {
this.downloads.push(input);
return this.downloadResult;
}
}
class FakeRegistry implements BrowserRegistry {
@@ -470,50 +434,20 @@ describe("executeAutomationCommand", () => {
});
});
test("page info reads the explicit browser id from command args", () => {
const browser = new BrowserAutomationHarness();
const result = executeAutomationCommand(
automationRequest(
{ command: "page_info", args: { browserId: BROWSER_A } },
{ requestId: "req-page" },
),
browser.registry,
);
expect(result).toEqual({
requestId: "req-page",
ok: true,
result: {
command: "page_info",
tab: {
browserId: BROWSER_A,
workspaceId: WORKSPACE_A,
url: "https://a.test/form",
title: "Fixture",
isActive: true,
isLoading: false,
canGoBack: true,
canGoForward: false,
},
},
});
});
test("page info returns tab not found for an id in another workspace", () => {
test("tab commands return tab not found for an id in another workspace", async () => {
const registry = new FakeRegistry();
registry.register(BROWSER_A, WORKSPACE_B, new FakeTab(1, "https://a.test", "A"));
const result = executeAutomationCommand(
const result = await executeAutomationCommand(
automationRequest(
{ command: "page_info", args: { browserId: BROWSER_A } },
{ requestId: "req-page" },
{ command: "snapshot", args: { browserId: BROWSER_A } },
{ requestId: "req-snapshot" },
),
registry,
);
expect(result).toEqual({
requestId: "req-page",
requestId: "req-snapshot",
ok: false,
error: {
code: "browser_tab_not_found",
@@ -523,22 +457,22 @@ describe("executeAutomationCommand", () => {
});
});
test("page info returns tab closed for a destroyed explicit tab", () => {
test("tab commands return tab closed for a destroyed explicit tab", async () => {
const tab = new FakeTab(1, "https://a.test", "A");
tab.destroyed = true;
const registry = new FakeRegistry();
registry.register(BROWSER_A, WORKSPACE_A, tab);
const result = executeAutomationCommand(
const result = await executeAutomationCommand(
automationRequest(
{ command: "page_info", args: { browserId: BROWSER_A } },
{ requestId: "req-page" },
{ command: "snapshot", args: { browserId: BROWSER_A } },
{ requestId: "req-snapshot" },
),
registry,
);
expect(result).toEqual({
requestId: "req-page",
requestId: "req-snapshot",
ok: false,
error: {
code: "browser_tab_closed",
@@ -595,22 +529,6 @@ describe("executeAutomationCommand", () => {
expect(containsScript(browser.tab, "#submit", ".click()")).toBe(true);
});
test("set background writes the requested color into the explicit page", async () => {
const browser = new BrowserAutomationHarness();
const result = await browser.execute({
command: "set_background",
args: { browserId: BROWSER_A, color: "red" },
});
expect(result).toEqual({
requestId: "req-set_background",
ok: true,
result: { command: "set_background", browserId: BROWSER_A, color: "red" },
});
expect(containsScript(browser.tab, "document.body.style.background", "red")).toBe(true);
});
test.each([
{
name: "fill updates a ref from the latest snapshot",
@@ -618,24 +536,6 @@ describe("executeAutomationCommand", () => {
result: { command: "fill", browserId: BROWSER_A, ref: "@e1" },
scriptParts: ["#name", "Ada"],
},
{
name: "focus focuses a ref from the latest snapshot",
command: { command: "focus", args: { browserId: BROWSER_A, ref: "@e1" } },
result: { command: "focus", browserId: BROWSER_A, ref: "@e1" },
scriptParts: ["#name", "focus"],
},
{
name: "clear clears a ref from the latest snapshot",
command: { command: "clear", args: { browserId: BROWSER_A, ref: "@e1" } },
result: { command: "clear", browserId: BROWSER_A, ref: "@e1" },
scriptParts: ["#name", "deleteContent"],
},
{
name: "check sets the requested checked state on a ref",
command: { command: "check", args: { browserId: BROWSER_A, ref: "@e2", checked: false } },
result: { command: "check", browserId: BROWSER_A, ref: "@e2", checked: false },
scriptParts: ["#agree", "nextChecked = false"],
},
{
name: "select sets the requested value on a ref",
command: { command: "select", args: { browserId: BROWSER_A, ref: "@e3", value: "us" } },
@@ -672,6 +572,24 @@ describe("executeAutomationCommand", () => {
expect(containsScript(browser.tab, ...scriptParts)).toBe(true);
});
test("fill with an empty string clears a ref through the regular fill path", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.snapshotElements = formElements();
requireSnapshotRefs(await browser.snapshot());
const action = await browser.execute({
command: "fill",
args: { browserId: BROWSER_A, ref: "@e1", value: "" },
});
expect(action).toEqual({
requestId: "req-fill",
ok: true,
result: { command: "fill", browserId: BROWSER_A, ref: "@e1" },
});
expect(containsScript(browser.tab, "#name", 'const nextValue = "";')).toBe(true);
});
test("refs become stale after navigation changes the tab URL", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.snapshotElements = formElements();
@@ -912,71 +830,6 @@ describe("executeAutomationCommand", () => {
});
});
test("storage reads cookies and web storage from the explicit tab", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.cookies = [
{ name: "theme", value: "dark", domain: "a.test", path: "/", secure: true },
];
browser.tab.storageState = {
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
};
const result = await browser.execute({
command: "storage",
args: { browserId: BROWSER_A },
});
expect(result).toEqual({
requestId: "req-storage",
ok: true,
result: {
command: "storage",
browserId: BROWSER_A,
url: "https://a.test/form",
cookies: [{ name: "theme", value: "dark", domain: "a.test", path: "/", secure: true }],
localStorage: [{ key: "token", value: "abc" }],
sessionStorage: [{ key: "tab", value: "1" }],
},
});
});
test("environment applies viewport and geolocation before reporting the current viewport", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.viewport = { width: 390, height: 844, deviceScaleFactor: 3 };
const result = await browser.execute({
command: "environment",
args: {
browserId: BROWSER_A,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
});
expect(result).toEqual({
requestId: "req-environment",
ok: true,
result: {
command: "environment",
browserId: BROWSER_A,
viewport: { width: 390, height: 844, deviceScaleFactor: 3 },
geolocation: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
});
expect(browser.tab.debugCommands).toEqual([
{
command: "Emulation.setDeviceMetricsOverride",
params: { width: 390, height: 844, deviceScaleFactor: 3, mobile: false },
},
{
command: "Emulation.setGeolocationOverride",
params: { latitude: 37.7749, longitude: -122.4194, accuracy: 5 },
},
]);
expect(containsScript(browser.tab, "navigator", "geolocation")).toBe(true);
});
test("screenshot serializes the painted viewport and restores throttling", async () => {
const browser = new BrowserAutomationHarness();
@@ -1039,19 +892,19 @@ describe("executeAutomationCommand", () => {
}
});
test("full page screenshot captures the page content area through CDP", async () => {
test("screenshot with fullPage captures the page content area through CDP", async () => {
const browser = new BrowserAutomationHarness();
const result = await browser.execute({
command: "full_page_screenshot",
args: { browserId: BROWSER_A },
command: "screenshot",
args: { browserId: BROWSER_A, fullPage: true },
});
expect(result).toEqual({
requestId: "req-full_page_screenshot",
requestId: "req-screenshot",
ok: true,
result: {
command: "full_page_screenshot",
command: "screenshot",
browserId: BROWSER_A,
mimeType: "image/png",
dataBase64: "fullPagePng",
@@ -1072,56 +925,6 @@ describe("executeAutomationCommand", () => {
]);
});
test("pdf exports the explicit tab with requested print options", async () => {
const browser = new BrowserAutomationHarness();
const result = await browser.execute({
command: "pdf",
args: { browserId: BROWSER_A, landscape: true, printBackground: false },
});
expect(result).toEqual({
requestId: "req-pdf",
ok: true,
result: {
command: "pdf",
browserId: BROWSER_A,
mimeType: "application/pdf",
dataBase64: "JVBERg==",
},
});
expect(browser.tab.pdfOptions).toEqual([{ printBackground: false, landscape: true }]);
});
test("download saves the requested HTTP URL through the explicit tab", async () => {
const browser = new BrowserAutomationHarness();
const result = await browser.execute({
command: "download",
args: {
browserId: BROWSER_A,
url: "https://a.test/file.txt",
fileName: "file.txt",
},
});
expect(result).toEqual({
requestId: "req-download",
ok: true,
result: {
command: "download",
browserId: BROWSER_A,
url: "https://a.test/file.txt",
filePath: "/workspace/downloads/file.txt",
totalBytes: 42,
state: "completed",
},
});
expect(browser.tab.downloads).toEqual([
{ url: "https://a.test/file.txt", fileName: "file.txt" },
]);
});
test("upload resolves workspace files before setting them on the file input", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.snapshotElements = [

495
packages/desktop/src/features/browser-automation/service.ts
View File

@@ -3,12 +3,10 @@ import { isAbsolute, relative, resolve as resolvePath } from "node:path";
import type {
BrowserAutomationCommand,
BrowserAutomationConsoleLogEntry,
BrowserAutomationCookieEntry,
BrowserAutomationErrorCode,
BrowserAutomationExecuteResponse,
BrowserAutomationExecuteRequest,
BrowserAutomationNetworkLogEntry,
BrowserAutomationStorageEntry,
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
@@ -30,14 +28,7 @@ export interface TabContents {
isBackgroundThrottlingAllowed(): boolean;
setBackgroundThrottling(allowed: boolean): void;
getConsoleMessages?(): BrowserAutomationConsoleLogEntry[];
getCookies?(url: string): Promise<BrowserAutomationCookieEntry[]>;
sendDebugCommand?(command: string, params?: Record<string, unknown>): Promise<unknown>;
printToPDF?(options?: Record<string, unknown>): Promise<Uint8Array>;
downloadURL?(input: { url: string; fileName?: string }): Promise<{
filePath: string;
totalBytes?: number;
state: string;
}>;
}
export interface TabImage {
@@ -195,10 +186,6 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
executeListTabs(requestId, workspaceId, registry),
new_tab: ({ requestId }) =>
fail(requestId, "browser_unsupported", "browser_new_tab is handled by the app runtime."),
page_info: ({ command, requestId, workspaceId, registry }) => {
const pageInfoCommand = command as Extract<BrowserAutomationCommand, { command: "page_info" }>;
return executePageInfo(requestId, workspaceId, pageInfoCommand.args.browserId, registry);
},
snapshot: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const snapshotCommand = command as Extract<BrowserAutomationCommand, { command: "snapshot" }>;
return executeSnapshot(
@@ -315,37 +302,11 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
BrowserAutomationCommand,
{ command: "screenshot" }
>;
return executeScreenshot(requestId, workspaceId, screenshotCommand.args.browserId, registry);
},
full_page_screenshot: ({ command, requestId, workspaceId, registry }) => {
const screenshotCommand = command as Extract<
BrowserAutomationCommand,
{ command: "full_page_screenshot" }
>;
return executeFullPageScreenshot(
return executeScreenshot(
requestId,
workspaceId,
screenshotCommand.args.browserId,
registry,
);
},
pdf: ({ command, requestId, workspaceId, registry }) => {
const pdfCommand = command as Extract<BrowserAutomationCommand, { command: "pdf" }>;
return executePdf(
requestId,
workspaceId,
pdfCommand.args.browserId,
{ landscape: pdfCommand.args.landscape, printBackground: pdfCommand.args.printBackground },
registry,
);
},
download: ({ command, requestId, workspaceId, registry }) => {
const downloadCommand = command as Extract<BrowserAutomationCommand, { command: "download" }>;
return executeDownload(
requestId,
workspaceId,
downloadCommand.args.browserId,
{ url: downloadCommand.args.url, fileName: downloadCommand.args.fileName },
screenshotCommand.args.fullPage,
registry,
);
},
@@ -361,40 +322,6 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
snapshotEngine,
);
},
focus: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const focusCommand = command as Extract<BrowserAutomationCommand, { command: "focus" }>;
return executeFocus(
requestId,
workspaceId,
focusCommand.args.browserId,
focusCommand.args.ref,
registry,
snapshotEngine,
);
},
clear: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const clearCommand = command as Extract<BrowserAutomationCommand, { command: "clear" }>;
return executeClear(
requestId,
workspaceId,
clearCommand.args.browserId,
clearCommand.args.ref,
registry,
snapshotEngine,
);
},
check: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const checkCommand = command as Extract<BrowserAutomationCommand, { command: "check" }>;
return executeCheck(
requestId,
workspaceId,
checkCommand.args.browserId,
checkCommand.args.ref,
checkCommand.args.checked,
registry,
snapshotEngine,
);
},
select: ({ command, requestId, workspaceId, registry, snapshotEngine }) => {
const selectCommand = command as Extract<BrowserAutomationCommand, { command: "select" }>;
return executeSelect(
@@ -440,39 +367,6 @@ const commandHandlers: Record<BrowserAutomationCommand["command"], CommandHandle
registry,
);
},
storage: ({ command, requestId, workspaceId, registry }) => {
const storageCommand = command as Extract<BrowserAutomationCommand, { command: "storage" }>;
return executeStorage(requestId, workspaceId, storageCommand.args.browserId, registry);
},
environment: ({ command, requestId, workspaceId, registry }) => {
const environmentCommand = command as Extract<
BrowserAutomationCommand,
{ command: "environment" }
>;
return executeEnvironment(
requestId,
workspaceId,
environmentCommand.args.browserId,
{
viewport: environmentCommand.args.viewport,
geolocation: environmentCommand.args.geolocation,
},
registry,
);
},
set_background: ({ command, requestId, workspaceId, registry }) => {
const setBackgroundCommand = command as Extract<
BrowserAutomationCommand,
{ command: "set_background" }
>;
return executeSetBackground(
requestId,
workspaceId,
setBackgroundCommand.args.browserId,
setBackgroundCommand.args.color,
registry,
);
},
};
interface ResolvedTabTarget {
@@ -508,37 +402,6 @@ function executeListTabs(
return { requestId, ok: true, result: { command: "list_tabs", tabs } };
}
function executePageInfo(
requestId: string,
workspaceId: string | undefined,
browserId: string,
registry: BrowserRegistry,
): AutomationCommandPayload {
const target = resolveTabTarget({
requestId,
workspaceId,
browserId,
registry,
});
if ("ok" in target) {
return target;
}
return {
requestId,
ok: true,
result: {
command: "page_info",
tab: tabInfoFromContents(
target.browserId,
target.contents,
workspaceId ? registry.getWorkspaceActiveBrowserId(workspaceId) : null,
registry.getBrowserWorkspaceId(target.browserId),
),
},
};
}
async function executeSnapshot(
requestId: string,
workspaceId: string | undefined,
@@ -625,81 +488,6 @@ async function executeFill(
return { requestId, ok: true, result: { command: "fill", browserId: target.browserId, ref } };
}
async function executeFocus(
requestId: string,
workspaceId: string | undefined,
browserId: string,
ref: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
const result = await snapshotEngine.focus({
browserId: target.browserId,
page: target.contents,
ref,
});
if (!result.ok) {
return staleRefFailure(requestId, ref);
}
return { requestId, ok: true, result: { command: "focus", browserId: target.browserId, ref } };
}
async function executeClear(
requestId: string,
workspaceId: string | undefined,
browserId: string,
ref: string,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
const result = await snapshotEngine.clear({
browserId: target.browserId,
page: target.contents,
ref,
});
if (!result.ok) {
return staleRefFailure(requestId, ref);
}
return { requestId, ok: true, result: { command: "clear", browserId: target.browserId, ref } };
}
async function executeCheck(
requestId: string,
workspaceId: string | undefined,
browserId: string,
ref: string,
checked: boolean,
registry: BrowserRegistry,
snapshotEngine: BrowserSnapshotEngine,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
const result = await snapshotEngine.check({
browserId: target.browserId,
page: target.contents,
ref,
checked,
});
if (!result.ok) {
return staleRefFailure(requestId, ref);
}
return {
requestId,
ok: true,
result: { command: "check", browserId: target.browserId, ref, checked },
};
}
async function executeSelect(
requestId: string,
workspaceId: string | undefined,
@@ -808,84 +596,6 @@ async function executeLogs(
};
}
async function executeStorage(
requestId: string,
workspaceId: string | undefined,
browserId: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
const url = target.contents.getURL();
const cookies = (await target.contents.getCookies?.(url)) ?? [];
const storage = parseStorageState(await target.contents.executeJavaScript(STORAGE_STATE_SCRIPT));
return {
requestId,
ok: true,
result: {
command: "storage",
browserId: target.browserId,
url,
cookies,
localStorage: storage.localStorage,
sessionStorage: storage.sessionStorage,
},
};
}
async function executeEnvironment(
requestId: string,
workspaceId: string | undefined,
browserId: string,
environment: {
viewport?: { width: number; height: number; deviceScaleFactor?: number };
geolocation?: { latitude: number; longitude: number; accuracy?: number };
},
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
if (environment.viewport) {
await target.contents.sendDebugCommand?.("Emulation.setDeviceMetricsOverride", {
width: environment.viewport.width,
height: environment.viewport.height,
deviceScaleFactor: environment.viewport.deviceScaleFactor ?? 1,
mobile: false,
});
}
if (environment.geolocation) {
const geolocation = {
latitude: environment.geolocation.latitude,
longitude: environment.geolocation.longitude,
accuracy: environment.geolocation.accuracy ?? 1,
};
await target.contents.sendDebugCommand?.("Emulation.setGeolocationOverride", geolocation);
await target.contents.executeJavaScript(buildGeolocationShimScript(geolocation));
}
const viewport = parseViewport(await target.contents.executeJavaScript(VIEWPORT_SCRIPT));
return {
requestId,
ok: true,
result: {
command: "environment",
browserId: target.browserId,
viewport,
...(environment.geolocation
? {
geolocation: {
...environment.geolocation,
accuracy: environment.geolocation.accuracy ?? 1,
},
}
: {}),
},
};
}
function staleRefFailure(requestId: string, ref: string): FailurePayload {
return fail(
requestId,
@@ -952,39 +662,6 @@ async function executeWait(
return fail(requestId, "browser_unsupported", "browser_wait requires text or url");
}
async function executeSetBackground(
requestId: string,
workspaceId: string | undefined,
browserId: string,
color: string,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
await target.contents.executeJavaScript(buildSetBackgroundScript(color));
return {
requestId,
ok: true,
result: { command: "set_background", browserId: target.browserId, color },
};
}
function buildSetBackgroundScript(color: string): string {
return String.raw`(() => {
const color = ${JSON.stringify(color)};
document.documentElement.style.background = color;
if (document.body) {
document.body.style.background = color;
document.body.style.backgroundColor = color;
document.body.style.minHeight = '100vh';
}
return true;
})()`;
}
async function executeType(
requestId: string,
workspaceId: string | undefined,
@@ -1092,8 +769,13 @@ async function executeScreenshot(
requestId: string,
workspaceId: string | undefined,
browserId: string,
fullPage: boolean,
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
if (fullPage) {
return executeFullPageScreenshot(requestId, workspaceId, browserId, registry);
}
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
@@ -1176,7 +858,7 @@ async function executeFullPageScreenshot(
return target;
}
if (!target.contents.sendDebugCommand) {
return fail(requestId, "browser_unsupported", "browser_full_page_screenshot requires CDP");
return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP");
}
const metrics = await getCdpLayoutMetrics(target.contents);
const width = metrics.contentWidth;
@@ -1197,13 +879,13 @@ async function executeFullPageScreenshot(
throw error;
}
if (!screenshot.data) {
return fail(requestId, "browser_unsupported", "browser_full_page_screenshot returned no data");
return fail(requestId, "browser_unsupported", "browser_screenshot fullPage returned no data");
}
return {
requestId,
ok: true,
result: {
command: "full_page_screenshot",
command: "screenshot",
browserId: target.browserId,
mimeType: "image/png",
dataBase64: screenshot.data,
@@ -1213,68 +895,6 @@ async function executeFullPageScreenshot(
};
}
async function executePdf(
requestId: string,
workspaceId: string | undefined,
browserId: string,
options: { landscape?: boolean; printBackground: boolean },
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
if (!target.contents.printToPDF) {
return fail(requestId, "browser_unsupported", "browser_pdf requires PDF support");
}
const pdf = await target.contents.printToPDF({
printBackground: options.printBackground,
...(options.landscape !== undefined ? { landscape: options.landscape } : {}),
});
return {
requestId,
ok: true,
result: {
command: "pdf",
browserId: target.browserId,
mimeType: "application/pdf",
dataBase64: Buffer.from(pdf).toString("base64"),
},
};
}
async function executeDownload(
requestId: string,
workspaceId: string | undefined,
browserId: string,
input: { url: string; fileName?: string },
registry: BrowserRegistry,
): Promise<AutomationCommandPayload> {
const target = resolveTabTarget({ requestId, workspaceId, browserId, registry });
if ("ok" in target) {
return target;
}
if (!isAllowedPageUrl(input.url)) {
return fail(requestId, "browser_denied", "Browser download only supports http and https URLs.");
}
if (!target.contents.downloadURL) {
return fail(requestId, "browser_unsupported", "browser_download requires download support");
}
const download = await target.contents.downloadURL(input);
return {
requestId,
ok: true,
result: {
command: "download",
browserId: target.browserId,
url: input.url,
filePath: download.filePath,
...(download.totalBytes !== undefined ? { totalBytes: download.totalBytes } : {}),
state: download.state,
},
};
}
function isAllowedPageUrl(value: string): boolean {
try {
return ALLOWED_PAGE_URL_PROTOCOLS.has(new URL(value).protocol);
@@ -1408,36 +1028,6 @@ function parseNetworkEntries(value: unknown): BrowserAutomationNetworkLogEntry[]
});
}
function parseStorageState(value: unknown): {
localStorage: BrowserAutomationStorageEntry[];
sessionStorage: BrowserAutomationStorageEntry[];
} {
const parsed = typeof value === "string" ? JSON.parse(value) : value;
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return { localStorage: [], sessionStorage: [] };
}
const record = parsed as Record<string, unknown>;
return {
localStorage: parseStorageEntries(record.localStorage),
sessionStorage: parseStorageEntries(record.sessionStorage),
};
}
function parseStorageEntries(value: unknown): BrowserAutomationStorageEntry[] {
if (!Array.isArray(value)) {
return [];
}
return value.flatMap((entry): BrowserAutomationStorageEntry[] => {
if (!entry || typeof entry !== "object") {
return [];
}
const record = entry as Record<string, unknown>;
const key = readString(record.key);
const itemValue = readString(record.value);
return key !== null && itemValue !== null ? [{ key, value: itemValue }] : [];
});
}
function readString(value: unknown): string | null {
return typeof value === "string" ? value : null;
}
@@ -1446,43 +1036,6 @@ function readNumber(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}
function parseViewport(value: unknown): {
width: number;
height: number;
deviceScaleFactor: number;
} {
const parsed = typeof value === "string" ? JSON.parse(value) : value;
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return { width: 0, height: 0, deviceScaleFactor: 1 };
}
const record = parsed as Record<string, unknown>;
return {
width: readNumber(record.width) ?? 0,
height: readNumber(record.height) ?? 0,
deviceScaleFactor: readNumber(record.deviceScaleFactor) ?? 1,
};
}
function buildGeolocationShimScript(geolocation: {
latitude: number;
longitude: number;
accuracy: number;
}): string {
return String.raw`(() => {
const coords = ${JSON.stringify({ ...geolocation, altitude: null, altitudeAccuracy: null, heading: null, speed: null })};
const position = { coords, timestamp: Date.now() };
Object.defineProperty(navigator, 'geolocation', {
configurable: true,
value: {
getCurrentPosition(success) { setTimeout(() => success(position), 0); },
watchPosition(success) { setTimeout(() => success(position), 0); return 1; },
clearWatch() {},
},
});
return true;
})()`;
}
const NETWORK_PERFORMANCE_SCRIPT = String.raw`(() => {
const entries = performance.getEntriesByType('resource')
.concat(performance.getEntriesByType('navigation'))
@@ -1498,34 +1051,6 @@ const NETWORK_PERFORMANCE_SCRIPT = String.raw`(() => {
return JSON.stringify(entries);
})()`;
const STORAGE_STATE_SCRIPT = String.raw`(() => {
function entriesFor(storage) {
const entries = [];
for (let index = 0; index < storage.length; index += 1) {
const key = storage.key(index);
if (key !== null) entries.push({ key, value: storage.getItem(key) || '' });
}
return entries;
}
function safeEntries(readStorage) {
try {
return entriesFor(readStorage());
} catch {
return [];
}
}
return JSON.stringify({
localStorage: safeEntries(() => window.localStorage),
sessionStorage: safeEntries(() => window.sessionStorage),
});
})()`;
const VIEWPORT_SCRIPT = String.raw`(() => JSON.stringify({
width: window.innerWidth,
height: window.innerHeight,
deviceScaleFactor: window.devicePixelRatio || 1,
}))()`;
function resolveTabTarget(input: {
requestId: string;
workspaceId: string | undefined;

4
packages/desktop/src/features/browser-automation/snapshot-engine.test.ts
View File

@@ -44,7 +44,9 @@ describe("BrowserSnapshotEngine", () => {
ok: false,
reason: "stale_ref",
});
await expect(engine.focus({ browserId: "browser-1", page, ref: "@e1" })).resolves.toEqual({
await expect(
engine.select({ browserId: "browser-1", page, ref: "@e1", value: "us" }),
).resolves.toEqual({
ok: false,
reason: "stale_ref",
});

76
packages/desktop/src/features/browser-automation/snapshot-engine.ts
View File

@@ -108,31 +108,6 @@ export class BrowserSnapshotEngine {
return input.ref && result === false ? { ok: false, reason: "stale_ref" } : { ok: true };
}
async focus(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildFocusScript(selector));
}
async clear(input: {
browserId: string;
page: SnapshotPage;
ref: string;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildClearScript(selector));
}
async check(input: {
browserId: string;
page: SnapshotPage;
ref: string;
checked: boolean;
}): Promise<BrowserRefActionResult> {
return this.runRefScript(input, (selector) => buildCheckScript(selector, input.checked));
}
async select(input: {
browserId: string;
page: SnapshotPage;
@@ -304,57 +279,6 @@ function buildKeypressScript(selector: string | undefined, key: string): string
})()`;
}
function buildFocusScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
return document.activeElement === element;
})()`;
}
function buildClearScript(selector: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
if ('value' in element) {
element.value = '';
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
element.textContent = '';
element.dispatchEvent(new InputEvent('input', { bubbles: true, inputType: 'deleteContent' }));
return true;
})()`;
}
function buildCheckScript(selector: string, checked: boolean): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});
if (!element) return false;
element.scrollIntoView?.({ block: 'center', inline: 'center' });
element.focus?.();
const nextChecked = ${JSON.stringify(checked)};
if ('checked' in element) {
element.checked = nextChecked;
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
if (element.getAttribute('role') === 'checkbox' || element.getAttribute('role') === 'radio') {
element.setAttribute('aria-checked', String(nextChecked));
element.dispatchEvent(new Event('input', { bubbles: true }));
element.dispatchEvent(new Event('change', { bubbles: true }));
return true;
}
return false;
})()`;
}
function buildSelectScript(selector: string, value: string): string {
return String.raw`(() => {
const element = document.querySelector(${JSON.stringify(selector)});