Fix browser screenshot capture for parked webviews

This commit is contained in:
Mohamed Boudra
2026-07-03 00:27:35 +02:00
parent b0088303fd
commit 9430a897e8
17 changed files with 1721 additions and 58 deletions

View File

@@ -21,34 +21,35 @@ This is an npm workspace monorepo:
At the start of non-trivial work, list `docs/` and skim anything relevant to the task. When you learn something meta worth preserving — a gotcha, a convention, a workflow, a piece of system context that will outlive the current task — update an existing doc or propose a new one. Code-level facts belong in inline comments next to the code; system, process, and gotcha-level facts belong in `docs/`. At the start of non-trivial work, list `docs/` and skim anything relevant to the task. When you learn something meta worth preserving — a gotcha, a convention, a workflow, a piece of system context that will outlive the current task — update an existing doc or propose a new one. Code-level facts belong in inline comments next to the code; system, process, and gotcha-level facts belong in `docs/`.
| Doc | What's in it | | Doc | What's in it |
| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ |
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going | | [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow | | [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track | | [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations | | [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms | | [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization | | [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons | | [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it | | [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order | | [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash | | [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas | | [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer | | [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end | | [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries | | [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config | | [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP | | [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs | | [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage | | [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization | | [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows | | [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness | | [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows | | [docs/browser-capture-harness.md](docs/browser-capture-harness.md) | Real-Electron browser screenshot harness and compositor-surface gotcha |
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security | | [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist | | [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider | | [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth | | [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
## Quick start ## Quick start

View File

@@ -0,0 +1,39 @@
# Browser Capture Harness
The desktop capture harness is the real-Electron verification path for browser screenshots.
It validates the compositor behavior that unit tests cannot see:
- the resident automation `<webview>` starts in the production parking state;
- the parked guest has no copyable viewport frame;
- the resident webview guest is sized to 1280x800 logical pixels;
- the app-style prep sequence moves the host to a paintable 1x1 clipped state, waits two animation frames and a layout read, then both viewport `capturePage` and full-page CDP screenshots return real pixels;
- restore returns the host to offscreen parking after every capture.
Run it with the repo Electron:
```bash
npm run capture-harness --workspace=@getpaseo/desktop
```
The harness writes PNG evidence and `results.json` to:
```text
packages/desktop/capture-harness/out/
```
A passing run prints `PASS` lines for guest sizing, the expected parked-capture failure,
five viewport prep captures, five full-page prep captures, and final completion. The PNG
sizes may be device-pixel scaled; on a Retina display the 1280x800 logical viewport is
usually saved as 2560x1600.
## Mechanism
Electron captures copy from the guest web contents' compositor surface. A resident
webview parked at `left:-20000px` and `opacity:0` does not have a copyable surface, and
`capturePage({ stayHidden:false })` or CDP `Page.captureScreenshot` cannot rescue it.
Before pixel capture, the app renderer temporarily makes the resident host paintable:
`left:0`, `top:0`, `opacity:1`, `pointer-events:none`, host size `1x1`, and
`overflow:hidden`, with the full-size 1280x800 webview inside. Main captures only after
the renderer acknowledges two animation frames plus a `getBoundingClientRect()` read, and
the renderer restores parking in a `finally`.

View File

@@ -1,6 +1,9 @@
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host"; import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident"; import {
ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault,
installResidentBrowserCaptureBridge,
} from "@/components/browser-webview-resident";
import { createWorkspaceBrowser } from "@/stores/browser-store"; import { createWorkspaceBrowser } from "@/stores/browser-store";
import { import {
buildWorkspaceTabPersistenceKey, buildWorkspaceTabPersistenceKey,
@@ -40,7 +43,8 @@ export function mountBrowserAutomationHandler(
options: BrowserAutomationHandlerOptions, options: BrowserAutomationHandlerOptions,
): () => void { ): () => void {
const getHost = options.getHost ?? getDesktopHost; const getHost = options.getHost ?? getDesktopHost;
return options.client.on("browser.automation.execute.request", (request) => { const uninstallCaptureBridge = installResidentBrowserCaptureBridge();
const unsubscribe = options.client.on("browser.automation.execute.request", (request) => {
void handleBrowserAutomationRequest({ void handleBrowserAutomationRequest({
client: options.client, client: options.client,
getHost, getHost,
@@ -56,6 +60,10 @@ export function mountBrowserAutomationHandler(
: {}), : {}),
}); });
}); });
return () => {
unsubscribe();
uninstallCaptureBridge();
};
} }
export function mountBrowserAutomationDaemonClientHandler( export function mountBrowserAutomationDaemonClientHandler(

View File

@@ -2,8 +2,10 @@ import { afterEach, describe, expect, it } from "vitest";
import { import {
clearResidentBrowserWebviewsForTests, clearResidentBrowserWebviewsForTests,
ensureResidentBrowserWebview, ensureResidentBrowserWebview,
prepareResidentBrowserWebviewForPixelCapture,
releaseResidentBrowserWebview, releaseResidentBrowserWebview,
removeResidentBrowserWebview, removeResidentBrowserWebview,
restoreResidentBrowserWebviewAfterPixelCapture,
takeResidentBrowserWebview, takeResidentBrowserWebview,
} from "./browser-webview-resident"; } from "./browser-webview-resident";
@@ -22,6 +24,7 @@ describe("resident browser webviews", () => {
expect(host.children).toHaveLength(0); expect(host.children).toHaveLength(0);
expect(webview.isConnected).toBe(true); expect(webview.isConnected).toBe(true);
expect(webview.style.display).toBe("inline-flex");
expect(webview.style.width).toBe("1280px"); expect(webview.style.width).toBe("1280px");
expect(webview.style.height).toBe("800px"); expect(webview.style.height).toBe("800px");
@@ -57,4 +60,65 @@ describe("resident browser webviews", () => {
expect(webview?.isConnected).toBe(false); expect(webview?.isConnected).toBe(false);
expect(takeResidentBrowserWebview("browser-closed")).toBeNull(); expect(takeResidentBrowserWebview("browser-closed")).toBeNull();
}); });
it("temporarily makes resident webviews paintable for pixel capture", async () => {
const webview = ensureResidentBrowserWebview({
browserId: "browser-capture",
url: "https://example.com",
});
if (!webview) {
throw new Error("Expected resident browser webview");
}
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
browserId: "browser-capture",
});
const host = document.getElementById("paseo-browser-resident-webviews");
expect(preparation.token).toBe("capture-1");
expect(host?.style.left).toBe("0px");
expect(host?.style.top).toBe("0px");
expect(host?.style.width).toBe("1px");
expect(host?.style.height).toBe("1px");
expect(host?.style.overflow).toBe("hidden");
expect(host?.style.opacity).toBe("1");
expect(host?.style.pointerEvents).toBe("none");
expect(webview.style.display).toBe("inline-flex");
expect(webview.style.width).toBe("1280px");
expect(webview.style.height).toBe("800px");
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
expect(host?.style.left).toBe("-20000px");
expect(host?.style.width).toBe("1280px");
expect(host?.style.height).toBe("800px");
expect(host?.style.opacity).toBe("0");
});
it("keeps the resident host paintable until every capture token is restored", async () => {
ensureResidentBrowserWebview({
browserId: "browser-overlap",
url: "https://example.com",
});
const first = await prepareResidentBrowserWebviewForPixelCapture({
browserId: "browser-overlap",
});
const second = await prepareResidentBrowserWebviewForPixelCapture({
browserId: "browser-overlap",
});
const host = document.getElementById("paseo-browser-resident-webviews");
await restoreResidentBrowserWebviewAfterPixelCapture(first);
expect(host?.style.left).toBe("0px");
expect(host?.style.width).toBe("1px");
expect(host?.style.opacity).toBe("1");
await restoreResidentBrowserWebviewAfterPixelCapture(second);
expect(host?.style.left).toBe("-20000px");
expect(host?.style.width).toBe("1280px");
expect(host?.style.opacity).toBe("0");
});
}); });

View File

@@ -1,9 +1,16 @@
import { getDesktopHost } from "@/desktop/host";
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews"; const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id"; const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
const RESIDENT_VIEWPORT_WIDTH = 1280; const RESIDENT_VIEWPORT_WIDTH = 1280;
const RESIDENT_VIEWPORT_HEIGHT = 800; const RESIDENT_VIEWPORT_HEIGHT = 800;
const residentWebviewsByBrowserId = new Map<string, HTMLElement>(); const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
const activeCapturePreparations = new Map<string, { preparesResidentHost: boolean }>();
let captureBridgeInstallCount = 0;
let captureBridgeDisposer: (() => void) | null = null;
let nextCapturePreparationId = 0;
interface BrowserWebviewElement extends HTMLElement { interface BrowserWebviewElement extends HTMLElement {
src: string; src: string;
@@ -21,14 +28,7 @@ function readDocument(): Document | null {
return typeof document === "undefined" ? null : document; return typeof document === "undefined" ? null : document;
} }
function getResidentBrowserHost(ownerDocument: Document): HTMLElement { function applyResidentHostParkingStyle(host: HTMLElement): void {
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
if (existing) {
return existing;
}
const host = ownerDocument.createElement("div");
host.id = RESIDENT_BROWSER_HOST_ID;
host.setAttribute("aria-hidden", "true"); host.setAttribute("aria-hidden", "true");
host.style.position = "fixed"; host.style.position = "fixed";
host.style.left = "-20000px"; host.style.left = "-20000px";
@@ -38,27 +38,88 @@ function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
host.style.overflow = "hidden"; host.style.overflow = "hidden";
host.style.opacity = "0"; host.style.opacity = "0";
host.style.pointerEvents = "none"; host.style.pointerEvents = "none";
host.style.zIndex = "";
host.style.clipPath = "";
host.style.visibility = "";
host.style.transform = "";
}
function applyResidentHostCaptureStyle(host: HTMLElement): void {
host.setAttribute("aria-hidden", "true");
host.style.position = "fixed";
host.style.left = "0";
host.style.top = "0";
host.style.width = "1px";
host.style.height = "1px";
host.style.overflow = "hidden";
host.style.opacity = "1";
host.style.pointerEvents = "none";
host.style.zIndex = "1";
host.style.clipPath = "";
host.style.visibility = "";
host.style.transform = "";
}
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
if (existing) {
return existing;
}
const host = ownerDocument.createElement("div");
host.id = RESIDENT_BROWSER_HOST_ID;
applyResidentHostParkingStyle(host);
ownerDocument.body.appendChild(host); ownerDocument.body.appendChild(host);
return host; return host;
} }
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null { function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) { for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
if (!(element instanceof HTMLElement)) {
continue;
}
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) { if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
return element as HTMLElement; return element;
} }
} }
return null; return null;
} }
function applyResidentWebviewStyle(webview: HTMLElement): void { function applyResidentWebviewStyle(webview: HTMLElement): void {
webview.style.display = "block"; webview.style.display = "inline-flex";
webview.style.flex = "0 0 auto";
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`; webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`; webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
webview.style.border = "0"; webview.style.border = "0";
webview.style.background = "transparent"; webview.style.background = "transparent";
} }
function hasActiveResidentHostPreparation(): boolean {
for (const preparation of activeCapturePreparations.values()) {
if (preparation.preparesResidentHost) {
return true;
}
}
return false;
}
function nextAnimationFrame(): Promise<void> {
if (typeof requestAnimationFrame !== "function") {
return Promise.resolve();
}
return new Promise((resolve) => {
requestAnimationFrame(() => {
resolve();
});
});
}
async function waitForCapturePaint(webview: HTMLElement): Promise<void> {
await nextAnimationFrame();
await nextAnimationFrame();
webview.getBoundingClientRect();
}
export function prepareBrowserWebview( export function prepareBrowserWebview(
webview: HTMLElement, webview: HTMLElement,
input: { browserId: string; initialUrl?: string | null }, input: { browserId: string; initialUrl?: string | null },
@@ -133,6 +194,92 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl
getResidentBrowserHost(ownerDocument).appendChild(webview); getResidentBrowserHost(ownerDocument).appendChild(webview);
} }
export async function prepareResidentBrowserWebviewForPixelCapture(input: {
browserId: string;
}): Promise<{ token: string }> {
const browserId = trimNonEmpty(input.browserId);
if (!browserId) {
throw new Error("Browser id is required for pixel capture preparation.");
}
const ownerDocument = readDocument();
if (!ownerDocument) {
throw new Error("Browser pixel capture preparation requires a document.");
}
const host = getResidentBrowserHost(ownerDocument);
const webview = findBrowserWebview(browserId, ownerDocument);
if (!webview) {
throw new Error(`Browser webview ${browserId} is not mounted.`);
}
const token = `capture-${++nextCapturePreparationId}`;
const preparesResidentHost = webview.parentElement === host;
activeCapturePreparations.set(token, { preparesResidentHost });
try {
if (preparesResidentHost) {
applyResidentHostCaptureStyle(host);
applyResidentWebviewStyle(webview);
}
await waitForCapturePaint(webview);
return { token };
} catch (error) {
activeCapturePreparations.delete(token);
if (!hasActiveResidentHostPreparation()) {
applyResidentHostParkingStyle(host);
}
throw error;
}
}
export async function restoreResidentBrowserWebviewAfterPixelCapture(input: {
token: string;
}): Promise<void> {
const preparation = activeCapturePreparations.get(input.token);
if (!preparation) {
return;
}
activeCapturePreparations.delete(input.token);
if (!preparation.preparesResidentHost || hasActiveResidentHostPreparation()) {
return;
}
const ownerDocument = readDocument();
if (!ownerDocument) {
return;
}
const host = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
if (host instanceof HTMLElement) {
applyResidentHostParkingStyle(host);
}
}
export function installResidentBrowserCaptureBridge(): () => void {
captureBridgeInstallCount += 1;
if (!captureBridgeDisposer) {
const browserBridge = getDesktopHost()?.browser;
const disposePrepare = browserBridge?.onPrepareForPixelCapture?.(
prepareResidentBrowserWebviewForPixelCapture,
);
const disposeRestore = browserBridge?.onRestorePixelCapture?.(
restoreResidentBrowserWebviewAfterPixelCapture,
);
captureBridgeDisposer = () => {
disposePrepare?.();
disposeRestore?.();
};
}
return () => {
captureBridgeInstallCount = Math.max(0, captureBridgeInstallCount - 1);
if (captureBridgeInstallCount > 0) {
return;
}
captureBridgeDisposer?.();
captureBridgeDisposer = null;
};
}
export function removeResidentBrowserWebview(browserId: string): void { export function removeResidentBrowserWebview(browserId: string): void {
const normalizedBrowserId = trimNonEmpty(browserId); const normalizedBrowserId = trimNonEmpty(browserId);
if (!normalizedBrowserId) { if (!normalizedBrowserId) {
@@ -149,5 +296,7 @@ export function clearResidentBrowserWebviewsForTests(): void {
webview.remove(); webview.remove();
} }
residentWebviewsByBrowserId.clear(); residentWebviewsByBrowserId.clear();
activeCapturePreparations.clear();
nextCapturePreparationId = 0;
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove(); readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
} }

View File

@@ -121,6 +121,10 @@ export interface DesktopBrowserShortcutEvent {
action: "focus-url"; action: "focus-url";
} }
export interface DesktopBrowserPixelCapturePreparation {
token: string;
}
export interface DesktopBrowserNewTabRequestEvent { export interface DesktopBrowserNewTabRequestEvent {
sourceBrowserId: string; sourceBrowserId: string;
url: string; url: string;
@@ -144,6 +148,12 @@ export interface DesktopBrowserBridge {
) => Promise<string | null>; ) => Promise<string | null>;
/** Copy element text and/or an image to the system clipboard from main. */ /** Copy element text and/or an image to the system clipboard from main. */
copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>; copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
onPrepareForPixelCapture?: (
handler: (input: { browserId: string }) => Promise<DesktopBrowserPixelCapturePreparation>,
) => () => void;
onRestorePixelCapture?: (
handler: (input: DesktopBrowserPixelCapturePreparation) => Promise<void>,
) => () => void;
} }
export interface DesktopInvokeBridge { export interface DesktopInvokeBridge {

View File

@@ -0,0 +1,2 @@
out/
fatal-error.txt

View File

@@ -0,0 +1,74 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>Paseo Capture Harness Target</title>
<style>
html,
body {
width: 1280px;
min-height: 1600px;
margin: 0;
background: #ff00ff;
overflow: auto;
font-family: Arial, Helvetica, sans-serif;
}
.label {
position: absolute;
left: 48px;
top: 48px;
color: #ffffff;
font-size: 72px;
font-weight: 800;
line-height: 1;
text-shadow:
5px 5px 0 #000000,
-2px -2px 0 #000000;
}
.sub {
position: absolute;
left: 54px;
top: 150px;
color: #000000;
background: #ffffff;
font-size: 34px;
font-weight: 700;
padding: 10px 18px;
}
.block {
position: absolute;
left: 56px;
top: 220px;
width: 360px;
height: 92px;
background: linear-gradient(
90deg,
#000000 0 25%,
#ffffff 25% 50%,
#000000 50% 75%,
#ffffff 75% 100%
);
border: 8px solid #000000;
}
.bottom {
position: absolute;
left: 48px;
top: 1370px;
color: #ffffff;
font-size: 60px;
font-weight: 800;
text-shadow: 5px 5px 0 #000000;
}
</style>
</head>
<body>
<div class="label">PASEO CAPTURE HARNESS</div>
<div class="sub">LOCAL MAGENTA TARGET 1280x800 VIEWPORT</div>
<div class="block"></div>
<div class="bottom">FULL PAGE MARKER AT Y=1370</div>
</body>
</html>

View File

@@ -0,0 +1,166 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>Paseo Browser Capture Harness</title>
<style>
html,
body {
width: 100%;
height: 100%;
margin: 0;
background: #202020;
overflow: hidden;
font-family: Arial, sans-serif;
}
#paseo-browser-resident-webviews {
position: fixed;
left: -20000px;
top: 0;
width: 1280px;
height: 800px;
overflow: hidden;
opacity: 0;
pointer-events: none;
}
#target-webview {
display: inline-flex;
flex: 0 0 auto;
width: 1280px;
height: 800px;
border: 0;
background: transparent;
}
</style>
</head>
<body>
<div id="paseo-browser-resident-webviews" aria-hidden="true"></div>
<script>
const RESIDENT_VIEWPORT_WIDTH = 1280;
const RESIDENT_VIEWPORT_HEIGHT = 800;
const host = document.getElementById("paseo-browser-resident-webviews");
const params = new URLSearchParams(window.location.search);
const activeTokens = new Set();
let nextTokenId = 0;
const webview = document.createElement("webview");
webview.id = "target-webview";
webview.setAttribute("data-paseo-browser-id", "capture-harness");
webview.setAttribute("partition", "persist:paseo-capture-harness");
webview.setAttribute("allowpopups", "true");
webview.setAttribute("spellcheck", "false");
webview.setAttribute("autosize", "on");
webview.src = params.get("targetUrl") || "bright.html";
host.appendChild(webview);
function applyParking() {
host.style.position = "fixed";
host.style.left = "-20000px";
host.style.top = "0";
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
host.style.overflow = "hidden";
host.style.opacity = "0";
host.style.pointerEvents = "none";
host.style.zIndex = "";
host.style.clipPath = "";
host.style.visibility = "";
host.style.transform = "";
webview.style.display = "inline-flex";
webview.style.flex = "0 0 auto";
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
webview.style.border = "0";
webview.style.background = "transparent";
}
function applyCapturePrep() {
host.style.position = "fixed";
host.style.left = "0";
host.style.top = "0";
host.style.width = "1px";
host.style.height = "1px";
host.style.overflow = "hidden";
host.style.opacity = "1";
host.style.pointerEvents = "none";
host.style.zIndex = "1";
host.style.clipPath = "";
host.style.visibility = "";
host.style.transform = "";
}
function waitFrames(count) {
return new Promise((resolve) => {
function step(remaining) {
if (remaining <= 0) {
resolve();
return;
}
requestAnimationFrame(() => step(remaining - 1));
}
step(count);
});
}
function state() {
const hostRect = host.getBoundingClientRect();
const webviewRect = webview.getBoundingClientRect();
return {
hostStyle: {
left: host.style.left,
top: host.style.top,
width: host.style.width,
height: host.style.height,
overflow: host.style.overflow,
opacity: host.style.opacity,
pointerEvents: host.style.pointerEvents,
},
hostRect: {
x: hostRect.x,
y: hostRect.y,
width: hostRect.width,
height: hostRect.height,
},
webviewRect: {
x: webviewRect.x,
y: webviewRect.y,
width: webviewRect.width,
height: webviewRect.height,
},
};
}
window.captureHarness = {
async waitForFrames(count = 2) {
await waitFrames(count);
return state();
},
async prepareForPixelCapture() {
const token = `capture-${++nextTokenId}`;
activeTokens.add(token);
applyCapturePrep();
await waitFrames(2);
webview.getBoundingClientRect();
return { token, state: state() };
},
async restorePixelCapture(token) {
activeTokens.delete(token);
if (activeTokens.size === 0) {
applyParking();
}
await waitFrames(2);
return state();
},
async restoreParking() {
activeTokens.clear();
applyParking();
await waitFrames(2);
return state();
},
};
</script>
</body>
</html>

View File

@@ -0,0 +1,378 @@
const fs = require("node:fs");
const fsp = require("node:fs/promises");
const path = require("node:path");
const { app, BrowserWindow, nativeImage } = require("electron");
const ROOT = __dirname;
const OUT_DIR = process.env.PASEO_CAPTURE_HARNESS_OUT_DIR || path.join(ROOT, "out");
const VIEWPORT_WIDTH = 1280;
const VIEWPORT_HEIGHT = 800;
const FULL_PAGE_HEIGHT = 1600;
const CAPTURE_TIMEOUT_MS = 5000;
const REPEAT_COUNT = 5;
function fileUrl(filePath) {
return new URL(`file://${filePath}`).toString();
}
function ensureDirSync(dir) {
fs.mkdirSync(dir, { recursive: true });
}
function withTimeout(promise, label) {
let timeoutId;
const timeout = new Promise((_, reject) => {
timeoutId = setTimeout(() => {
reject(new Error(`${label} timed out after ${CAPTURE_TIMEOUT_MS}ms`));
}, CAPTURE_TIMEOUT_MS);
});
return Promise.race([promise, timeout]).finally(() => {
clearTimeout(timeoutId);
});
}
function pixelOffset(width, x, y) {
return (y * width + x) * 4;
}
function isBrightMagenta(bitmap, offset) {
const c0 = bitmap[offset];
const c1 = bitmap[offset + 1];
const c2 = bitmap[offset + 2];
return c0 > 200 && c1 < 90 && c2 > 200;
}
function analyzeImage(image, expected, guestMetrics) {
if (!image || image.isEmpty()) {
return {
width: 0,
height: 0,
brightRatio: 0,
textNonUniform: false,
pass: false,
};
}
const size = image.getSize();
const width = size.width;
const height = size.height;
const bitmap = image.toBitmap();
const totalPixels = width * height;
let brightPixels = 0;
for (let offset = 0; offset < bitmap.length; offset += 4) {
if (isBrightMagenta(bitmap, offset)) {
brightPixels += 1;
}
}
const crop = {
left: Math.min(40, Math.max(0, width - 1)),
top: Math.min(40, Math.max(0, height - 1)),
right: Math.min(width, 940),
bottom: Math.min(height, 260),
};
let cropPixels = 0;
let cropNonBright = 0;
let luminanceSum = 0;
let luminanceSqSum = 0;
const quantized = new Set();
for (let y = crop.top; y < crop.bottom; y += 1) {
for (let x = crop.left; x < crop.right; x += 1) {
const offset = pixelOffset(width, x, y);
cropPixels += 1;
if (!isBrightMagenta(bitmap, offset)) {
cropNonBright += 1;
}
const r = bitmap[offset + 2];
const g = bitmap[offset + 1];
const b = bitmap[offset];
const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b;
luminanceSum += luma;
luminanceSqSum += luma * luma;
quantized.add(`${r >> 5},${g >> 5},${b >> 5},${bitmap[offset + 3] >> 6}`);
}
}
const devicePixelRatio =
typeof guestMetrics.devicePixelRatio === "number" && guestMetrics.devicePixelRatio > 0
? guestMetrics.devicePixelRatio
: 1;
const sizeTargets = [
{ width: expected.width, height: expected.height },
{
width: Math.round(expected.width * devicePixelRatio),
height: Math.round(expected.height * devicePixelRatio),
},
];
const matchedSize = sizeTargets.some(
(target) => Math.abs(width - target.width) <= 2 && Math.abs(height - target.height) <= 2,
);
const luminanceMean = cropPixels ? luminanceSum / cropPixels : 0;
const luminanceVariance = cropPixels
? luminanceSqSum / cropPixels - luminanceMean * luminanceMean
: 0;
const brightRatio = totalPixels ? brightPixels / totalPixels : 0;
const textNonUniform =
cropPixels > 0 &&
cropNonBright / cropPixels > 0.02 &&
quantized.size >= 4 &&
luminanceVariance > 100;
return {
width,
height,
logicalWidthAtDpr: width / devicePixelRatio,
logicalHeightAtDpr: height / devicePixelRatio,
brightRatio,
textNonUniform,
matchedSize,
pass: matchedSize && brightRatio >= expected.minBrightRatio && textNonUniform,
};
}
function pass(message) {
console.log(`PASS ${message}`);
}
function fail(message) {
console.log(`FAIL ${message}`);
throw new Error(message);
}
async function saveImage(image, outputPath) {
await fsp.writeFile(outputPath, image.toPNG());
}
async function waitForGuestLoad(contents) {
await new Promise((resolve) => {
if (!contents.isLoading()) {
resolve();
return;
}
contents.once("did-finish-load", resolve);
contents.once("did-fail-load", resolve);
});
await new Promise((resolve) => setTimeout(resolve, 500));
}
async function renderer(win, expression) {
return await win.webContents.executeJavaScript(expression, true);
}
async function readGuestMetrics(contents) {
return await contents.executeJavaScript(
`({
innerWidth: window.innerWidth,
innerHeight: window.innerHeight,
devicePixelRatio: window.devicePixelRatio,
documentClientWidth: document.documentElement.clientWidth,
documentClientHeight: document.documentElement.clientHeight,
scrollWidth: document.documentElement.scrollWidth,
scrollHeight: document.documentElement.scrollHeight,
visualViewport: window.visualViewport ? {
width: window.visualViewport.width,
height: window.visualViewport.height,
scale: window.visualViewport.scale
} : null
})`,
true,
);
}
async function capturePageSequence(contents) {
const previousBackgroundThrottling = contents.getBackgroundThrottling();
contents.setBackgroundThrottling(false);
try {
contents.invalidate();
return await withTimeout(contents.capturePage(undefined, { stayHidden: false }), "capturePage");
} finally {
contents.setBackgroundThrottling(previousBackgroundThrottling);
}
}
async function captureFullPage(contents) {
let attachedHere = false;
if (!contents.debugger.isAttached()) {
contents.debugger.attach("1.3");
attachedHere = true;
}
try {
const metrics = await contents.debugger.sendCommand("Page.getLayoutMetrics");
const contentSize = metrics.cssContentSize ||
metrics.contentSize || {
x: 0,
y: 0,
width: VIEWPORT_WIDTH,
height: FULL_PAGE_HEIGHT,
};
const clip = {
x: Math.floor(contentSize.x || 0),
y: Math.floor(contentSize.y || 0),
width: Math.ceil(contentSize.width || VIEWPORT_WIDTH),
height: Math.ceil(contentSize.height || FULL_PAGE_HEIGHT),
scale: 1,
};
const result = await withTimeout(
contents.debugger.sendCommand("Page.captureScreenshot", {
format: "png",
captureBeyondViewport: true,
clip,
}),
"CDP Page.captureScreenshot",
);
return nativeImage.createFromBuffer(Buffer.from(result.data, "base64"));
} finally {
if (attachedHere && contents.debugger.isAttached()) {
contents.debugger.detach();
}
}
}
async function captureWithPrep({ win, contents, mode, repeatIndex, guestMetrics }) {
const preparation = await renderer(win, "window.captureHarness.prepareForPixelCapture()");
const outputPath = path.join(OUT_DIR, `${mode}-prep-${repeatIndex}.png`);
try {
const image =
mode === "viewport" ? await capturePageSequence(contents) : await captureFullPage(contents);
await saveImage(image, outputPath);
const expected =
mode === "viewport"
? { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 }
: { width: VIEWPORT_WIDTH, height: FULL_PAGE_HEIGHT, minBrightRatio: 0.55 };
const analysis = analyzeImage(image, expected, guestMetrics);
const size = `${analysis.width}x${analysis.height}`;
const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`;
const bright = analysis.brightRatio.toFixed(4);
if (!analysis.pass) {
fail(
`${mode} prep ${repeatIndex}/${REPEAT_COUNT} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
);
}
pass(
`${mode} prep ${repeatIndex}/${REPEAT_COUNT} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
);
return analysis;
} finally {
const restoredState = await renderer(
win,
`window.captureHarness.restorePixelCapture(${JSON.stringify(preparation.token)})`,
);
const style = restoredState.hostStyle;
if (style.left !== "-20000px" || style.opacity !== "0") {
fail(`restore left=${style.left} opacity=${style.opacity}`);
}
}
}
async function main() {
ensureDirSync(OUT_DIR);
let resolveGuest;
const guestPromise = new Promise((resolve) => {
resolveGuest = resolve;
});
const win = new BrowserWindow({
width: 1000,
height: 700,
show: true,
backgroundColor: "#202020",
webPreferences: {
webviewTag: true,
contextIsolation: true,
nodeIntegration: false,
sandbox: false,
},
});
win.webContents.on("will-attach-webview", (_event, webPreferences) => {
webPreferences.nodeIntegration = false;
webPreferences.contextIsolation = true;
});
win.webContents.on("did-attach-webview", (_event, contents) => {
resolveGuest(contents);
});
await win.loadFile(path.join(ROOT, "index.html"), {
query: { targetUrl: fileUrl(path.join(ROOT, "bright.html")) },
});
const guest = await withTimeout(guestPromise, "did-attach-webview");
await waitForGuestLoad(guest);
await renderer(win, "window.captureHarness.waitForFrames(2)");
const guestMetrics = await readGuestMetrics(guest);
if (guestMetrics.innerWidth !== VIEWPORT_WIDTH || guestMetrics.innerHeight !== VIEWPORT_HEIGHT) {
fail(
`guest viewport sizing inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} expected=${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
);
}
pass(
`guest viewport sizing inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} dpr=${guestMetrics.devicePixelRatio}`,
);
await renderer(win, "window.captureHarness.restoreParking()");
try {
const image = await capturePageSequence(guest);
const analysis = analyzeImage(
image,
{ width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 },
guestMetrics,
);
fail(
`parked webview unexpectedly captured size=${analysis.width}x${analysis.height} bright=${analysis.brightRatio.toFixed(4)}`,
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
pass(`parked webview has no copyable viewport frame error=${message}`);
}
const results = [];
for (let index = 1; index <= REPEAT_COUNT; index += 1) {
results.push(
await captureWithPrep({
win,
contents: guest,
mode: "viewport",
repeatIndex: index,
guestMetrics,
}),
);
}
for (let index = 1; index <= REPEAT_COUNT; index += 1) {
results.push(
await captureWithPrep({
win,
contents: guest,
mode: "full-page",
repeatIndex: index,
guestMetrics,
}),
);
}
await fsp.writeFile(
path.join(OUT_DIR, "results.json"),
`${JSON.stringify({ generatedAt: new Date().toISOString(), guestMetrics, results }, null, 2)}\n`,
);
pass(`capture harness complete output=${OUT_DIR}`);
if (!win.isDestroyed()) {
win.close();
}
}
app
.whenReady()
.then(main)
.then(() => app.quit())
.catch(async (error) => {
console.error(error);
try {
await fsp.writeFile(
path.join(OUT_DIR, "fatal-error.txt"),
`${error && error.stack ? error.stack : String(error)}\n`,
);
} catch {
// Ignore reporting failures during shutdown.
}
app.exit(1);
});

View File

@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd)
exec "$REPO_ROOT/node_modules/.bin/electron" "$SCRIPT_DIR/main.js"

View File

@@ -17,6 +17,7 @@
"scripts": { "scripts": {
"build": "npm --prefix ../.. run build:server:clean && npm run build:main && electron-builder --config electron-builder.yml", "build": "npm --prefix ../.. run build:server:clean && npm run build:main && electron-builder --config electron-builder.yml",
"build:main": "tsc -p tsconfig.json", "build:main": "tsc -p tsconfig.json",
"capture-harness": "./capture-harness/run.sh",
"dev": "./scripts/dev.sh", "dev": "./scripts/dev.sh",
"dev:win": "powershell ./scripts/dev.ps1", "dev:win": "powershell ./scripts/dev.ps1",
"test": "vitest run", "test": "vitest run",

View File

@@ -0,0 +1,239 @@
import type { Rectangle } from "electron";
import { describe, expect, test } from "vitest";
import type { TabImage } from "./service.js";
import { adaptWebContents } from "./ipc.js";
class FakeImage implements TabImage {
public toPNG(): Uint8Array {
return new Uint8Array([137, 80, 78, 71]);
}
public getSize(): { width: number; height: number } {
return { width: 640, height: 480 };
}
}
class FakeDebugger {
public isAttached(): boolean {
return false;
}
public attach(): void {}
public async sendCommand(): Promise<unknown> {
return {};
}
}
class FakeHostWebContents {
public readonly sentMessages: Array<{ channel: string; payload: unknown }> = [];
public destroyed = false;
public constructor(public readonly id: number) {}
public isDestroyed(): boolean {
return this.destroyed;
}
public send(channel: string, payload: unknown): void {
this.sentMessages.push({ channel, payload });
}
}
type IpcListener = (event: unknown, payload: unknown) => void;
class FakeIpcBridge {
private readonly listeners = new Map<string, IpcListener[]>();
public on(channel: string, listener: IpcListener): void {
const listeners = this.listeners.get(channel) ?? [];
listeners.push(listener);
this.listeners.set(channel, listeners);
}
public removeListener(channel: string, listener: IpcListener): void {
const listeners = this.listeners.get(channel) ?? [];
this.listeners.set(
channel,
listeners.filter((candidate) => candidate !== listener),
);
}
public emit(channel: string, payload: unknown): void {
for (const listener of this.listeners.get(channel) ?? []) {
listener({}, payload);
}
}
public listenerCount(channel: string): number {
return this.listeners.get(channel)?.length ?? 0;
}
}
class FakeWebContents {
public readonly debugger = new FakeDebugger();
public readonly consoleMessages: unknown[] = [];
public readonly destroyedListeners: Array<() => void> = [];
public destroyed = false;
public constructor(
public readonly id: number,
public hostWebContents: FakeHostWebContents | null,
) {}
public getURL(): string {
return "https://example.com";
}
public getTitle(): string {
return "Example";
}
public canGoBack(): boolean {
return false;
}
public canGoForward(): boolean {
return false;
}
public isLoading(): boolean {
return false;
}
public isDestroyed(): boolean {
return this.destroyed;
}
public async executeJavaScript(): Promise<unknown> {
return null;
}
public async loadURL(): Promise<void> {}
public goBack(): void {}
public goForward(): void {}
public reload(): void {}
public async capturePage(
_rect?: Rectangle,
_options?: { stayHidden?: boolean },
): Promise<TabImage> {
return new FakeImage();
}
public invalidate(): void {}
public getBackgroundThrottling(): boolean {
return true;
}
public setBackgroundThrottling(): void {}
public on(
event: "console-message",
listener: (
event: unknown,
level: unknown,
message: unknown,
line: unknown,
sourceId: unknown,
) => void,
): void {
this.consoleMessages.push({ event, listener });
}
public once(event: "destroyed", listener: () => void): void {
expect(event).toBe("destroyed");
this.destroyedListeners.push(listener);
}
}
describe("browser automation IPC adapter", () => {
test("prepareForPixelCapture asks the embedder renderer and resolves the ack token", async () => {
const host = new FakeHostWebContents(10);
const contents = new FakeWebContents(20, host);
const ipc = new FakeIpcBridge();
const tab = adaptWebContents(contents, "browser-a", {
ipc,
createRequestId: () => "prepare-1",
});
const preparation = tab.prepareForPixelCapture();
expect(host.sentMessages).toEqual([
{
channel: "paseo:browser:capture-prepare",
payload: { requestId: "prepare-1", browserId: "browser-a" },
},
]);
ipc.emit("paseo:browser:capture-prepared", {
requestId: "other",
ok: true,
token: "wrong-token",
});
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1);
ipc.emit("paseo:browser:capture-prepared", {
requestId: "prepare-1",
ok: true,
token: "token-a",
});
await expect(preparation).resolves.toEqual({ token: "token-a" });
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
});
test("restorePixelCapture sends the capture token back to the embedder renderer", async () => {
const host = new FakeHostWebContents(10);
const contents = new FakeWebContents(20, host);
const ipc = new FakeIpcBridge();
const tab = adaptWebContents(contents, "browser-a", {
ipc,
createRequestId: () => "restore-1",
});
const restored = tab.restorePixelCapture({ token: "token-a" });
expect(host.sentMessages).toEqual([
{
channel: "paseo:browser:capture-restore",
payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" },
},
]);
ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true });
await expect(restored).resolves.toBeUndefined();
});
test("prepareForPixelCapture rejects when the renderer reports preparation failure", async () => {
const host = new FakeHostWebContents(10);
const contents = new FakeWebContents(20, host);
const ipc = new FakeIpcBridge();
const tab = adaptWebContents(contents, "browser-a", {
ipc,
createRequestId: () => "prepare-1",
});
const preparation = tab.prepareForPixelCapture();
ipc.emit("paseo:browser:capture-prepared", {
requestId: "prepare-1",
ok: false,
message: "renderer could not prep",
});
await expect(preparation).rejects.toThrow("renderer could not prep");
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
});
test("prepareForPixelCapture rejects when the guest has no embedder renderer", async () => {
const contents = new FakeWebContents(20, null);
const tab = adaptWebContents(contents, "browser-a");
await expect(tab.prepareForPixelCapture()).rejects.toThrow(
"Browser host renderer is not available.",
);
});
});

View File

@@ -1,8 +1,8 @@
import type { WebContents } from "electron"; import type { Rectangle } from "electron";
import { ipcMain } from "electron"; import { ipcMain } from "electron";
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import type { TabContents, BrowserRegistry } from "./service.js"; import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
import { executeAutomationCommand } from "./service.js"; import { executeAutomationCommand } from "./service.js";
import { import {
listRegisteredPaseoBrowserIds, listRegisteredPaseoBrowserIds,
@@ -13,14 +13,86 @@ import {
} from "../browser-webviews/index.js"; } from "../browser-webviews/index.js";
const MAX_CONSOLE_MESSAGES_PER_TAB = 200; const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
const PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS = 5_000;
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>(); const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
const observedContentsIds = new Set<number>(); const observedContentsIds = new Set<number>();
let nextPixelCaptureBridgeRequest = 0;
interface IpcHandlerRegistry { interface IpcHandlerRegistry {
handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void; handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
} }
function adaptWebContents(contents: WebContents): TabContents { type IpcListener = (event: unknown, payload: unknown) => void;
interface IpcCaptureBridge {
on(channel: string, listener: IpcListener): void;
removeListener(channel: string, listener: IpcListener): void;
}
interface HostWebContents {
readonly id: number;
isDestroyed(): boolean;
send(channel: string, payload: unknown): void;
}
interface WebContentsDebugger {
isAttached(): boolean;
attach(protocolVersion?: string): void;
sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown>;
}
interface ConsoleMessageEmitter {
on(
event: "console-message",
listener: (
event: unknown,
level: unknown,
message: unknown,
line: unknown,
sourceId: unknown,
) => void,
): void;
once(event: "destroyed", listener: () => void): void;
}
interface BrowserAutomationWebContents extends ConsoleMessageEmitter {
readonly id: number;
readonly hostWebContents: HostWebContents | null;
readonly debugger: WebContentsDebugger;
getURL(): string;
getTitle(): string;
canGoBack(): boolean;
canGoForward(): boolean;
isLoading(): boolean;
isDestroyed(): boolean;
executeJavaScript(code: string): Promise<unknown>;
loadURL(url: string): Promise<void>;
goBack(): void;
goForward(): void;
reload(): void;
capturePage(rect?: Rectangle, options?: { stayHidden?: boolean }): Promise<TabImage>;
invalidate(): void;
getBackgroundThrottling(): boolean;
setBackgroundThrottling(allowed: boolean): void;
}
type PixelCaptureBridgeKind = "prepare" | "restore";
interface PixelCaptureBridgeSuccess {
token?: string;
}
interface PixelCaptureBridgeOptions {
ipc?: IpcCaptureBridge;
createRequestId?: () => string;
timeoutMs?: number;
}
export function adaptWebContents(
contents: BrowserAutomationWebContents,
browserId: string,
options?: PixelCaptureBridgeOptions,
): TabContents {
observeConsoleMessages(contents); observeConsoleMessages(contents);
return { return {
id: contents.id, id: contents.id,
@@ -35,7 +107,19 @@ function adaptWebContents(contents: WebContents): TabContents {
goBack: () => contents.goBack(), goBack: () => contents.goBack(),
goForward: () => contents.goForward(), goForward: () => contents.goForward(),
reload: () => contents.reload(), reload: () => contents.reload(),
capturePage: (options) => contents.capturePage(undefined, options), capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
prepareForPixelCapture: async () => {
const result = await requestPixelCaptureBridge(contents, browserId, "prepare", options);
if (!result.token) {
throw new Error("Browser pixel capture preparation did not return a token.");
}
return { token: result.token };
},
restorePixelCapture: async (preparation) => {
await requestPixelCaptureBridge(contents, browserId, "restore", options, {
token: preparation.token,
});
},
invalidate: () => contents.invalidate(), invalidate: () => contents.invalidate(),
isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(), isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(),
setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed), setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed),
@@ -49,7 +133,101 @@ function adaptWebContents(contents: WebContents): TabContents {
}; };
} }
function observeConsoleMessages(contents: WebContents): void { function requestPixelCaptureBridge(
contents: BrowserAutomationWebContents,
browserId: string,
kind: PixelCaptureBridgeKind,
options: PixelCaptureBridgeOptions | undefined,
extraPayload?: { token: string },
): Promise<PixelCaptureBridgeSuccess> {
const host = contents.hostWebContents;
if (!host || host.isDestroyed()) {
return Promise.reject(new Error("Browser host renderer is not available."));
}
const ipc = options?.ipc ?? ipcMain;
const requestId =
options?.createRequestId?.() ?? `browser-pixel-capture-${++nextPixelCaptureBridgeRequest}`;
const timeoutMs = options?.timeoutMs ?? PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS;
const requestChannel =
kind === "prepare" ? "paseo:browser:capture-prepare" : "paseo:browser:capture-restore";
const responseChannel =
kind === "prepare" ? "paseo:browser:capture-prepared" : "paseo:browser:capture-restored";
return new Promise((resolve, reject) => {
let timeoutId: ReturnType<typeof setTimeout> | undefined;
const cleanup = () => {
if (timeoutId) {
clearTimeout(timeoutId);
}
ipc.removeListener(responseChannel, listener);
};
const listener: IpcListener = (_event, payload) => {
const response = readPixelCaptureBridgeResponse(payload, requestId);
if (!response) {
return;
}
cleanup();
if (response.ok) {
resolve(response);
} else {
reject(new Error(response.message));
}
};
ipc.on(responseChannel, listener);
timeoutId = setTimeout(() => {
cleanup();
reject(new Error(`Browser pixel capture ${kind} timed out.`));
}, timeoutMs);
try {
host.send(requestChannel, {
requestId,
browserId,
...(extraPayload ? { token: extraPayload.token } : {}),
});
} catch (error) {
cleanup();
reject(error);
}
});
}
function readPixelCaptureBridgeResponse(
payload: unknown,
requestId: string,
): ({ ok: true } & PixelCaptureBridgeSuccess) | { ok: false; message: string } | null {
if (!isRecord(payload)) {
return null;
}
const record = payload;
if (record.requestId !== requestId) {
return null;
}
if (record.ok === true) {
return {
ok: true,
...(typeof record.token === "string" && record.token.length > 0
? { token: record.token }
: {}),
};
}
if (record.ok === false) {
const message =
typeof record.message === "string" && record.message.length > 0
? record.message
: "Browser pixel capture bridge failed.";
return { ok: false, message };
}
return null;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
if (observedContentsIds.has(contents.id)) { if (observedContentsIds.has(contents.id)) {
return; return;
} }
@@ -89,7 +267,7 @@ function createRegistry(): BrowserRegistry {
listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace, listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace,
getTabContents(browserId: string): TabContents | null { getTabContents(browserId: string): TabContents | null {
const contents = getPaseoBrowserWebContents(browserId); const contents = getPaseoBrowserWebContents(browserId);
return contents ? adaptWebContents(contents) : null; return contents ? adaptWebContents(contents, browserId) : null;
}, },
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId, getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId, getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,

View File

@@ -7,7 +7,12 @@ import type {
BrowserAutomationExecuteRequest, BrowserAutomationExecuteRequest,
} from "@getpaseo/protocol/browser-automation/rpc-schemas"; } from "@getpaseo/protocol/browser-automation/rpc-schemas";
import { BrowserSnapshotEngine } from "./snapshot-engine.js"; import { BrowserSnapshotEngine } from "./snapshot-engine.js";
import type { BrowserRegistry, TabContents, TabImage } from "./service.js"; import type {
BrowserRegistry,
TabContents,
TabImage,
TabPixelCapturePreparation,
} from "./service.js";
import { executeAutomationCommand } from "./service.js"; import { executeAutomationCommand } from "./service.js";
const BROWSER_A = "11111111-1111-4111-8111-111111111111"; const BROWSER_A = "11111111-1111-4111-8111-111111111111";
@@ -36,6 +41,9 @@ class FakeTab implements TabContents {
public readonly actions: string[] = []; public readonly actions: string[] = [];
public readonly capturedViewports: Array<{ stayHidden?: boolean }> = []; public readonly capturedViewports: Array<{ stayHidden?: boolean }> = [];
public readonly debugCommands: Array<{ command: string; params?: Record<string, unknown> }> = []; public readonly debugCommands: Array<{ command: string; params?: Record<string, unknown> }> = [];
public readonly restoredPixelCaptureTokens: string[] = [];
private readonly captureStartWaiters: Array<() => void> = [];
private readonly deferredCaptures: Array<(image: TabImage) => void> = [];
public destroyed = false; public destroyed = false;
public bodyText = ""; public bodyText = "";
@@ -44,6 +52,9 @@ class FakeTab implements TabContents {
public networkEntries: unknown[] = []; public networkEntries: unknown[] = [];
public consoleMessages: BrowserAutomationConsoleLogEntry[] = []; public consoleMessages: BrowserAutomationConsoleLogEntry[] = [];
public captureNeverPaints = false; public captureNeverPaints = false;
public captureThrows = false;
public deferCaptures = false;
public prepareNeverAcks = false;
public layoutMetrics = { public layoutMetrics = {
cssLayoutViewport: { clientWidth: 390, clientHeight: 844 }, cssLayoutViewport: { clientWidth: 390, clientHeight: 844 },
cssContentSize: { width: 390, height: 1200 }, cssContentSize: { width: 390, height: 1200 },
@@ -52,6 +63,7 @@ class FakeTab implements TabContents {
public documentNodeId = 1; public documentNodeId = 1;
public queriedNodeId = 2; public queriedNodeId = 2;
public backgroundThrottlingAllowed = true; public backgroundThrottlingAllowed = true;
private nextPixelCapturePreparationId = 0;
public constructor( public constructor(
public readonly id: number, public readonly id: number,
@@ -116,12 +128,35 @@ class FakeTab implements TabContents {
public async capturePage(options?: { stayHidden?: boolean }): Promise<TabImage> { public async capturePage(options?: { stayHidden?: boolean }): Promise<TabImage> {
this.capturedViewports.push(options ?? {}); this.capturedViewports.push(options ?? {});
this.actions.push("capture"); this.actions.push("capture");
this.resolveCaptureStartWaiters();
if (this.captureThrows) {
throw new Error("capture failed");
}
if (this.captureNeverPaints) { if (this.captureNeverPaints) {
return new Promise<never>(() => {}); return new Promise<never>(() => {});
} }
if (this.deferCaptures) {
return new Promise<TabImage>((resolve) => {
this.deferredCaptures.push(resolve);
});
}
return new FakeImage(); return new FakeImage();
} }
public async prepareForPixelCapture(): Promise<TabPixelCapturePreparation> {
this.actions.push("prepare");
if (this.prepareNeverAcks) {
return new Promise<never>(() => {});
}
const token = `capture-${++this.nextPixelCapturePreparationId}`;
return { token };
}
public async restorePixelCapture(preparation: TabPixelCapturePreparation): Promise<void> {
this.actions.push(`restore:${preparation.token}`);
this.restoredPixelCaptureTokens.push(preparation.token);
}
public invalidate(): void { public invalidate(): void {
this.actions.push("invalidate"); this.actions.push("invalidate");
} }
@@ -143,6 +178,7 @@ class FakeTab implements TabContents {
command: string, command: string,
params?: Record<string, unknown>, params?: Record<string, unknown>,
): Promise<unknown> { ): Promise<unknown> {
this.actions.push(`debug:${command}`);
this.debugCommands.push({ command, ...(params ? { params } : {}) }); this.debugCommands.push({ command, ...(params ? { params } : {}) });
if (command === "Page.getLayoutMetrics") { if (command === "Page.getLayoutMetrics") {
return this.layoutMetrics; return this.layoutMetrics;
@@ -158,6 +194,33 @@ class FakeTab implements TabContents {
} }
return {}; return {};
} }
public waitForCaptureStart(count: number): Promise<void> {
if (this.capturedViewports.length >= count) {
return Promise.resolve();
}
return new Promise((resolve) => {
this.captureStartWaiters.push(() => {
if (this.capturedViewports.length >= count) {
resolve();
}
});
});
}
public finishNextCapture(): void {
const resolve = this.deferredCaptures.shift();
if (!resolve) {
throw new Error("No deferred capture is waiting");
}
resolve(new FakeImage());
}
private resolveCaptureStartWaiters(): void {
for (const waiter of this.captureStartWaiters.splice(0)) {
waiter();
}
}
} }
class FakeRegistry implements BrowserRegistry { class FakeRegistry implements BrowserRegistry {
@@ -852,9 +915,11 @@ describe("executeAutomationCommand", () => {
}); });
expect(browser.tab.capturedViewports).toEqual([{ stayHidden: false }]); expect(browser.tab.capturedViewports).toEqual([{ stayHidden: false }]);
expect(browser.tab.actions).toEqual([ expect(browser.tab.actions).toEqual([
"prepare",
"background:false", "background:false",
"invalidate", "invalidate",
"capture", "capture",
"restore:capture-1",
"background:true", "background:true",
]); ]);
}); });
@@ -882,9 +947,11 @@ describe("executeAutomationCommand", () => {
}, },
}); });
expect(browser.tab.actions).toEqual([ expect(browser.tab.actions).toEqual([
"prepare",
"background:false", "background:false",
"invalidate", "invalidate",
"capture", "capture",
"restore:capture-1",
"background:true", "background:true",
]); ]);
} finally { } finally {
@@ -892,6 +959,111 @@ describe("executeAutomationCommand", () => {
} }
}); });
test("screenshot restores capture preparation when viewport capture fails", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.captureThrows = true;
await expect(
browser.execute({
command: "screenshot",
args: { browserId: BROWSER_A },
}),
).rejects.toThrow("capture failed");
expect(browser.tab.actions).toEqual([
"prepare",
"background:false",
"invalidate",
"capture",
"restore:capture-1",
"background:true",
]);
});
test("screenshot returns no-frame when capture preparation does not ack", async () => {
vi.useFakeTimers();
try {
const browser = new BrowserAutomationHarness();
browser.tab.prepareNeverAcks = true;
const resultPromise = browser.execute({
command: "screenshot",
args: { browserId: BROWSER_A },
});
await vi.advanceTimersByTimeAsync(5_000);
await expect(resultPromise).resolves.toEqual({
requestId: "req-screenshot",
ok: false,
error: {
code: "screenshot_no_frame",
message:
"The browser tab has no painted frame. Focus the tab in the app, then try again.",
retryable: false,
},
});
expect(browser.tab.actions).toEqual(["prepare", "background:true"]);
} finally {
vi.useRealTimers();
}
});
test("overlapping screenshots serialize capture preparation and restore", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.deferCaptures = true;
const first = browser.execute({
command: "screenshot",
args: { browserId: BROWSER_A },
});
await browser.tab.waitForCaptureStart(1);
const second = browser.execute(
{
command: "screenshot",
args: { browserId: BROWSER_A },
},
{ requestId: "req-screenshot-2" },
);
await Promise.resolve();
expect(browser.tab.actions).toEqual(["prepare", "background:false", "invalidate", "capture"]);
browser.tab.finishNextCapture();
await browser.tab.waitForCaptureStart(2);
expect(browser.tab.actions).toEqual([
"prepare",
"background:false",
"invalidate",
"capture",
"restore:capture-1",
"background:true",
"prepare",
"background:false",
"invalidate",
"capture",
]);
browser.tab.finishNextCapture();
await expect(first).resolves.toMatchObject({ requestId: "req-screenshot", ok: true });
await expect(second).resolves.toMatchObject({ requestId: "req-screenshot-2", ok: true });
expect(browser.tab.actions).toEqual([
"prepare",
"background:false",
"invalidate",
"capture",
"restore:capture-1",
"background:true",
"prepare",
"background:false",
"invalidate",
"capture",
"restore:capture-2",
"background:true",
]);
});
test("screenshot with fullPage captures the page content area through CDP", async () => { test("screenshot with fullPage captures the page content area through CDP", async () => {
const browser = new BrowserAutomationHarness(); const browser = new BrowserAutomationHarness();
@@ -923,6 +1095,44 @@ describe("executeAutomationCommand", () => {
}, },
}, },
]); ]);
expect(browser.tab.actions).toEqual([
"prepare",
"background:false",
"invalidate",
"debug:Page.getLayoutMetrics",
"debug:Page.captureScreenshot",
"restore:capture-1",
"background:true",
]);
});
test("screenshot with fullPage restores capture preparation when CDP returns no image", async () => {
const browser = new BrowserAutomationHarness();
browser.tab.fullPageScreenshotData = "";
const result = await browser.execute({
command: "screenshot",
args: { browserId: BROWSER_A, fullPage: true },
});
expect(result).toEqual({
requestId: "req-screenshot",
ok: false,
error: {
code: "browser_unsupported",
message: "browser_screenshot fullPage returned no data",
retryable: false,
},
});
expect(browser.tab.actions).toEqual([
"prepare",
"background:false",
"invalidate",
"debug:Page.getLayoutMetrics",
"debug:Page.captureScreenshot",
"restore:capture-1",
"background:true",
]);
}); });
test("upload resolves workspace files before setting them on the file input", async () => { test("upload resolves workspace files before setting them on the file input", async () => {

View File

@@ -24,6 +24,8 @@ export interface TabContents {
goForward(): void; goForward(): void;
reload(): void; reload(): void;
capturePage(options?: TabCapturePageOptions): Promise<TabImage>; capturePage(options?: TabCapturePageOptions): Promise<TabImage>;
prepareForPixelCapture(): Promise<TabPixelCapturePreparation>;
restorePixelCapture(preparation: TabPixelCapturePreparation): Promise<void>;
invalidate(): void; invalidate(): void;
isBackgroundThrottlingAllowed(): boolean; isBackgroundThrottlingAllowed(): boolean;
setBackgroundThrottling(allowed: boolean): void; setBackgroundThrottling(allowed: boolean): void;
@@ -31,6 +33,10 @@ export interface TabContents {
sendDebugCommand?(command: string, params?: Record<string, unknown>): Promise<unknown>; sendDebugCommand?(command: string, params?: Record<string, unknown>): Promise<unknown>;
} }
export interface TabPixelCapturePreparation {
token: string;
}
export interface TabImage { export interface TabImage {
toPNG(): Uint8Array; toPNG(): Uint8Array;
getSize(): { width: number; height: number }; getSize(): { width: number; height: number };
@@ -124,19 +130,56 @@ async function runSerializedPixelCapture<T>(
} }
} }
async function capturePaintedViewport(contents: TabContents): Promise<TabImage> { async function prepareForPixelCapture(contents: TabContents): Promise<TabPixelCapturePreparation> {
try {
return await withPixelCaptureTimeout(contents.prepareForPixelCapture());
} catch {
throw new ScreenshotNoFrameError();
}
}
async function restorePixelCapture(
contents: TabContents,
preparation: TabPixelCapturePreparation,
): Promise<void> {
try {
await withPixelCaptureTimeout(contents.restorePixelCapture(preparation));
} catch {
throw new ScreenshotNoFrameError();
}
}
async function runPreparedPixelCapture<T>(
contents: TabContents,
capture: () => Promise<T>,
): Promise<T> {
return runSerializedPixelCapture(contents, async () => { return runSerializedPixelCapture(contents, async () => {
const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed(); const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed();
contents.setBackgroundThrottling(false); let preparation: TabPixelCapturePreparation | null = null;
try { try {
// Offscreen-parked webview guests have no compositor surface for
// capturePage/CDP to copy. The renderer must briefly make the host
// paintable before capture; see ~/.paseo/plans/browser-capture-harness-results.md.
preparation = await prepareForPixelCapture(contents);
contents.setBackgroundThrottling(false);
contents.invalidate(); contents.invalidate();
return await withPixelCaptureTimeout(contents.capturePage({ stayHidden: false })); return await withPixelCaptureTimeout(capture());
} finally { } finally {
contents.setBackgroundThrottling(previousBackgroundThrottling); try {
if (preparation) {
await restorePixelCapture(contents, preparation);
}
} finally {
contents.setBackgroundThrottling(previousBackgroundThrottling);
}
} }
}); });
} }
async function capturePaintedViewport(contents: TabContents): Promise<TabImage> {
return runPreparedPixelCapture(contents, () => contents.capturePage({ stayHidden: false }));
}
function tabInfoFromContents( function tabInfoFromContents(
browserId: string, browserId: string,
contents: TabContents, contents: TabContents,
@@ -860,18 +903,21 @@ async function executeFullPageScreenshot(
if (!target.contents.sendDebugCommand) { if (!target.contents.sendDebugCommand) {
return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP"); return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP");
} }
const metrics = await getCdpLayoutMetrics(target.contents); const sendDebugCommand = target.contents.sendDebugCommand.bind(target.contents);
const width = metrics.contentWidth;
const height = metrics.contentHeight;
let screenshot: CdpCaptureScreenshotResult; let screenshot: CdpCaptureScreenshotResult;
let width = 0;
let height = 0;
try { try {
screenshot = (await withPixelCaptureTimeout( screenshot = await runPreparedPixelCapture(target.contents, async () => {
target.contents.sendDebugCommand("Page.captureScreenshot", { const metrics = await getCdpLayoutMetrics(target.contents);
width = metrics.contentWidth;
height = metrics.contentHeight;
return (await sendDebugCommand("Page.captureScreenshot", {
format: "png", format: "png",
captureBeyondViewport: true, captureBeyondViewport: true,
clip: { x: 0, y: 0, width, height, scale: 1 }, clip: { x: 0, y: 0, width, height, scale: 1 },
}), })) as CdpCaptureScreenshotResult;
)) as CdpCaptureScreenshotResult; });
} catch (error) { } catch (error) {
if (isScreenshotNoFrameError(error)) { if (isScreenshotNoFrameError(error)) {
return screenshotNoFrameFailure(requestId); return screenshotNoFrameFailure(requestId);

View File

@@ -1,6 +1,81 @@
import { contextBridge, ipcRenderer, webUtils } from "electron"; import { contextBridge, ipcRenderer, webUtils } from "electron";
type EventHandler = (payload: unknown) => void; type EventHandler = (payload: unknown) => void;
type BrowserPixelCapturePrepareHandler = (input: {
browserId: string;
}) => Promise<{ token: string }>;
type BrowserPixelCaptureRestoreHandler = (input: { token: string }) => Promise<void>;
let prepareForPixelCaptureHandler: BrowserPixelCapturePrepareHandler | null = null;
let restorePixelCaptureHandler: BrowserPixelCaptureRestoreHandler | null = null;
function readStringField(payload: unknown, key: string): string | null {
if (!isRecord(payload)) {
return null;
}
const value = payload[key];
return typeof value === "string" && value.length > 0 ? value : null;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
ipcRenderer.on("paseo:browser:capture-prepare", async (_event, payload: unknown) => {
const requestId = readStringField(payload, "requestId");
const browserId = readStringField(payload, "browserId");
if (!requestId || !browserId || !prepareForPixelCaptureHandler) {
ipcRenderer.send("paseo:browser:capture-prepared", {
requestId: requestId ?? "unknown",
ok: false,
message: "Browser pixel capture preparation is unavailable.",
});
return;
}
try {
const preparation = await prepareForPixelCaptureHandler({ browserId });
ipcRenderer.send("paseo:browser:capture-prepared", {
requestId,
ok: true,
token: preparation.token,
});
} catch (error) {
ipcRenderer.send("paseo:browser:capture-prepared", {
requestId,
ok: false,
message: errorMessage(error),
});
}
});
ipcRenderer.on("paseo:browser:capture-restore", async (_event, payload: unknown) => {
const requestId = readStringField(payload, "requestId");
const token = readStringField(payload, "token");
if (!requestId || !token || !restorePixelCaptureHandler) {
ipcRenderer.send("paseo:browser:capture-restored", {
requestId: requestId ?? "unknown",
ok: false,
message: "Browser pixel capture restore is unavailable.",
});
return;
}
try {
await restorePixelCaptureHandler({ token });
ipcRenderer.send("paseo:browser:capture-restored", { requestId, ok: true });
} catch (error) {
ipcRenderer.send("paseo:browser:capture-restored", {
requestId,
ok: false,
message: errorMessage(error),
});
}
});
contextBridge.exposeInMainWorld("paseoDesktop", { contextBridge.exposeInMainWorld("paseoDesktop", {
platform: process.platform, platform: process.platform,
@@ -90,5 +165,21 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect), ) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect),
copyElement: (payload: { text?: string; imageDataUrl?: string }) => copyElement: (payload: { text?: string; imageDataUrl?: string }) =>
ipcRenderer.invoke("paseo:browser:copy-element", payload), ipcRenderer.invoke("paseo:browser:copy-element", payload),
onPrepareForPixelCapture: (handler: BrowserPixelCapturePrepareHandler): (() => void) => {
prepareForPixelCaptureHandler = handler;
return () => {
if (prepareForPixelCaptureHandler === handler) {
prepareForPixelCaptureHandler = null;
}
};
},
onRestorePixelCapture: (handler: BrowserPixelCaptureRestoreHandler): (() => void) => {
restorePixelCaptureHandler = handler;
return () => {
if (restorePixelCaptureHandler === handler) {
restorePixelCaptureHandler = null;
}
};
},
}, },
}); });