diff --git a/CLAUDE.md b/CLAUDE.md index 0dac6f101..ae9cab8fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,34 +21,35 @@ This is an npm workspace monorepo: At the start of non-trivial work, list `docs/` and skim anything relevant to the task. When you learn something meta worth preserving — a gotcha, a convention, a workflow, a piece of system context that will outlive the current task — update an existing doc or propose a new one. Code-level facts belong in inline comments next to the code; system, process, and gotcha-level facts belong in `docs/`. -| Doc | What's in it | -| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going | -| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow | -| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track | -| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations | -| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms | -| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization | -| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons | -| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it | -| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order | -| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash | -| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas | -| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer | -| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end | -| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries | -| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config | -| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP | -| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs | -| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage | -| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization | -| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows | -| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness | -| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows | -| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security | -| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist | -| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider | -| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth | +| Doc | What's in it | +| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ | +| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going | +| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow | +| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track | +| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations | +| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms | +| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization | +| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons | +| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it | +| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order | +| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash | +| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas | +| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer | +| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end | +| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries | +| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config | +| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP | +| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs | +| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage | +| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization | +| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows | +| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness | +| [docs/browser-capture-harness.md](docs/browser-capture-harness.md) | Real-Electron browser screenshot harness and compositor-surface gotcha | +| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows | +| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security | +| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist | +| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider | +| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth | ## Quick start diff --git a/docs/browser-capture-harness.md b/docs/browser-capture-harness.md new file mode 100644 index 000000000..9e768c4c5 --- /dev/null +++ b/docs/browser-capture-harness.md @@ -0,0 +1,39 @@ +# Browser Capture Harness + +The desktop capture harness is the real-Electron verification path for browser screenshots. +It validates the compositor behavior that unit tests cannot see: + +- the resident automation `` starts in the production parking state; +- the parked guest has no copyable viewport frame; +- the resident webview guest is sized to 1280x800 logical pixels; +- the app-style prep sequence moves the host to a paintable 1x1 clipped state, waits two animation frames and a layout read, then both viewport `capturePage` and full-page CDP screenshots return real pixels; +- restore returns the host to offscreen parking after every capture. + +Run it with the repo Electron: + +```bash +npm run capture-harness --workspace=@getpaseo/desktop +``` + +The harness writes PNG evidence and `results.json` to: + +```text +packages/desktop/capture-harness/out/ +``` + +A passing run prints `PASS` lines for guest sizing, the expected parked-capture failure, +five viewport prep captures, five full-page prep captures, and final completion. The PNG +sizes may be device-pixel scaled; on a Retina display the 1280x800 logical viewport is +usually saved as 2560x1600. + +## Mechanism + +Electron captures copy from the guest web contents' compositor surface. A resident +webview parked at `left:-20000px` and `opacity:0` does not have a copyable surface, and +`capturePage({ stayHidden:false })` or CDP `Page.captureScreenshot` cannot rescue it. + +Before pixel capture, the app renderer temporarily makes the resident host paintable: +`left:0`, `top:0`, `opacity:1`, `pointer-events:none`, host size `1x1`, and +`overflow:hidden`, with the full-size 1280x800 webview inside. Main captures only after +the renderer acknowledges two animation frames plus a `getBoundingClientRect()` read, and +the renderer restores parking in a `finally`. diff --git a/packages/app/src/browser-automation/handler.ts b/packages/app/src/browser-automation/handler.ts index a2ea259e9..00fb05dfc 100644 --- a/packages/app/src/browser-automation/handler.ts +++ b/packages/app/src/browser-automation/handler.ts @@ -1,6 +1,9 @@ import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host"; -import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident"; +import { + ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault, + installResidentBrowserCaptureBridge, +} from "@/components/browser-webview-resident"; import { createWorkspaceBrowser } from "@/stores/browser-store"; import { buildWorkspaceTabPersistenceKey, @@ -40,7 +43,8 @@ export function mountBrowserAutomationHandler( options: BrowserAutomationHandlerOptions, ): () => void { const getHost = options.getHost ?? getDesktopHost; - return options.client.on("browser.automation.execute.request", (request) => { + const uninstallCaptureBridge = installResidentBrowserCaptureBridge(); + const unsubscribe = options.client.on("browser.automation.execute.request", (request) => { void handleBrowserAutomationRequest({ client: options.client, getHost, @@ -56,6 +60,10 @@ export function mountBrowserAutomationHandler( : {}), }); }); + return () => { + unsubscribe(); + uninstallCaptureBridge(); + }; } export function mountBrowserAutomationDaemonClientHandler( diff --git a/packages/app/src/components/browser-webview-resident.browser.test.ts b/packages/app/src/components/browser-webview-resident.browser.test.ts index 48ab1a8c4..015c533a6 100644 --- a/packages/app/src/components/browser-webview-resident.browser.test.ts +++ b/packages/app/src/components/browser-webview-resident.browser.test.ts @@ -2,8 +2,10 @@ import { afterEach, describe, expect, it } from "vitest"; import { clearResidentBrowserWebviewsForTests, ensureResidentBrowserWebview, + prepareResidentBrowserWebviewForPixelCapture, releaseResidentBrowserWebview, removeResidentBrowserWebview, + restoreResidentBrowserWebviewAfterPixelCapture, takeResidentBrowserWebview, } from "./browser-webview-resident"; @@ -22,6 +24,7 @@ describe("resident browser webviews", () => { expect(host.children).toHaveLength(0); expect(webview.isConnected).toBe(true); + expect(webview.style.display).toBe("inline-flex"); expect(webview.style.width).toBe("1280px"); expect(webview.style.height).toBe("800px"); @@ -57,4 +60,65 @@ describe("resident browser webviews", () => { expect(webview?.isConnected).toBe(false); expect(takeResidentBrowserWebview("browser-closed")).toBeNull(); }); + + it("temporarily makes resident webviews paintable for pixel capture", async () => { + const webview = ensureResidentBrowserWebview({ + browserId: "browser-capture", + url: "https://example.com", + }); + if (!webview) { + throw new Error("Expected resident browser webview"); + } + + const preparation = await prepareResidentBrowserWebviewForPixelCapture({ + browserId: "browser-capture", + }); + const host = document.getElementById("paseo-browser-resident-webviews"); + + expect(preparation.token).toBe("capture-1"); + expect(host?.style.left).toBe("0px"); + expect(host?.style.top).toBe("0px"); + expect(host?.style.width).toBe("1px"); + expect(host?.style.height).toBe("1px"); + expect(host?.style.overflow).toBe("hidden"); + expect(host?.style.opacity).toBe("1"); + expect(host?.style.pointerEvents).toBe("none"); + expect(webview.style.display).toBe("inline-flex"); + expect(webview.style.width).toBe("1280px"); + expect(webview.style.height).toBe("800px"); + + await restoreResidentBrowserWebviewAfterPixelCapture(preparation); + + expect(host?.style.left).toBe("-20000px"); + expect(host?.style.width).toBe("1280px"); + expect(host?.style.height).toBe("800px"); + expect(host?.style.opacity).toBe("0"); + }); + + it("keeps the resident host paintable until every capture token is restored", async () => { + ensureResidentBrowserWebview({ + browserId: "browser-overlap", + url: "https://example.com", + }); + + const first = await prepareResidentBrowserWebviewForPixelCapture({ + browserId: "browser-overlap", + }); + const second = await prepareResidentBrowserWebviewForPixelCapture({ + browserId: "browser-overlap", + }); + const host = document.getElementById("paseo-browser-resident-webviews"); + + await restoreResidentBrowserWebviewAfterPixelCapture(first); + + expect(host?.style.left).toBe("0px"); + expect(host?.style.width).toBe("1px"); + expect(host?.style.opacity).toBe("1"); + + await restoreResidentBrowserWebviewAfterPixelCapture(second); + + expect(host?.style.left).toBe("-20000px"); + expect(host?.style.width).toBe("1280px"); + expect(host?.style.opacity).toBe("0"); + }); }); diff --git a/packages/app/src/components/browser-webview-resident.ts b/packages/app/src/components/browser-webview-resident.ts index a33ff8c69..dba212088 100644 --- a/packages/app/src/components/browser-webview-resident.ts +++ b/packages/app/src/components/browser-webview-resident.ts @@ -1,9 +1,16 @@ +import { getDesktopHost } from "@/desktop/host"; + const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews"; const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id"; const RESIDENT_VIEWPORT_WIDTH = 1280; const RESIDENT_VIEWPORT_HEIGHT = 800; const residentWebviewsByBrowserId = new Map(); +const activeCapturePreparations = new Map(); + +let captureBridgeInstallCount = 0; +let captureBridgeDisposer: (() => void) | null = null; +let nextCapturePreparationId = 0; interface BrowserWebviewElement extends HTMLElement { src: string; @@ -21,14 +28,7 @@ function readDocument(): Document | null { return typeof document === "undefined" ? null : document; } -function getResidentBrowserHost(ownerDocument: Document): HTMLElement { - const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID); - if (existing) { - return existing; - } - - const host = ownerDocument.createElement("div"); - host.id = RESIDENT_BROWSER_HOST_ID; +function applyResidentHostParkingStyle(host: HTMLElement): void { host.setAttribute("aria-hidden", "true"); host.style.position = "fixed"; host.style.left = "-20000px"; @@ -38,27 +38,88 @@ function getResidentBrowserHost(ownerDocument: Document): HTMLElement { host.style.overflow = "hidden"; host.style.opacity = "0"; host.style.pointerEvents = "none"; + host.style.zIndex = ""; + host.style.clipPath = ""; + host.style.visibility = ""; + host.style.transform = ""; +} + +function applyResidentHostCaptureStyle(host: HTMLElement): void { + host.setAttribute("aria-hidden", "true"); + host.style.position = "fixed"; + host.style.left = "0"; + host.style.top = "0"; + host.style.width = "1px"; + host.style.height = "1px"; + host.style.overflow = "hidden"; + host.style.opacity = "1"; + host.style.pointerEvents = "none"; + host.style.zIndex = "1"; + host.style.clipPath = ""; + host.style.visibility = ""; + host.style.transform = ""; +} + +function getResidentBrowserHost(ownerDocument: Document): HTMLElement { + const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID); + if (existing) { + return existing; + } + + const host = ownerDocument.createElement("div"); + host.id = RESIDENT_BROWSER_HOST_ID; + applyResidentHostParkingStyle(host); ownerDocument.body.appendChild(host); return host; } function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null { for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) { + if (!(element instanceof HTMLElement)) { + continue; + } if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) { - return element as HTMLElement; + return element; } } return null; } function applyResidentWebviewStyle(webview: HTMLElement): void { - webview.style.display = "block"; + webview.style.display = "inline-flex"; + webview.style.flex = "0 0 auto"; webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`; webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`; webview.style.border = "0"; webview.style.background = "transparent"; } +function hasActiveResidentHostPreparation(): boolean { + for (const preparation of activeCapturePreparations.values()) { + if (preparation.preparesResidentHost) { + return true; + } + } + return false; +} + +function nextAnimationFrame(): Promise { + if (typeof requestAnimationFrame !== "function") { + return Promise.resolve(); + } + return new Promise((resolve) => { + requestAnimationFrame(() => { + resolve(); + }); + }); +} + +async function waitForCapturePaint(webview: HTMLElement): Promise { + await nextAnimationFrame(); + await nextAnimationFrame(); + webview.getBoundingClientRect(); +} + export function prepareBrowserWebview( webview: HTMLElement, input: { browserId: string; initialUrl?: string | null }, @@ -133,6 +194,92 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl getResidentBrowserHost(ownerDocument).appendChild(webview); } +export async function prepareResidentBrowserWebviewForPixelCapture(input: { + browserId: string; +}): Promise<{ token: string }> { + const browserId = trimNonEmpty(input.browserId); + if (!browserId) { + throw new Error("Browser id is required for pixel capture preparation."); + } + const ownerDocument = readDocument(); + if (!ownerDocument) { + throw new Error("Browser pixel capture preparation requires a document."); + } + + const host = getResidentBrowserHost(ownerDocument); + const webview = findBrowserWebview(browserId, ownerDocument); + if (!webview) { + throw new Error(`Browser webview ${browserId} is not mounted.`); + } + + const token = `capture-${++nextCapturePreparationId}`; + const preparesResidentHost = webview.parentElement === host; + activeCapturePreparations.set(token, { preparesResidentHost }); + try { + if (preparesResidentHost) { + applyResidentHostCaptureStyle(host); + applyResidentWebviewStyle(webview); + } + await waitForCapturePaint(webview); + return { token }; + } catch (error) { + activeCapturePreparations.delete(token); + if (!hasActiveResidentHostPreparation()) { + applyResidentHostParkingStyle(host); + } + throw error; + } +} + +export async function restoreResidentBrowserWebviewAfterPixelCapture(input: { + token: string; +}): Promise { + const preparation = activeCapturePreparations.get(input.token); + if (!preparation) { + return; + } + + activeCapturePreparations.delete(input.token); + if (!preparation.preparesResidentHost || hasActiveResidentHostPreparation()) { + return; + } + + const ownerDocument = readDocument(); + if (!ownerDocument) { + return; + } + const host = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID); + if (host instanceof HTMLElement) { + applyResidentHostParkingStyle(host); + } +} + +export function installResidentBrowserCaptureBridge(): () => void { + captureBridgeInstallCount += 1; + if (!captureBridgeDisposer) { + const browserBridge = getDesktopHost()?.browser; + const disposePrepare = browserBridge?.onPrepareForPixelCapture?.( + prepareResidentBrowserWebviewForPixelCapture, + ); + const disposeRestore = browserBridge?.onRestorePixelCapture?.( + restoreResidentBrowserWebviewAfterPixelCapture, + ); + captureBridgeDisposer = () => { + disposePrepare?.(); + disposeRestore?.(); + }; + } + + return () => { + captureBridgeInstallCount = Math.max(0, captureBridgeInstallCount - 1); + if (captureBridgeInstallCount > 0) { + return; + } + captureBridgeDisposer?.(); + captureBridgeDisposer = null; + }; +} + export function removeResidentBrowserWebview(browserId: string): void { const normalizedBrowserId = trimNonEmpty(browserId); if (!normalizedBrowserId) { @@ -149,5 +296,7 @@ export function clearResidentBrowserWebviewsForTests(): void { webview.remove(); } residentWebviewsByBrowserId.clear(); + activeCapturePreparations.clear(); + nextCapturePreparationId = 0; readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove(); } diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index 2380fcc81..1d57468c0 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -121,6 +121,10 @@ export interface DesktopBrowserShortcutEvent { action: "focus-url"; } +export interface DesktopBrowserPixelCapturePreparation { + token: string; +} + export interface DesktopBrowserNewTabRequestEvent { sourceBrowserId: string; url: string; @@ -144,6 +148,12 @@ export interface DesktopBrowserBridge { ) => Promise; /** Copy element text and/or an image to the system clipboard from main. */ copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise; + onPrepareForPixelCapture?: ( + handler: (input: { browserId: string }) => Promise, + ) => () => void; + onRestorePixelCapture?: ( + handler: (input: DesktopBrowserPixelCapturePreparation) => Promise, + ) => () => void; } export interface DesktopInvokeBridge { diff --git a/packages/desktop/capture-harness/.gitignore b/packages/desktop/capture-harness/.gitignore new file mode 100644 index 000000000..c4fdaa757 --- /dev/null +++ b/packages/desktop/capture-harness/.gitignore @@ -0,0 +1,2 @@ +out/ +fatal-error.txt diff --git a/packages/desktop/capture-harness/bright.html b/packages/desktop/capture-harness/bright.html new file mode 100644 index 000000000..9e37a4f5a --- /dev/null +++ b/packages/desktop/capture-harness/bright.html @@ -0,0 +1,74 @@ + + + + + Paseo Capture Harness Target + + + +
PASEO CAPTURE HARNESS
+
LOCAL MAGENTA TARGET 1280x800 VIEWPORT
+
+
FULL PAGE MARKER AT Y=1370
+ + diff --git a/packages/desktop/capture-harness/index.html b/packages/desktop/capture-harness/index.html new file mode 100644 index 000000000..ef5a96f82 --- /dev/null +++ b/packages/desktop/capture-harness/index.html @@ -0,0 +1,166 @@ + + + + + Paseo Browser Capture Harness + + + + + + + + diff --git a/packages/desktop/capture-harness/main.js b/packages/desktop/capture-harness/main.js new file mode 100644 index 000000000..69067e9c6 --- /dev/null +++ b/packages/desktop/capture-harness/main.js @@ -0,0 +1,378 @@ +const fs = require("node:fs"); +const fsp = require("node:fs/promises"); +const path = require("node:path"); +const { app, BrowserWindow, nativeImage } = require("electron"); + +const ROOT = __dirname; +const OUT_DIR = process.env.PASEO_CAPTURE_HARNESS_OUT_DIR || path.join(ROOT, "out"); +const VIEWPORT_WIDTH = 1280; +const VIEWPORT_HEIGHT = 800; +const FULL_PAGE_HEIGHT = 1600; +const CAPTURE_TIMEOUT_MS = 5000; +const REPEAT_COUNT = 5; + +function fileUrl(filePath) { + return new URL(`file://${filePath}`).toString(); +} + +function ensureDirSync(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function withTimeout(promise, label) { + let timeoutId; + const timeout = new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(`${label} timed out after ${CAPTURE_TIMEOUT_MS}ms`)); + }, CAPTURE_TIMEOUT_MS); + }); + return Promise.race([promise, timeout]).finally(() => { + clearTimeout(timeoutId); + }); +} + +function pixelOffset(width, x, y) { + return (y * width + x) * 4; +} + +function isBrightMagenta(bitmap, offset) { + const c0 = bitmap[offset]; + const c1 = bitmap[offset + 1]; + const c2 = bitmap[offset + 2]; + return c0 > 200 && c1 < 90 && c2 > 200; +} + +function analyzeImage(image, expected, guestMetrics) { + if (!image || image.isEmpty()) { + return { + width: 0, + height: 0, + brightRatio: 0, + textNonUniform: false, + pass: false, + }; + } + + const size = image.getSize(); + const width = size.width; + const height = size.height; + const bitmap = image.toBitmap(); + const totalPixels = width * height; + let brightPixels = 0; + for (let offset = 0; offset < bitmap.length; offset += 4) { + if (isBrightMagenta(bitmap, offset)) { + brightPixels += 1; + } + } + + const crop = { + left: Math.min(40, Math.max(0, width - 1)), + top: Math.min(40, Math.max(0, height - 1)), + right: Math.min(width, 940), + bottom: Math.min(height, 260), + }; + let cropPixels = 0; + let cropNonBright = 0; + let luminanceSum = 0; + let luminanceSqSum = 0; + const quantized = new Set(); + for (let y = crop.top; y < crop.bottom; y += 1) { + for (let x = crop.left; x < crop.right; x += 1) { + const offset = pixelOffset(width, x, y); + cropPixels += 1; + if (!isBrightMagenta(bitmap, offset)) { + cropNonBright += 1; + } + const r = bitmap[offset + 2]; + const g = bitmap[offset + 1]; + const b = bitmap[offset]; + const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b; + luminanceSum += luma; + luminanceSqSum += luma * luma; + quantized.add(`${r >> 5},${g >> 5},${b >> 5},${bitmap[offset + 3] >> 6}`); + } + } + + const devicePixelRatio = + typeof guestMetrics.devicePixelRatio === "number" && guestMetrics.devicePixelRatio > 0 + ? guestMetrics.devicePixelRatio + : 1; + const sizeTargets = [ + { width: expected.width, height: expected.height }, + { + width: Math.round(expected.width * devicePixelRatio), + height: Math.round(expected.height * devicePixelRatio), + }, + ]; + const matchedSize = sizeTargets.some( + (target) => Math.abs(width - target.width) <= 2 && Math.abs(height - target.height) <= 2, + ); + const luminanceMean = cropPixels ? luminanceSum / cropPixels : 0; + const luminanceVariance = cropPixels + ? luminanceSqSum / cropPixels - luminanceMean * luminanceMean + : 0; + const brightRatio = totalPixels ? brightPixels / totalPixels : 0; + const textNonUniform = + cropPixels > 0 && + cropNonBright / cropPixels > 0.02 && + quantized.size >= 4 && + luminanceVariance > 100; + + return { + width, + height, + logicalWidthAtDpr: width / devicePixelRatio, + logicalHeightAtDpr: height / devicePixelRatio, + brightRatio, + textNonUniform, + matchedSize, + pass: matchedSize && brightRatio >= expected.minBrightRatio && textNonUniform, + }; +} + +function pass(message) { + console.log(`PASS ${message}`); +} + +function fail(message) { + console.log(`FAIL ${message}`); + throw new Error(message); +} + +async function saveImage(image, outputPath) { + await fsp.writeFile(outputPath, image.toPNG()); +} + +async function waitForGuestLoad(contents) { + await new Promise((resolve) => { + if (!contents.isLoading()) { + resolve(); + return; + } + contents.once("did-finish-load", resolve); + contents.once("did-fail-load", resolve); + }); + await new Promise((resolve) => setTimeout(resolve, 500)); +} + +async function renderer(win, expression) { + return await win.webContents.executeJavaScript(expression, true); +} + +async function readGuestMetrics(contents) { + return await contents.executeJavaScript( + `({ + innerWidth: window.innerWidth, + innerHeight: window.innerHeight, + devicePixelRatio: window.devicePixelRatio, + documentClientWidth: document.documentElement.clientWidth, + documentClientHeight: document.documentElement.clientHeight, + scrollWidth: document.documentElement.scrollWidth, + scrollHeight: document.documentElement.scrollHeight, + visualViewport: window.visualViewport ? { + width: window.visualViewport.width, + height: window.visualViewport.height, + scale: window.visualViewport.scale + } : null + })`, + true, + ); +} + +async function capturePageSequence(contents) { + const previousBackgroundThrottling = contents.getBackgroundThrottling(); + contents.setBackgroundThrottling(false); + try { + contents.invalidate(); + return await withTimeout(contents.capturePage(undefined, { stayHidden: false }), "capturePage"); + } finally { + contents.setBackgroundThrottling(previousBackgroundThrottling); + } +} + +async function captureFullPage(contents) { + let attachedHere = false; + if (!contents.debugger.isAttached()) { + contents.debugger.attach("1.3"); + attachedHere = true; + } + try { + const metrics = await contents.debugger.sendCommand("Page.getLayoutMetrics"); + const contentSize = metrics.cssContentSize || + metrics.contentSize || { + x: 0, + y: 0, + width: VIEWPORT_WIDTH, + height: FULL_PAGE_HEIGHT, + }; + const clip = { + x: Math.floor(contentSize.x || 0), + y: Math.floor(contentSize.y || 0), + width: Math.ceil(contentSize.width || VIEWPORT_WIDTH), + height: Math.ceil(contentSize.height || FULL_PAGE_HEIGHT), + scale: 1, + }; + const result = await withTimeout( + contents.debugger.sendCommand("Page.captureScreenshot", { + format: "png", + captureBeyondViewport: true, + clip, + }), + "CDP Page.captureScreenshot", + ); + return nativeImage.createFromBuffer(Buffer.from(result.data, "base64")); + } finally { + if (attachedHere && contents.debugger.isAttached()) { + contents.debugger.detach(); + } + } +} + +async function captureWithPrep({ win, contents, mode, repeatIndex, guestMetrics }) { + const preparation = await renderer(win, "window.captureHarness.prepareForPixelCapture()"); + const outputPath = path.join(OUT_DIR, `${mode}-prep-${repeatIndex}.png`); + try { + const image = + mode === "viewport" ? await capturePageSequence(contents) : await captureFullPage(contents); + await saveImage(image, outputPath); + const expected = + mode === "viewport" + ? { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 } + : { width: VIEWPORT_WIDTH, height: FULL_PAGE_HEIGHT, minBrightRatio: 0.55 }; + const analysis = analyzeImage(image, expected, guestMetrics); + const size = `${analysis.width}x${analysis.height}`; + const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`; + const bright = analysis.brightRatio.toFixed(4); + if (!analysis.pass) { + fail( + `${mode} prep ${repeatIndex}/${REPEAT_COUNT} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`, + ); + } + pass( + `${mode} prep ${repeatIndex}/${REPEAT_COUNT} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`, + ); + return analysis; + } finally { + const restoredState = await renderer( + win, + `window.captureHarness.restorePixelCapture(${JSON.stringify(preparation.token)})`, + ); + const style = restoredState.hostStyle; + if (style.left !== "-20000px" || style.opacity !== "0") { + fail(`restore left=${style.left} opacity=${style.opacity}`); + } + } +} + +async function main() { + ensureDirSync(OUT_DIR); + + let resolveGuest; + const guestPromise = new Promise((resolve) => { + resolveGuest = resolve; + }); + const win = new BrowserWindow({ + width: 1000, + height: 700, + show: true, + backgroundColor: "#202020", + webPreferences: { + webviewTag: true, + contextIsolation: true, + nodeIntegration: false, + sandbox: false, + }, + }); + + win.webContents.on("will-attach-webview", (_event, webPreferences) => { + webPreferences.nodeIntegration = false; + webPreferences.contextIsolation = true; + }); + win.webContents.on("did-attach-webview", (_event, contents) => { + resolveGuest(contents); + }); + + await win.loadFile(path.join(ROOT, "index.html"), { + query: { targetUrl: fileUrl(path.join(ROOT, "bright.html")) }, + }); + const guest = await withTimeout(guestPromise, "did-attach-webview"); + await waitForGuestLoad(guest); + await renderer(win, "window.captureHarness.waitForFrames(2)"); + const guestMetrics = await readGuestMetrics(guest); + + if (guestMetrics.innerWidth !== VIEWPORT_WIDTH || guestMetrics.innerHeight !== VIEWPORT_HEIGHT) { + fail( + `guest viewport sizing inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} expected=${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`, + ); + } + pass( + `guest viewport sizing inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} dpr=${guestMetrics.devicePixelRatio}`, + ); + + await renderer(win, "window.captureHarness.restoreParking()"); + try { + const image = await capturePageSequence(guest); + const analysis = analyzeImage( + image, + { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 }, + guestMetrics, + ); + fail( + `parked webview unexpectedly captured size=${analysis.width}x${analysis.height} bright=${analysis.brightRatio.toFixed(4)}`, + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + pass(`parked webview has no copyable viewport frame error=${message}`); + } + + const results = []; + for (let index = 1; index <= REPEAT_COUNT; index += 1) { + results.push( + await captureWithPrep({ + win, + contents: guest, + mode: "viewport", + repeatIndex: index, + guestMetrics, + }), + ); + } + for (let index = 1; index <= REPEAT_COUNT; index += 1) { + results.push( + await captureWithPrep({ + win, + contents: guest, + mode: "full-page", + repeatIndex: index, + guestMetrics, + }), + ); + } + + await fsp.writeFile( + path.join(OUT_DIR, "results.json"), + `${JSON.stringify({ generatedAt: new Date().toISOString(), guestMetrics, results }, null, 2)}\n`, + ); + pass(`capture harness complete output=${OUT_DIR}`); + + if (!win.isDestroyed()) { + win.close(); + } +} + +app + .whenReady() + .then(main) + .then(() => app.quit()) + .catch(async (error) => { + console.error(error); + try { + await fsp.writeFile( + path.join(OUT_DIR, "fatal-error.txt"), + `${error && error.stack ? error.stack : String(error)}\n`, + ); + } catch { + // Ignore reporting failures during shutdown. + } + app.exit(1); + }); diff --git a/packages/desktop/capture-harness/run.sh b/packages/desktop/capture-harness/run.sh new file mode 100755 index 000000000..fc1bf139b --- /dev/null +++ b/packages/desktop/capture-harness/run.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd) + +exec "$REPO_ROOT/node_modules/.bin/electron" "$SCRIPT_DIR/main.js" diff --git a/packages/desktop/package.json b/packages/desktop/package.json index f1c0182b8..eac38d383 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -17,6 +17,7 @@ "scripts": { "build": "npm --prefix ../.. run build:server:clean && npm run build:main && electron-builder --config electron-builder.yml", "build:main": "tsc -p tsconfig.json", + "capture-harness": "./capture-harness/run.sh", "dev": "./scripts/dev.sh", "dev:win": "powershell ./scripts/dev.ps1", "test": "vitest run", diff --git a/packages/desktop/src/features/browser-automation/ipc.test.ts b/packages/desktop/src/features/browser-automation/ipc.test.ts new file mode 100644 index 000000000..cf3fa7e33 --- /dev/null +++ b/packages/desktop/src/features/browser-automation/ipc.test.ts @@ -0,0 +1,239 @@ +import type { Rectangle } from "electron"; +import { describe, expect, test } from "vitest"; +import type { TabImage } from "./service.js"; +import { adaptWebContents } from "./ipc.js"; + +class FakeImage implements TabImage { + public toPNG(): Uint8Array { + return new Uint8Array([137, 80, 78, 71]); + } + + public getSize(): { width: number; height: number } { + return { width: 640, height: 480 }; + } +} + +class FakeDebugger { + public isAttached(): boolean { + return false; + } + + public attach(): void {} + + public async sendCommand(): Promise { + return {}; + } +} + +class FakeHostWebContents { + public readonly sentMessages: Array<{ channel: string; payload: unknown }> = []; + public destroyed = false; + + public constructor(public readonly id: number) {} + + public isDestroyed(): boolean { + return this.destroyed; + } + + public send(channel: string, payload: unknown): void { + this.sentMessages.push({ channel, payload }); + } +} + +type IpcListener = (event: unknown, payload: unknown) => void; + +class FakeIpcBridge { + private readonly listeners = new Map(); + + public on(channel: string, listener: IpcListener): void { + const listeners = this.listeners.get(channel) ?? []; + listeners.push(listener); + this.listeners.set(channel, listeners); + } + + public removeListener(channel: string, listener: IpcListener): void { + const listeners = this.listeners.get(channel) ?? []; + this.listeners.set( + channel, + listeners.filter((candidate) => candidate !== listener), + ); + } + + public emit(channel: string, payload: unknown): void { + for (const listener of this.listeners.get(channel) ?? []) { + listener({}, payload); + } + } + + public listenerCount(channel: string): number { + return this.listeners.get(channel)?.length ?? 0; + } +} + +class FakeWebContents { + public readonly debugger = new FakeDebugger(); + public readonly consoleMessages: unknown[] = []; + public readonly destroyedListeners: Array<() => void> = []; + public destroyed = false; + + public constructor( + public readonly id: number, + public hostWebContents: FakeHostWebContents | null, + ) {} + + public getURL(): string { + return "https://example.com"; + } + + public getTitle(): string { + return "Example"; + } + + public canGoBack(): boolean { + return false; + } + + public canGoForward(): boolean { + return false; + } + + public isLoading(): boolean { + return false; + } + + public isDestroyed(): boolean { + return this.destroyed; + } + + public async executeJavaScript(): Promise { + return null; + } + + public async loadURL(): Promise {} + + public goBack(): void {} + + public goForward(): void {} + + public reload(): void {} + + public async capturePage( + _rect?: Rectangle, + _options?: { stayHidden?: boolean }, + ): Promise { + return new FakeImage(); + } + + public invalidate(): void {} + + public getBackgroundThrottling(): boolean { + return true; + } + + public setBackgroundThrottling(): void {} + + public on( + event: "console-message", + listener: ( + event: unknown, + level: unknown, + message: unknown, + line: unknown, + sourceId: unknown, + ) => void, + ): void { + this.consoleMessages.push({ event, listener }); + } + + public once(event: "destroyed", listener: () => void): void { + expect(event).toBe("destroyed"); + this.destroyedListeners.push(listener); + } +} + +describe("browser automation IPC adapter", () => { + test("prepareForPixelCapture asks the embedder renderer and resolves the ack token", async () => { + const host = new FakeHostWebContents(10); + const contents = new FakeWebContents(20, host); + const ipc = new FakeIpcBridge(); + const tab = adaptWebContents(contents, "browser-a", { + ipc, + createRequestId: () => "prepare-1", + }); + + const preparation = tab.prepareForPixelCapture(); + + expect(host.sentMessages).toEqual([ + { + channel: "paseo:browser:capture-prepare", + payload: { requestId: "prepare-1", browserId: "browser-a" }, + }, + ]); + ipc.emit("paseo:browser:capture-prepared", { + requestId: "other", + ok: true, + token: "wrong-token", + }); + expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1); + + ipc.emit("paseo:browser:capture-prepared", { + requestId: "prepare-1", + ok: true, + token: "token-a", + }); + + await expect(preparation).resolves.toEqual({ token: "token-a" }); + expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0); + }); + + test("restorePixelCapture sends the capture token back to the embedder renderer", async () => { + const host = new FakeHostWebContents(10); + const contents = new FakeWebContents(20, host); + const ipc = new FakeIpcBridge(); + const tab = adaptWebContents(contents, "browser-a", { + ipc, + createRequestId: () => "restore-1", + }); + + const restored = tab.restorePixelCapture({ token: "token-a" }); + + expect(host.sentMessages).toEqual([ + { + channel: "paseo:browser:capture-restore", + payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" }, + }, + ]); + ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true }); + + await expect(restored).resolves.toBeUndefined(); + }); + + test("prepareForPixelCapture rejects when the renderer reports preparation failure", async () => { + const host = new FakeHostWebContents(10); + const contents = new FakeWebContents(20, host); + const ipc = new FakeIpcBridge(); + const tab = adaptWebContents(contents, "browser-a", { + ipc, + createRequestId: () => "prepare-1", + }); + + const preparation = tab.prepareForPixelCapture(); + ipc.emit("paseo:browser:capture-prepared", { + requestId: "prepare-1", + ok: false, + message: "renderer could not prep", + }); + + await expect(preparation).rejects.toThrow("renderer could not prep"); + expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0); + }); + + test("prepareForPixelCapture rejects when the guest has no embedder renderer", async () => { + const contents = new FakeWebContents(20, null); + const tab = adaptWebContents(contents, "browser-a"); + + await expect(tab.prepareForPixelCapture()).rejects.toThrow( + "Browser host renderer is not available.", + ); + }); +}); diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 06b235c8c..14050f676 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -1,8 +1,8 @@ -import type { WebContents } from "electron"; +import type { Rectangle } from "electron"; import { ipcMain } from "electron"; import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas"; -import type { TabContents, BrowserRegistry } from "./service.js"; +import type { TabContents, BrowserRegistry, TabImage } from "./service.js"; import { executeAutomationCommand } from "./service.js"; import { listRegisteredPaseoBrowserIds, @@ -13,14 +13,86 @@ import { } from "../browser-webviews/index.js"; const MAX_CONSOLE_MESSAGES_PER_TAB = 200; +const PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS = 5_000; const consoleMessagesByContentsId = new Map(); const observedContentsIds = new Set(); +let nextPixelCaptureBridgeRequest = 0; interface IpcHandlerRegistry { handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void; } -function adaptWebContents(contents: WebContents): TabContents { +type IpcListener = (event: unknown, payload: unknown) => void; + +interface IpcCaptureBridge { + on(channel: string, listener: IpcListener): void; + removeListener(channel: string, listener: IpcListener): void; +} + +interface HostWebContents { + readonly id: number; + isDestroyed(): boolean; + send(channel: string, payload: unknown): void; +} + +interface WebContentsDebugger { + isAttached(): boolean; + attach(protocolVersion?: string): void; + sendCommand(command: string, params?: Record): Promise; +} + +interface ConsoleMessageEmitter { + on( + event: "console-message", + listener: ( + event: unknown, + level: unknown, + message: unknown, + line: unknown, + sourceId: unknown, + ) => void, + ): void; + once(event: "destroyed", listener: () => void): void; +} + +interface BrowserAutomationWebContents extends ConsoleMessageEmitter { + readonly id: number; + readonly hostWebContents: HostWebContents | null; + readonly debugger: WebContentsDebugger; + getURL(): string; + getTitle(): string; + canGoBack(): boolean; + canGoForward(): boolean; + isLoading(): boolean; + isDestroyed(): boolean; + executeJavaScript(code: string): Promise; + loadURL(url: string): Promise; + goBack(): void; + goForward(): void; + reload(): void; + capturePage(rect?: Rectangle, options?: { stayHidden?: boolean }): Promise; + invalidate(): void; + getBackgroundThrottling(): boolean; + setBackgroundThrottling(allowed: boolean): void; +} + +type PixelCaptureBridgeKind = "prepare" | "restore"; + +interface PixelCaptureBridgeSuccess { + token?: string; +} + +interface PixelCaptureBridgeOptions { + ipc?: IpcCaptureBridge; + createRequestId?: () => string; + timeoutMs?: number; +} + +export function adaptWebContents( + contents: BrowserAutomationWebContents, + browserId: string, + options?: PixelCaptureBridgeOptions, +): TabContents { observeConsoleMessages(contents); return { id: contents.id, @@ -35,7 +107,19 @@ function adaptWebContents(contents: WebContents): TabContents { goBack: () => contents.goBack(), goForward: () => contents.goForward(), reload: () => contents.reload(), - capturePage: (options) => contents.capturePage(undefined, options), + capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions), + prepareForPixelCapture: async () => { + const result = await requestPixelCaptureBridge(contents, browserId, "prepare", options); + if (!result.token) { + throw new Error("Browser pixel capture preparation did not return a token."); + } + return { token: result.token }; + }, + restorePixelCapture: async (preparation) => { + await requestPixelCaptureBridge(contents, browserId, "restore", options, { + token: preparation.token, + }); + }, invalidate: () => contents.invalidate(), isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(), setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed), @@ -49,7 +133,101 @@ function adaptWebContents(contents: WebContents): TabContents { }; } -function observeConsoleMessages(contents: WebContents): void { +function requestPixelCaptureBridge( + contents: BrowserAutomationWebContents, + browserId: string, + kind: PixelCaptureBridgeKind, + options: PixelCaptureBridgeOptions | undefined, + extraPayload?: { token: string }, +): Promise { + const host = contents.hostWebContents; + if (!host || host.isDestroyed()) { + return Promise.reject(new Error("Browser host renderer is not available.")); + } + + const ipc = options?.ipc ?? ipcMain; + const requestId = + options?.createRequestId?.() ?? `browser-pixel-capture-${++nextPixelCaptureBridgeRequest}`; + const timeoutMs = options?.timeoutMs ?? PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS; + const requestChannel = + kind === "prepare" ? "paseo:browser:capture-prepare" : "paseo:browser:capture-restore"; + const responseChannel = + kind === "prepare" ? "paseo:browser:capture-prepared" : "paseo:browser:capture-restored"; + + return new Promise((resolve, reject) => { + let timeoutId: ReturnType | undefined; + const cleanup = () => { + if (timeoutId) { + clearTimeout(timeoutId); + } + ipc.removeListener(responseChannel, listener); + }; + const listener: IpcListener = (_event, payload) => { + const response = readPixelCaptureBridgeResponse(payload, requestId); + if (!response) { + return; + } + cleanup(); + if (response.ok) { + resolve(response); + } else { + reject(new Error(response.message)); + } + }; + + ipc.on(responseChannel, listener); + timeoutId = setTimeout(() => { + cleanup(); + reject(new Error(`Browser pixel capture ${kind} timed out.`)); + }, timeoutMs); + + try { + host.send(requestChannel, { + requestId, + browserId, + ...(extraPayload ? { token: extraPayload.token } : {}), + }); + } catch (error) { + cleanup(); + reject(error); + } + }); +} + +function readPixelCaptureBridgeResponse( + payload: unknown, + requestId: string, +): ({ ok: true } & PixelCaptureBridgeSuccess) | { ok: false; message: string } | null { + if (!isRecord(payload)) { + return null; + } + const record = payload; + if (record.requestId !== requestId) { + return null; + } + if (record.ok === true) { + return { + ok: true, + ...(typeof record.token === "string" && record.token.length > 0 + ? { token: record.token } + : {}), + }; + } + if (record.ok === false) { + const message = + typeof record.message === "string" && record.message.length > 0 + ? record.message + : "Browser pixel capture bridge failed."; + return { ok: false, message }; + } + return null; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function observeConsoleMessages(contents: BrowserAutomationWebContents): void { if (observedContentsIds.has(contents.id)) { return; } @@ -89,7 +267,7 @@ function createRegistry(): BrowserRegistry { listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace, getTabContents(browserId: string): TabContents | null { const contents = getPaseoBrowserWebContents(browserId); - return contents ? adaptWebContents(contents) : null; + return contents ? adaptWebContents(contents, browserId) : null; }, getBrowserWorkspaceId: getPaseoBrowserWorkspaceId, getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId, diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index f949c3982..66367cd20 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -7,7 +7,12 @@ import type { BrowserAutomationExecuteRequest, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; -import type { BrowserRegistry, TabContents, TabImage } from "./service.js"; +import type { + BrowserRegistry, + TabContents, + TabImage, + TabPixelCapturePreparation, +} from "./service.js"; import { executeAutomationCommand } from "./service.js"; const BROWSER_A = "11111111-1111-4111-8111-111111111111"; @@ -36,6 +41,9 @@ class FakeTab implements TabContents { public readonly actions: string[] = []; public readonly capturedViewports: Array<{ stayHidden?: boolean }> = []; public readonly debugCommands: Array<{ command: string; params?: Record }> = []; + public readonly restoredPixelCaptureTokens: string[] = []; + private readonly captureStartWaiters: Array<() => void> = []; + private readonly deferredCaptures: Array<(image: TabImage) => void> = []; public destroyed = false; public bodyText = ""; @@ -44,6 +52,9 @@ class FakeTab implements TabContents { public networkEntries: unknown[] = []; public consoleMessages: BrowserAutomationConsoleLogEntry[] = []; public captureNeverPaints = false; + public captureThrows = false; + public deferCaptures = false; + public prepareNeverAcks = false; public layoutMetrics = { cssLayoutViewport: { clientWidth: 390, clientHeight: 844 }, cssContentSize: { width: 390, height: 1200 }, @@ -52,6 +63,7 @@ class FakeTab implements TabContents { public documentNodeId = 1; public queriedNodeId = 2; public backgroundThrottlingAllowed = true; + private nextPixelCapturePreparationId = 0; public constructor( public readonly id: number, @@ -116,12 +128,35 @@ class FakeTab implements TabContents { public async capturePage(options?: { stayHidden?: boolean }): Promise { this.capturedViewports.push(options ?? {}); this.actions.push("capture"); + this.resolveCaptureStartWaiters(); + if (this.captureThrows) { + throw new Error("capture failed"); + } if (this.captureNeverPaints) { return new Promise(() => {}); } + if (this.deferCaptures) { + return new Promise((resolve) => { + this.deferredCaptures.push(resolve); + }); + } return new FakeImage(); } + public async prepareForPixelCapture(): Promise { + this.actions.push("prepare"); + if (this.prepareNeverAcks) { + return new Promise(() => {}); + } + const token = `capture-${++this.nextPixelCapturePreparationId}`; + return { token }; + } + + public async restorePixelCapture(preparation: TabPixelCapturePreparation): Promise { + this.actions.push(`restore:${preparation.token}`); + this.restoredPixelCaptureTokens.push(preparation.token); + } + public invalidate(): void { this.actions.push("invalidate"); } @@ -143,6 +178,7 @@ class FakeTab implements TabContents { command: string, params?: Record, ): Promise { + this.actions.push(`debug:${command}`); this.debugCommands.push({ command, ...(params ? { params } : {}) }); if (command === "Page.getLayoutMetrics") { return this.layoutMetrics; @@ -158,6 +194,33 @@ class FakeTab implements TabContents { } return {}; } + + public waitForCaptureStart(count: number): Promise { + if (this.capturedViewports.length >= count) { + return Promise.resolve(); + } + return new Promise((resolve) => { + this.captureStartWaiters.push(() => { + if (this.capturedViewports.length >= count) { + resolve(); + } + }); + }); + } + + public finishNextCapture(): void { + const resolve = this.deferredCaptures.shift(); + if (!resolve) { + throw new Error("No deferred capture is waiting"); + } + resolve(new FakeImage()); + } + + private resolveCaptureStartWaiters(): void { + for (const waiter of this.captureStartWaiters.splice(0)) { + waiter(); + } + } } class FakeRegistry implements BrowserRegistry { @@ -852,9 +915,11 @@ describe("executeAutomationCommand", () => { }); expect(browser.tab.capturedViewports).toEqual([{ stayHidden: false }]); expect(browser.tab.actions).toEqual([ + "prepare", "background:false", "invalidate", "capture", + "restore:capture-1", "background:true", ]); }); @@ -882,9 +947,11 @@ describe("executeAutomationCommand", () => { }, }); expect(browser.tab.actions).toEqual([ + "prepare", "background:false", "invalidate", "capture", + "restore:capture-1", "background:true", ]); } finally { @@ -892,6 +959,111 @@ describe("executeAutomationCommand", () => { } }); + test("screenshot restores capture preparation when viewport capture fails", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.captureThrows = true; + + await expect( + browser.execute({ + command: "screenshot", + args: { browserId: BROWSER_A }, + }), + ).rejects.toThrow("capture failed"); + + expect(browser.tab.actions).toEqual([ + "prepare", + "background:false", + "invalidate", + "capture", + "restore:capture-1", + "background:true", + ]); + }); + + test("screenshot returns no-frame when capture preparation does not ack", async () => { + vi.useFakeTimers(); + try { + const browser = new BrowserAutomationHarness(); + browser.tab.prepareNeverAcks = true; + + const resultPromise = browser.execute({ + command: "screenshot", + args: { browserId: BROWSER_A }, + }); + await vi.advanceTimersByTimeAsync(5_000); + + await expect(resultPromise).resolves.toEqual({ + requestId: "req-screenshot", + ok: false, + error: { + code: "screenshot_no_frame", + message: + "The browser tab has no painted frame. Focus the tab in the app, then try again.", + retryable: false, + }, + }); + expect(browser.tab.actions).toEqual(["prepare", "background:true"]); + } finally { + vi.useRealTimers(); + } + }); + + test("overlapping screenshots serialize capture preparation and restore", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.deferCaptures = true; + + const first = browser.execute({ + command: "screenshot", + args: { browserId: BROWSER_A }, + }); + await browser.tab.waitForCaptureStart(1); + + const second = browser.execute( + { + command: "screenshot", + args: { browserId: BROWSER_A }, + }, + { requestId: "req-screenshot-2" }, + ); + await Promise.resolve(); + + expect(browser.tab.actions).toEqual(["prepare", "background:false", "invalidate", "capture"]); + + browser.tab.finishNextCapture(); + await browser.tab.waitForCaptureStart(2); + + expect(browser.tab.actions).toEqual([ + "prepare", + "background:false", + "invalidate", + "capture", + "restore:capture-1", + "background:true", + "prepare", + "background:false", + "invalidate", + "capture", + ]); + + browser.tab.finishNextCapture(); + await expect(first).resolves.toMatchObject({ requestId: "req-screenshot", ok: true }); + await expect(second).resolves.toMatchObject({ requestId: "req-screenshot-2", ok: true }); + expect(browser.tab.actions).toEqual([ + "prepare", + "background:false", + "invalidate", + "capture", + "restore:capture-1", + "background:true", + "prepare", + "background:false", + "invalidate", + "capture", + "restore:capture-2", + "background:true", + ]); + }); + test("screenshot with fullPage captures the page content area through CDP", async () => { const browser = new BrowserAutomationHarness(); @@ -923,6 +1095,44 @@ describe("executeAutomationCommand", () => { }, }, ]); + expect(browser.tab.actions).toEqual([ + "prepare", + "background:false", + "invalidate", + "debug:Page.getLayoutMetrics", + "debug:Page.captureScreenshot", + "restore:capture-1", + "background:true", + ]); + }); + + test("screenshot with fullPage restores capture preparation when CDP returns no image", async () => { + const browser = new BrowserAutomationHarness(); + browser.tab.fullPageScreenshotData = ""; + + const result = await browser.execute({ + command: "screenshot", + args: { browserId: BROWSER_A, fullPage: true }, + }); + + expect(result).toEqual({ + requestId: "req-screenshot", + ok: false, + error: { + code: "browser_unsupported", + message: "browser_screenshot fullPage returned no data", + retryable: false, + }, + }); + expect(browser.tab.actions).toEqual([ + "prepare", + "background:false", + "invalidate", + "debug:Page.getLayoutMetrics", + "debug:Page.captureScreenshot", + "restore:capture-1", + "background:true", + ]); }); test("upload resolves workspace files before setting them on the file input", async () => { diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index 9b0c8b60d..e04d43f9f 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -24,6 +24,8 @@ export interface TabContents { goForward(): void; reload(): void; capturePage(options?: TabCapturePageOptions): Promise; + prepareForPixelCapture(): Promise; + restorePixelCapture(preparation: TabPixelCapturePreparation): Promise; invalidate(): void; isBackgroundThrottlingAllowed(): boolean; setBackgroundThrottling(allowed: boolean): void; @@ -31,6 +33,10 @@ export interface TabContents { sendDebugCommand?(command: string, params?: Record): Promise; } +export interface TabPixelCapturePreparation { + token: string; +} + export interface TabImage { toPNG(): Uint8Array; getSize(): { width: number; height: number }; @@ -124,19 +130,56 @@ async function runSerializedPixelCapture( } } -async function capturePaintedViewport(contents: TabContents): Promise { +async function prepareForPixelCapture(contents: TabContents): Promise { + try { + return await withPixelCaptureTimeout(contents.prepareForPixelCapture()); + } catch { + throw new ScreenshotNoFrameError(); + } +} + +async function restorePixelCapture( + contents: TabContents, + preparation: TabPixelCapturePreparation, +): Promise { + try { + await withPixelCaptureTimeout(contents.restorePixelCapture(preparation)); + } catch { + throw new ScreenshotNoFrameError(); + } +} + +async function runPreparedPixelCapture( + contents: TabContents, + capture: () => Promise, +): Promise { return runSerializedPixelCapture(contents, async () => { const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed(); - contents.setBackgroundThrottling(false); + let preparation: TabPixelCapturePreparation | null = null; try { + // Offscreen-parked webview guests have no compositor surface for + // capturePage/CDP to copy. The renderer must briefly make the host + // paintable before capture; see ~/.paseo/plans/browser-capture-harness-results.md. + preparation = await prepareForPixelCapture(contents); + contents.setBackgroundThrottling(false); contents.invalidate(); - return await withPixelCaptureTimeout(contents.capturePage({ stayHidden: false })); + return await withPixelCaptureTimeout(capture()); } finally { - contents.setBackgroundThrottling(previousBackgroundThrottling); + try { + if (preparation) { + await restorePixelCapture(contents, preparation); + } + } finally { + contents.setBackgroundThrottling(previousBackgroundThrottling); + } } }); } +async function capturePaintedViewport(contents: TabContents): Promise { + return runPreparedPixelCapture(contents, () => contents.capturePage({ stayHidden: false })); +} + function tabInfoFromContents( browserId: string, contents: TabContents, @@ -860,18 +903,21 @@ async function executeFullPageScreenshot( if (!target.contents.sendDebugCommand) { return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP"); } - const metrics = await getCdpLayoutMetrics(target.contents); - const width = metrics.contentWidth; - const height = metrics.contentHeight; + const sendDebugCommand = target.contents.sendDebugCommand.bind(target.contents); let screenshot: CdpCaptureScreenshotResult; + let width = 0; + let height = 0; try { - screenshot = (await withPixelCaptureTimeout( - target.contents.sendDebugCommand("Page.captureScreenshot", { + screenshot = await runPreparedPixelCapture(target.contents, async () => { + const metrics = await getCdpLayoutMetrics(target.contents); + width = metrics.contentWidth; + height = metrics.contentHeight; + return (await sendDebugCommand("Page.captureScreenshot", { format: "png", captureBeyondViewport: true, clip: { x: 0, y: 0, width, height, scale: 1 }, - }), - )) as CdpCaptureScreenshotResult; + })) as CdpCaptureScreenshotResult; + }); } catch (error) { if (isScreenshotNoFrameError(error)) { return screenshotNoFrameFailure(requestId); diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index 02fe9dc0e..3785c6ec9 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -1,6 +1,81 @@ import { contextBridge, ipcRenderer, webUtils } from "electron"; type EventHandler = (payload: unknown) => void; +type BrowserPixelCapturePrepareHandler = (input: { + browserId: string; +}) => Promise<{ token: string }>; +type BrowserPixelCaptureRestoreHandler = (input: { token: string }) => Promise; + +let prepareForPixelCaptureHandler: BrowserPixelCapturePrepareHandler | null = null; +let restorePixelCaptureHandler: BrowserPixelCaptureRestoreHandler | null = null; + +function readStringField(payload: unknown, key: string): string | null { + if (!isRecord(payload)) { + return null; + } + const value = payload[key]; + return typeof value === "string" && value.length > 0 ? value : null; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +ipcRenderer.on("paseo:browser:capture-prepare", async (_event, payload: unknown) => { + const requestId = readStringField(payload, "requestId"); + const browserId = readStringField(payload, "browserId"); + if (!requestId || !browserId || !prepareForPixelCaptureHandler) { + ipcRenderer.send("paseo:browser:capture-prepared", { + requestId: requestId ?? "unknown", + ok: false, + message: "Browser pixel capture preparation is unavailable.", + }); + return; + } + + try { + const preparation = await prepareForPixelCaptureHandler({ browserId }); + ipcRenderer.send("paseo:browser:capture-prepared", { + requestId, + ok: true, + token: preparation.token, + }); + } catch (error) { + ipcRenderer.send("paseo:browser:capture-prepared", { + requestId, + ok: false, + message: errorMessage(error), + }); + } +}); + +ipcRenderer.on("paseo:browser:capture-restore", async (_event, payload: unknown) => { + const requestId = readStringField(payload, "requestId"); + const token = readStringField(payload, "token"); + if (!requestId || !token || !restorePixelCaptureHandler) { + ipcRenderer.send("paseo:browser:capture-restored", { + requestId: requestId ?? "unknown", + ok: false, + message: "Browser pixel capture restore is unavailable.", + }); + return; + } + + try { + await restorePixelCaptureHandler({ token }); + ipcRenderer.send("paseo:browser:capture-restored", { requestId, ok: true }); + } catch (error) { + ipcRenderer.send("paseo:browser:capture-restored", { + requestId, + ok: false, + message: errorMessage(error), + }); + } +}); contextBridge.exposeInMainWorld("paseoDesktop", { platform: process.platform, @@ -90,5 +165,21 @@ contextBridge.exposeInMainWorld("paseoDesktop", { ) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect), copyElement: (payload: { text?: string; imageDataUrl?: string }) => ipcRenderer.invoke("paseo:browser:copy-element", payload), + onPrepareForPixelCapture: (handler: BrowserPixelCapturePrepareHandler): (() => void) => { + prepareForPixelCaptureHandler = handler; + return () => { + if (prepareForPixelCaptureHandler === handler) { + prepareForPixelCaptureHandler = null; + } + }; + }, + onRestorePixelCapture: (handler: BrowserPixelCaptureRestoreHandler): (() => void) => { + restorePixelCaptureHandler = handler; + return () => { + if (restorePixelCaptureHandler === handler) { + restorePixelCaptureHandler = null; + } + }; + }, }, });