From ab62a023fd24c3b2ea03ad1325f83b04b8afa738 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Fri, 3 Jul 2026 11:34:48 +0200 Subject: [PATCH] fix(browser): park webviews permanently paintable Delete the renderer capture prep handshake and keep screenshots on the serialized invalidate/retry path. The capture harness now defaults to the production P1 attach-off parking check. --- docs/browser-capture-harness.md | 45 +- .../app/src/browser-automation/handler.ts | 7 +- .../browser-webview-resident.browser.test.ts | 369 +++++-------- .../components/browser-webview-resident.ts | 255 +-------- packages/app/src/desktop/host.ts | 16 - packages/desktop/capture-harness/index.html | 28 +- packages/desktop/capture-harness/main.js | 30 +- .../features/browser-automation/ipc.test.ts | 484 ++++-------------- .../src/features/browser-automation/ipc.ts | 277 +--------- .../browser-automation/service.test.ts | 212 +------- .../features/browser-automation/service.ts | 82 +-- .../features/browser-webviews/index.test.ts | 44 ++ .../src/features/browser-webviews/index.ts | 20 +- packages/desktop/src/preload.ts | 135 ----- .../src/server/browser-tools/tools.test.ts | 6 +- 15 files changed, 358 insertions(+), 1652 deletions(-) create mode 100644 packages/desktop/src/features/browser-webviews/index.test.ts diff --git a/docs/browser-capture-harness.md b/docs/browser-capture-harness.md index b6e98ae49..dd09d5a02 100644 --- a/docs/browser-capture-harness.md +++ b/docs/browser-capture-harness.md @@ -4,14 +4,16 @@ The desktop capture harness is the real-Electron verification path for browser s It validates the compositor behavior that unit tests cannot see: - the resident automation `` starts in the production parking state; -- the parked guest has no copyable viewport frame; +- the parked guest remains paintable and has a copyable viewport frame; - the resident webview guest is sized to 1280x800 logical pixels; -- multiple resident webviews are parked as an overlapping stack, and the capture - target is raised above its sibling webviews before capture; +- multiple resident webviews are parked as an overlapping stack without per-capture + stacking changes; - a newly attached resident webview whose first useful frame is delayed can be captured - by holding prep active and retrying until the frame appears; -- the app-style prep sequence moves the host to a paintable 1x1 clipped state, waits two animation frames and a layout read, then both viewport `capturePage` and full-page CDP screenshots return real pixels; -- restore returns the host to offscreen parking after every capture. + by retrying until the frame appears; +- both viewport `capturePage` and full-page CDP screenshots return real pixels from + the permanent production parking state; +- guest background throttling can be disabled once at attach without per-capture + renderer coordination. Run it with the repo Electron: @@ -34,25 +36,22 @@ The harness writes PNG evidence and `results.json` to: packages/desktop/capture-harness/out/ ``` -A passing run prints `PASS` lines for both guest sizes, the expected parked-capture -failure, the legacy stacked-below-the-clip failure for the second webview, five viewport -prep captures and five full-page prep captures for each of the first two webviews, -fresh-delayed-first-frame viewport and full-page captures for a newly attached webview, -and final completion. The PNG sizes may be device-pixel scaled; on a Retina display the -1280x800 logical viewport is usually saved as 2560x1600. +A passing run prints `PASS` lines for the production P1 attach-off parking state, +including fresh, settled, 75-second soak, multi-tab, viewport, and full-page checks. The +PNG sizes may be device-pixel scaled; on a Retina display the 1280x800 logical viewport +is usually saved as 2560x1600. ## Mechanism Electron captures copy from the guest web contents' compositor surface. A resident -webview parked at `left:-20000px` and `opacity:0` does not have a copyable surface, and -`capturePage({ stayHidden:false })` or CDP `Page.captureScreenshot` cannot rescue it. +webview parked with `display:none`, offscreen coordinates, or `opacity:0` can lose its +copyable surface. The production parking state keeps the host fixed at `left:0`, `top:0`, +`width:1px`, `height:1px`, `overflow:hidden`, `opacity:1`, and `pointer-events:none`. +The webviews inside stay full-size at 1280x800, `display:inline-flex`, and absolutely +overlap at `left:0`, `top:0`. -Before pixel capture, the app renderer temporarily makes the resident host paintable: -`left:0`, `top:0`, `opacity:1`, `pointer-events:none`, host size `1x1`, and -`overflow:hidden`, with the full-size 1280x800 webview inside. Resident webviews are -parked absolutely at `0,0` inside that host because a second webview stacked below the -1px clip still has no copyable surface. During capture, the renderer raises the target -webview above the other resident webviews; an overlay or sibling above the target can make -full-page CDP capture fail. Main captures only after the renderer acknowledges two -animation frames plus a `getBoundingClientRect()` read, and the renderer restores parking -in a `finally`. +There is no renderer prep/restore handshake. Main disables guest background throttling +once when the webview attaches, then screenshot capture uses the shared serialized queue, +invalidates before each attempt, and retries known first-frame failures within the +5-second capture budget. Viewport screenshots use `capturePage({ stayHidden:false })`; +full-page screenshots use the existing CDP path with layout metrics and screenshot clip. diff --git a/packages/app/src/browser-automation/handler.ts b/packages/app/src/browser-automation/handler.ts index 00fb05dfc..35b3f5599 100644 --- a/packages/app/src/browser-automation/handler.ts +++ b/packages/app/src/browser-automation/handler.ts @@ -1,9 +1,6 @@ import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host"; -import { - ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault, - installResidentBrowserCaptureBridge, -} from "@/components/browser-webview-resident"; +import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident"; import { createWorkspaceBrowser } from "@/stores/browser-store"; import { buildWorkspaceTabPersistenceKey, @@ -43,7 +40,6 @@ export function mountBrowserAutomationHandler( options: BrowserAutomationHandlerOptions, ): () => void { const getHost = options.getHost ?? getDesktopHost; - const uninstallCaptureBridge = installResidentBrowserCaptureBridge(); const unsubscribe = options.client.on("browser.automation.execute.request", (request) => { void handleBrowserAutomationRequest({ client: options.client, @@ -62,7 +58,6 @@ export function mountBrowserAutomationHandler( }); return () => { unsubscribe(); - uninstallCaptureBridge(); }; } diff --git a/packages/app/src/components/browser-webview-resident.browser.test.ts b/packages/app/src/components/browser-webview-resident.browser.test.ts index fa22d3f86..3935a3b28 100644 --- a/packages/app/src/components/browser-webview-resident.browser.test.ts +++ b/packages/app/src/components/browser-webview-resident.browser.test.ts @@ -1,47 +1,67 @@ import { afterEach, describe, expect, it } from "vitest"; import { - cancelResidentBrowserWebviewPixelCapture, clearResidentBrowserWebviewsForTests, ensureResidentBrowserWebview, prepareBrowserWebview, - prepareResidentBrowserWebviewForPixelCapture, releaseResidentBrowserWebview, removeResidentBrowserWebview, - restoreResidentBrowserWebviewAfterPixelCapture, takeResidentBrowserWebview, } from "./browser-webview-resident"; +const RESIDENT_HOST_ID = "paseo-browser-resident-webviews"; + +function residentHost(): HTMLElement { + const host = document.getElementById(RESIDENT_HOST_ID); + if (!host) { + throw new Error("Expected resident browser host"); + } + return host; +} + +function expectPermanentHostParking(host: HTMLElement): void { + expect(host.style.position).toBe("fixed"); + expect(host.style.left).toBe("0px"); + expect(host.style.top).toBe("0px"); + expect(host.style.width).toBe("1px"); + expect(host.style.height).toBe("1px"); + expect(host.style.overflow).toBe("hidden"); + expect(host.style.opacity).toBe("1"); + expect(host.style.pointerEvents).toBe("none"); + expect(host.style.display).toBe("block"); + expect(host.style.visibility).toBe("visible"); + expect(host.style.transform).toBe(""); +} + +function expectResidentWebviewParking(webview: HTMLElement): void { + expect(webview.style.display).toBe("inline-flex"); + expect(webview.style.flex).toBe("0 0 auto"); + expect(webview.style.width).toBe("1280px"); + expect(webview.style.height).toBe("800px"); + expect(webview.style.position).toBe("absolute"); + expect(webview.style.left).toBe("0px"); + expect(webview.style.top).toBe("0px"); + expect(webview.style.zIndex).toBe("0"); +} + describe("resident browser webviews", () => { afterEach(() => { clearResidentBrowserWebviewsForTests(); }); - it("keeps a browser webview mounted offscreen and reuses the same node", () => { - const host = document.createElement("div"); + it("parks a browser webview in the permanent paintable 1x1 host", () => { + const visibleHost = document.createElement("div"); const webview = document.createElement("webview"); - host.appendChild(webview); - document.body.appendChild(host); + visibleHost.appendChild(webview); + document.body.appendChild(visibleHost); releaseResidentBrowserWebview("browser-a", webview); - expect(host.children).toHaveLength(0); + const host = residentHost(); + expect(visibleHost.children).toHaveLength(0); + expect(Array.from(host.children)).toEqual([webview]); expect(webview.isConnected).toBe(true); - expect(webview.style.display).toBe("inline-flex"); - expect(webview.style.width).toBe("1280px"); - expect(webview.style.height).toBe("800px"); - expect(webview.style.position).toBe("absolute"); - expect(webview.style.left).toBe("0px"); - expect(webview.style.top).toBe("0px"); - expect(webview.style.zIndex).toBe("0"); - - const reused = takeResidentBrowserWebview("browser-a"); - - expect(reused).toBe(webview); - expect(webview.style.position).toBe(""); - expect(webview.style.left).toBe(""); - expect(webview.style.top).toBe(""); - expect(webview.style.zIndex).toBe(""); - expect(takeResidentBrowserWebview("browser-a")).toBeNull(); + expectPermanentHostParking(host); + expectResidentWebviewParking(webview); }); it("creates a resident webview for an agent-created unfocused tab", () => { @@ -57,6 +77,80 @@ describe("resident browser webviews", () => { expect((webview as HTMLUnknownElement & { src?: string })?.src).toContain( "https://example.com", ); + expectPermanentHostParking(residentHost()); + expectResidentWebviewParking(webview as HTMLElement); + }); + + it("normalizes an existing resident host back to permanent parking", () => { + const staleHost = document.createElement("div"); + staleHost.id = RESIDENT_HOST_ID; + staleHost.style.left = "-20000px"; + staleHost.style.width = "1280px"; + staleHost.style.height = "800px"; + staleHost.style.opacity = "0"; + staleHost.style.display = "none"; + document.body.appendChild(staleHost); + + const webview = ensureResidentBrowserWebview({ + browserId: "browser-stale-host", + url: "https://example.com", + }); + + expect(webview).not.toBeNull(); + expectPermanentHostParking(staleHost); + expectResidentWebviewParking(webview as HTMLElement); + }); + + it("parks resident webviews as an overlapping stack", () => { + const firstWebview = ensureResidentBrowserWebview({ + browserId: "browser-first", + url: "https://example.com/first", + }); + const secondWebview = ensureResidentBrowserWebview({ + browserId: "browser-second", + url: "https://example.com/second", + }); + + const host = residentHost(); + expect(firstWebview?.parentElement).toBe(host); + expect(secondWebview?.parentElement).toBe(host); + expectResidentWebviewParking(firstWebview as HTMLElement); + expectResidentWebviewParking(secondWebview as HTMLElement); + }); + + it("moves a resident webview into a visible pane without recreating the node", () => { + const webview = ensureResidentBrowserWebview({ + browserId: "browser-visible", + url: "https://example.com", + }); + + const visibleWebview = takeResidentBrowserWebview("browser-visible"); + + expect(visibleWebview).toBe(webview); + expect(webview?.style.position).toBe(""); + expect(webview?.style.left).toBe(""); + expect(webview?.style.top).toBe(""); + expect(webview?.style.zIndex).toBe(""); + expect(takeResidentBrowserWebview("browser-visible")).toBeNull(); + }); + + it("returns an existing visible pane webview instead of creating a resident duplicate", () => { + const visibleHost = document.createElement("div"); + const visibleWebview = document.createElement("webview"); + prepareBrowserWebview(visibleWebview, { + browserId: "browser-visible-pane", + initialUrl: "https://example.com", + }); + visibleHost.appendChild(visibleWebview); + document.body.appendChild(visibleHost); + + const webview = ensureResidentBrowserWebview({ + browserId: "browser-visible-pane", + url: "https://example.com/agent", + }); + + expect(webview).toBe(visibleWebview); + expect(document.getElementById(RESIDENT_HOST_ID)).toBeNull(); }); it("removes a resident webview when its browser tab closes", () => { @@ -70,231 +164,4 @@ describe("resident browser webviews", () => { expect(webview?.isConnected).toBe(false); expect(takeResidentBrowserWebview("browser-closed")).toBeNull(); }); - - it("temporarily makes resident webviews paintable for pixel capture", async () => { - const webview = ensureResidentBrowserWebview({ - browserId: "browser-capture", - url: "https://example.com", - }); - if (!webview) { - throw new Error("Expected resident browser webview"); - } - - const preparation = await prepareResidentBrowserWebviewForPixelCapture({ - browserId: "browser-capture", - }); - const host = document.getElementById("paseo-browser-resident-webviews"); - - expect(preparation.token).toBe("capture-1"); - expect(host?.style.left).toBe("0px"); - expect(host?.style.top).toBe("0px"); - expect(host?.style.width).toBe("1px"); - expect(host?.style.height).toBe("1px"); - expect(host?.style.overflow).toBe("hidden"); - expect(host?.style.opacity).toBe("1"); - expect(host?.style.pointerEvents).toBe("none"); - expect(webview.style.display).toBe("inline-flex"); - expect(webview.style.width).toBe("1280px"); - expect(webview.style.height).toBe("800px"); - expect(webview.style.position).toBe("absolute"); - expect(webview.style.left).toBe("0px"); - expect(webview.style.top).toBe("0px"); - expect(webview.style.zIndex).toBe("2"); - - await restoreResidentBrowserWebviewAfterPixelCapture(preparation); - - expect(host?.style.left).toBe("-20000px"); - expect(host?.style.width).toBe("1280px"); - expect(host?.style.height).toBe("800px"); - expect(host?.style.opacity).toBe("0"); - expect(webview.style.zIndex).toBe("0"); - }); - - it("parks resident webviews as an overlapping stack and raises the capture target", async () => { - const firstWebview = ensureResidentBrowserWebview({ - browserId: "browser-first", - url: "https://example.com/first", - }); - const secondWebview = ensureResidentBrowserWebview({ - browserId: "browser-second", - url: "https://example.com/second", - }); - if (!firstWebview || !secondWebview) { - throw new Error("Expected resident browser webviews"); - } - - expect(firstWebview.style.position).toBe("absolute"); - expect(firstWebview.style.left).toBe("0px"); - expect(firstWebview.style.top).toBe("0px"); - expect(firstWebview.style.zIndex).toBe("0"); - expect(secondWebview.style.position).toBe("absolute"); - expect(secondWebview.style.left).toBe("0px"); - expect(secondWebview.style.top).toBe("0px"); - expect(secondWebview.style.zIndex).toBe("0"); - - const preparation = await prepareResidentBrowserWebviewForPixelCapture({ - browserId: "browser-second", - }); - - expect(firstWebview.style.zIndex).toBe("0"); - expect(secondWebview.style.zIndex).toBe("2"); - - await restoreResidentBrowserWebviewAfterPixelCapture(preparation); - - expect(firstWebview.style.zIndex).toBe("0"); - expect(secondWebview.style.zIndex).toBe("0"); - }); - - it("prepares the physically resident webview when a focused duplicate exists outside the resident host", async () => { - const residentWebview = ensureResidentBrowserWebview({ - browserId: "browser-focused-undisplayed", - url: "https://example.com/resident", - }); - if (!residentWebview) { - throw new Error("Expected resident browser webview"); - } - const visibleHost = document.createElement("div"); - const duplicateWebview = document.createElement("webview"); - prepareBrowserWebview(duplicateWebview, { - browserId: "browser-focused-undisplayed", - initialUrl: "https://example.com/duplicate", - }); - visibleHost.appendChild(duplicateWebview); - document.body.prepend(visibleHost); - - const preparation = await prepareResidentBrowserWebviewForPixelCapture({ - browserId: "browser-focused-undisplayed", - }); - const residentHost = document.getElementById("paseo-browser-resident-webviews"); - - expect(residentHost?.style.left).toBe("0px"); - expect(residentHost?.style.width).toBe("1px"); - expect(residentWebview.style.zIndex).toBe("2"); - expect(duplicateWebview.style.zIndex).toBe(""); - - await restoreResidentBrowserWebviewAfterPixelCapture(preparation); - }); - - it("does not apply resident prep styles for a genuinely visible pane webview", async () => { - const visibleHost = document.createElement("div"); - const visibleWebview = document.createElement("webview"); - prepareBrowserWebview(visibleWebview, { - browserId: "browser-visible-pane", - initialUrl: "https://example.com", - }); - visibleWebview.style.display = "flex"; - visibleWebview.style.width = "100%"; - visibleWebview.style.height = "100%"; - visibleHost.appendChild(visibleWebview); - document.body.appendChild(visibleHost); - - const preparation = await prepareResidentBrowserWebviewForPixelCapture({ - browserId: "browser-visible-pane", - }); - const residentHost = document.getElementById("paseo-browser-resident-webviews"); - - expect(residentHost?.style.left).toBe("-20000px"); - expect(residentHost?.style.width).toBe("1280px"); - expect(visibleWebview.style.position).toBe(""); - expect(visibleWebview.style.zIndex).toBe(""); - - await restoreResidentBrowserWebviewAfterPixelCapture(preparation); - }); - - it("clears capture preparation when a resident webview is taken visible", async () => { - const webview = ensureResidentBrowserWebview({ - browserId: "browser-visible", - url: "https://example.com", - }); - if (!webview) { - throw new Error("Expected resident browser webview"); - } - - const preparation = await prepareResidentBrowserWebviewForPixelCapture({ - browserId: "browser-visible", - }); - const visibleWebview = takeResidentBrowserWebview("browser-visible"); - - expect(visibleWebview).toBe(webview); - expect(webview.style.position).toBe(""); - expect(webview.style.left).toBe(""); - expect(webview.style.top).toBe(""); - expect(webview.style.zIndex).toBe(""); - - await restoreResidentBrowserWebviewAfterPixelCapture(preparation); - - expect(webview.style.position).toBe(""); - expect(webview.style.left).toBe(""); - expect(webview.style.top).toBe(""); - expect(webview.style.zIndex).toBe(""); - }); - - it("keeps the resident host paintable until every capture token is restored", async () => { - ensureResidentBrowserWebview({ - browserId: "browser-overlap", - url: "https://example.com", - }); - - const first = await prepareResidentBrowserWebviewForPixelCapture({ - browserId: "browser-overlap", - }); - const second = await prepareResidentBrowserWebviewForPixelCapture({ - browserId: "browser-overlap", - }); - const host = document.getElementById("paseo-browser-resident-webviews"); - - await restoreResidentBrowserWebviewAfterPixelCapture(first); - - expect(host?.style.left).toBe("0px"); - expect(host?.style.width).toBe("1px"); - expect(host?.style.opacity).toBe("1"); - - await restoreResidentBrowserWebviewAfterPixelCapture(second); - - expect(host?.style.left).toBe("-20000px"); - expect(host?.style.width).toBe("1280px"); - expect(host?.style.opacity).toBe("0"); - }); - - it("cancels an in-flight pixel capture preparation by request id", async () => { - ensureResidentBrowserWebview({ - browserId: "browser-cancel", - url: "https://example.com", - }); - - const preparation = prepareResidentBrowserWebviewForPixelCapture({ - requestId: "prepare-1", - browserId: "browser-cancel", - }); - const host = document.getElementById("paseo-browser-resident-webviews"); - expect(host?.style.left).toBe("0px"); - expect(host?.style.opacity).toBe("1"); - - await cancelResidentBrowserWebviewPixelCapture({ requestId: "prepare-1" }); - - await expect(preparation).rejects.toThrow("Browser pixel capture preparation was canceled."); - expect(host?.style.left).toBe("-20000px"); - expect(host?.style.width).toBe("1280px"); - expect(host?.style.opacity).toBe("0"); - }); - - it("parks the resident host when a prepared browser tab is removed", async () => { - const webview = ensureResidentBrowserWebview({ - browserId: "browser-detached", - url: "https://example.com", - }); - const preparation = await prepareResidentBrowserWebviewForPixelCapture({ - browserId: "browser-detached", - }); - const host = document.getElementById("paseo-browser-resident-webviews"); - - removeResidentBrowserWebview("browser-detached"); - - expect(webview?.isConnected).toBe(false); - expect(host?.style.left).toBe("-20000px"); - expect(host?.style.width).toBe("1280px"); - expect(host?.style.opacity).toBe("0"); - await restoreResidentBrowserWebviewAfterPixelCapture(preparation); - expect(host?.style.left).toBe("-20000px"); - }); }); diff --git a/packages/app/src/components/browser-webview-resident.ts b/packages/app/src/components/browser-webview-resident.ts index 3a24abaca..6b1ad0f60 100644 --- a/packages/app/src/components/browser-webview-resident.ts +++ b/packages/app/src/components/browser-webview-resident.ts @@ -1,28 +1,14 @@ -import { getDesktopHost } from "@/desktop/host"; - const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews"; const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id"; const RESIDENT_VIEWPORT_WIDTH = 1280; const RESIDENT_VIEWPORT_HEIGHT = 800; const residentWebviewsByBrowserId = new Map(); -const activeCapturePreparations = new Map(); - -let captureBridgeInstallCount = 0; -let captureBridgeDisposer: (() => void) | null = null; -let nextCapturePreparationId = 0; interface BrowserWebviewElement extends HTMLElement { src: string; } -interface ActiveCapturePreparation { - browserId: string; - requestId?: string; - preparesResidentHost: boolean; - webview: HTMLElement; -} - function trimNonEmpty(value: string | null | undefined): string | null { if (typeof value !== "string") { return null; @@ -36,22 +22,8 @@ function readDocument(): Document | null { } function applyResidentHostParkingStyle(host: HTMLElement): void { - host.setAttribute("aria-hidden", "true"); - host.style.position = "fixed"; - host.style.left = "-20000px"; - host.style.top = "0"; - host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`; - host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`; - host.style.overflow = "hidden"; - host.style.opacity = "0"; - host.style.pointerEvents = "none"; - host.style.zIndex = ""; - host.style.clipPath = ""; - host.style.visibility = ""; - host.style.transform = ""; -} - -function applyResidentHostCaptureStyle(host: HTMLElement): void { + // Parked browser webviews must remain paintable at all times; screenshot + // correctness depends on the proven states in docs/browser-capture-harness.md. host.setAttribute("aria-hidden", "true"); host.style.position = "fixed"; host.style.left = "0"; @@ -61,15 +33,17 @@ function applyResidentHostCaptureStyle(host: HTMLElement): void { host.style.overflow = "hidden"; host.style.opacity = "1"; host.style.pointerEvents = "none"; - host.style.zIndex = "1"; + host.style.display = "block"; + host.style.zIndex = ""; host.style.clipPath = ""; - host.style.visibility = ""; + host.style.visibility = "visible"; host.style.transform = ""; } function getResidentBrowserHost(ownerDocument: Document): HTMLElement { const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID); if (existing) { + applyResidentHostParkingStyle(existing); return existing; } @@ -92,17 +66,6 @@ function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLEle return null; } -function findBrowserWebviewForPixelCapture( - browserId: string, - ownerDocument: Document, -): HTMLElement | null { - const resident = residentWebviewsByBrowserId.get(browserId) ?? null; - if (resident?.isConnected) { - return resident; - } - return findBrowserWebview(browserId, ownerDocument); -} - function applyResidentWebviewStyle(webview: HTMLElement): void { webview.style.display = "inline-flex"; webview.style.flex = "0 0 auto"; @@ -125,116 +88,6 @@ function clearResidentWebviewParkingStyle(webview: HTMLElement): void { webview.style.zIndex = ""; } -function residentWebviewChildren(host: HTMLElement): HTMLElement[] { - const webviews: HTMLElement[] = []; - for (const element of host.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) { - if (element instanceof HTMLElement) { - webviews.push(element); - } - } - return webviews; -} - -function raiseResidentWebviewForCapture(host: HTMLElement, target: HTMLElement): void { - for (const webview of residentWebviewChildren(host)) { - applyResidentWebviewStyle(webview); - } - applyResidentWebviewStyle(target); - target.style.zIndex = "2"; -} - -function nextAnimationFrame(): Promise { - if (typeof requestAnimationFrame !== "function") { - return Promise.resolve(); - } - return new Promise((resolve) => { - requestAnimationFrame(() => { - resolve(); - }); - }); -} - -async function waitForCapturePaint(webview: HTMLElement): Promise { - await nextAnimationFrame(); - await nextAnimationFrame(); - webview.getBoundingClientRect(); -} - -function activeCaptureTokenFor(input: { token?: string; requestId?: string }): string | null { - const token = trimNonEmpty(input.token); - if (token && activeCapturePreparations.has(token)) { - return token; - } - const requestId = trimNonEmpty(input.requestId); - if (!requestId) { - return null; - } - for (const [candidateToken, preparation] of activeCapturePreparations.entries()) { - if (preparation.requestId === requestId) { - return candidateToken; - } - } - return null; -} - -function latestActiveResidentHostPreparation(): ActiveCapturePreparation | null { - let latest: ActiveCapturePreparation | null = null; - for (const preparation of activeCapturePreparations.values()) { - if (preparation.preparesResidentHost) { - latest = preparation; - } - } - return latest; -} - -function syncResidentHostCaptureState(): void { - const host = readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID); - if (!(host instanceof HTMLElement)) { - return; - } - - const latest = latestActiveResidentHostPreparation(); - if (latest) { - applyResidentHostCaptureStyle(host); - raiseResidentWebviewForCapture(host, latest.webview); - return; - } - - applyResidentHostParkingStyle(host); - for (const webview of residentWebviewChildren(host)) { - applyResidentWebviewStyle(webview); - } -} - -function releaseCapturePreparationToken(token: string): void { - const preparation = activeCapturePreparations.get(token); - if (!preparation) { - return; - } - activeCapturePreparations.delete(token); - if (preparation.preparesResidentHost) { - const host = readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID); - if (host instanceof HTMLElement && preparation.webview.parentElement === host) { - applyResidentWebviewStyle(preparation.webview); - } - syncResidentHostCaptureState(); - } -} - -function releaseCapturePreparationsForBrowser(browserId: string): void { - for (const [token, preparation] of activeCapturePreparations.entries()) { - if (preparation.browserId === browserId) { - activeCapturePreparations.delete(token); - } - } - syncResidentHostCaptureState(); -} - -function releaseAllCapturePreparations(): void { - activeCapturePreparations.clear(); - syncResidentHostCaptureState(); -} - export function prepareBrowserWebview( webview: HTMLElement, input: { browserId: string; initialUrl?: string | null }, @@ -293,7 +146,6 @@ export function takeResidentBrowserWebview(browserId: string): HTMLElement | nul } residentWebviewsByBrowserId.delete(normalizedBrowserId); - releaseCapturePreparationsForBrowser(normalizedBrowserId); clearResidentWebviewParkingStyle(webview); return webview; } @@ -314,98 +166,6 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl getResidentBrowserHost(ownerDocument).appendChild(webview); } -export async function prepareResidentBrowserWebviewForPixelCapture(input: { - browserId: string; - requestId?: string; -}): Promise<{ token: string }> { - const browserId = trimNonEmpty(input.browserId); - if (!browserId) { - throw new Error("Browser id is required for pixel capture preparation."); - } - const ownerDocument = readDocument(); - if (!ownerDocument) { - throw new Error("Browser pixel capture preparation requires a document."); - } - - const host = getResidentBrowserHost(ownerDocument); - const webview = findBrowserWebviewForPixelCapture(browserId, ownerDocument); - if (!webview) { - throw new Error(`Browser webview ${browserId} is not mounted.`); - } - - const token = `capture-${++nextCapturePreparationId}`; - const preparesResidentHost = webview.parentElement === host; - const requestId = trimNonEmpty(input.requestId); - activeCapturePreparations.set(token, { - browserId, - ...(requestId ? { requestId } : {}), - preparesResidentHost, - webview, - }); - try { - if (preparesResidentHost) { - applyResidentHostCaptureStyle(host); - raiseResidentWebviewForCapture(host, webview); - } - await waitForCapturePaint(webview); - if (!activeCapturePreparations.has(token)) { - throw new Error("Browser pixel capture preparation was canceled."); - } - return { token }; - } catch (error) { - releaseCapturePreparationToken(token); - throw error; - } -} - -export async function restoreResidentBrowserWebviewAfterPixelCapture(input: { - token: string; -}): Promise { - releaseCapturePreparationToken(input.token); -} - -export async function cancelResidentBrowserWebviewPixelCapture(input: { - requestId?: string; - token?: string; -}): Promise { - const token = activeCaptureTokenFor(input); - if (!token) { - return; - } - releaseCapturePreparationToken(token); -} - -export function installResidentBrowserCaptureBridge(): () => void { - captureBridgeInstallCount += 1; - if (!captureBridgeDisposer) { - const browserBridge = getDesktopHost()?.browser; - const disposePrepare = browserBridge?.onPrepareForPixelCapture?.( - prepareResidentBrowserWebviewForPixelCapture, - ); - const disposeRestore = browserBridge?.onRestorePixelCapture?.( - restoreResidentBrowserWebviewAfterPixelCapture, - ); - const disposeCancel = browserBridge?.onCancelPixelCapture?.( - cancelResidentBrowserWebviewPixelCapture, - ); - captureBridgeDisposer = () => { - disposePrepare?.(); - disposeRestore?.(); - disposeCancel?.(); - }; - } - - return () => { - captureBridgeInstallCount = Math.max(0, captureBridgeInstallCount - 1); - if (captureBridgeInstallCount > 0) { - return; - } - captureBridgeDisposer?.(); - captureBridgeDisposer = null; - releaseAllCapturePreparations(); - }; -} - export function removeResidentBrowserWebview(browserId: string): void { const normalizedBrowserId = trimNonEmpty(browserId); if (!normalizedBrowserId) { @@ -414,7 +174,6 @@ export function removeResidentBrowserWebview(browserId: string): void { const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null; residentWebviewsByBrowserId.delete(normalizedBrowserId); - releaseCapturePreparationsForBrowser(normalizedBrowserId); resident?.remove(); } @@ -423,7 +182,5 @@ export function clearResidentBrowserWebviewsForTests(): void { webview.remove(); } residentWebviewsByBrowserId.clear(); - releaseAllCapturePreparations(); - nextCapturePreparationId = 0; readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove(); } diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index 86245416b..2380fcc81 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -121,10 +121,6 @@ export interface DesktopBrowserShortcutEvent { action: "focus-url"; } -export interface DesktopBrowserPixelCapturePreparation { - token: string; -} - export interface DesktopBrowserNewTabRequestEvent { sourceBrowserId: string; url: string; @@ -148,18 +144,6 @@ export interface DesktopBrowserBridge { ) => Promise; /** Copy element text and/or an image to the system clipboard from main. */ copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise; - onPrepareForPixelCapture?: ( - handler: (input: { - requestId: string; - browserId: string; - }) => Promise, - ) => () => void; - onRestorePixelCapture?: ( - handler: (input: DesktopBrowserPixelCapturePreparation) => Promise, - ) => () => void; - onCancelPixelCapture?: ( - handler: (input: { requestId?: string; token?: string }) => Promise, - ) => () => void; } export interface DesktopInvokeBridge { diff --git a/packages/desktop/capture-harness/index.html b/packages/desktop/capture-harness/index.html index 2541787fc..0efbfafc8 100644 --- a/packages/desktop/capture-harness/index.html +++ b/packages/desktop/capture-harness/index.html @@ -16,13 +16,15 @@ #paseo-browser-resident-webviews { position: fixed; - left: -20000px; + left: 0; top: 0; - width: 1280px; - height: 800px; + width: 1px; + height: 1px; overflow: hidden; - opacity: 0; + opacity: 1; pointer-events: none; + display: block; + visibility: visible; } .capture-harness-webview { @@ -152,23 +154,24 @@ host.style.position = "fixed"; host.style.left = "0"; host.style.top = "0"; - host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`; - host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`; + host.style.width = "1px"; + host.style.height = "1px"; host.style.overflow = "hidden"; host.style.opacity = "1"; host.style.pointerEvents = "none"; - host.style.zIndex = "1"; + host.style.display = "block"; + host.style.zIndex = ""; host.style.clipPath = ""; - host.style.visibility = ""; + host.style.visibility = "visible"; host.style.transform = ""; - host.style.transformOrigin = "0 0"; + host.style.transformOrigin = ""; } function applyPermanentWebviewBase(webview, index) { applyStackedWebviewStyle(webview); - webview.style.opacity = "1"; + webview.style.opacity = ""; webview.style.transform = ""; - webview.style.pointerEvents = "none"; + webview.style.pointerEvents = ""; webview.style.zIndex = "0"; webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`; webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`; @@ -183,9 +186,6 @@ webviews.forEach(applyPermanentWebviewBase); if (stateName === "p1-overflow-1x1") { - host.style.width = "1px"; - host.style.height = "1px"; - host.style.overflow = "hidden"; return state(); } diff --git a/packages/desktop/capture-harness/main.js b/packages/desktop/capture-harness/main.js index bea566a09..84294df4e 100644 --- a/packages/desktop/capture-harness/main.js +++ b/packages/desktop/capture-harness/main.js @@ -13,15 +13,15 @@ const CAPTURE_RETRY_INTERVAL_MS = 200; const REPEAT_COUNT = 5; const FRESH_REPEAT_COUNT = 3; const SOAK_MS = Number(process.env.PASEO_CAPTURE_HARNESS_SOAK_MS || 75000); -const HARNESS_GROUP = process.env.PASEO_CAPTURE_HARNESS_GROUP || "all"; +const HARNESS_GROUP = process.env.PASEO_CAPTURE_HARNESS_GROUP || "permanent-parking"; const PERMANENT_STATE_FILTER = new Set( - (process.env.PASEO_CAPTURE_HARNESS_STATES || "") + (process.env.PASEO_CAPTURE_HARNESS_STATES || "P1") .split(",") .map((state) => state.trim()) .filter(Boolean), ); const PERMANENT_VARIANT_FILTER = new Set( - (process.env.PASEO_CAPTURE_HARNESS_VARIANTS || "") + (process.env.PASEO_CAPTURE_HARNESS_VARIANTS || "attach-off") .split(",") .map((variant) => variant.trim()) .filter(Boolean), @@ -377,14 +377,8 @@ async function readGuestMetrics(contents) { } async function capturePageSequence(contents) { - const previousBackgroundThrottling = contents.getBackgroundThrottling(); - contents.setBackgroundThrottling(false); - try { - contents.invalidate(); - return await withTimeout(contents.capturePage(undefined, { stayHidden: false }), "capturePage"); - } finally { - contents.setBackgroundThrottling(previousBackgroundThrottling); - } + contents.invalidate(); + return await withTimeout(contents.capturePage(undefined, { stayHidden: false }), "capturePage"); } async function captureFullPage(contents) { @@ -426,14 +420,8 @@ async function captureFullPage(contents) { } async function captureFullPageSequence(contents) { - const previousBackgroundThrottling = contents.getBackgroundThrottling(); - contents.setBackgroundThrottling(false); - try { - contents.invalidate(); - return await captureFullPage(contents); - } finally { - contents.setBackgroundThrottling(previousBackgroundThrottling); - } + contents.invalidate(); + return await captureFullPage(contents); } function installHarnessWebviewGuards(win) { @@ -1100,6 +1088,10 @@ async function runPermanentParkingGroup() { } finally { await closeHarnessWindow(keeper); } + const failedResults = results.filter((result) => !result.pass); + if (failedResults.length > 0) { + fail(`permanent parking failed ${failedResults.length}/${results.length} checks`); + } return results; } diff --git a/packages/desktop/src/features/browser-automation/ipc.test.ts b/packages/desktop/src/features/browser-automation/ipc.test.ts index ead07c678..b3ed24994 100644 --- a/packages/desktop/src/features/browser-automation/ipc.test.ts +++ b/packages/desktop/src/features/browser-automation/ipc.test.ts @@ -1,5 +1,5 @@ import type { Rectangle } from "electron"; -import { describe, expect, test, vi } from "vitest"; +import { describe, expect, test } from "vitest"; import type { TabImage } from "./service.js"; import { adaptWebContents } from "./ipc.js"; @@ -14,79 +14,43 @@ class FakeImage implements TabImage { } class FakeDebugger { + public attachedProtocolVersions: string[] = []; + public commands: Array<{ command: string; params: Record }> = []; + public isAttached(): boolean { - return false; + return this.attachedProtocolVersions.length > 0; } - public attach(): void {} + public attach(protocolVersion?: string): void { + this.attachedProtocolVersions.push(protocolVersion ?? ""); + } - public async sendCommand(): Promise { - return {}; + public async sendCommand(command: string, params?: Record): Promise { + this.commands.push({ command, params: params ?? {} }); + return { ok: true }; } } -class FakeHostWebContents { - public readonly sentMessages: Array<{ channel: string; payload: unknown }> = []; - public destroyed = false; - - public constructor(public readonly id: number) {} - - public isDestroyed(): boolean { - return this.destroyed; - } - - public send(channel: string, payload: unknown): void { - this.sentMessages.push({ channel, payload }); - } -} - -interface FakeIpcEvent { - sender: { - id: number; - }; -} - -type IpcListener = (event: FakeIpcEvent, payload: unknown) => void; - -class FakeIpcBridge { - private readonly listeners = new Map(); - - public on(channel: string, listener: IpcListener): void { - const listeners = this.listeners.get(channel) ?? []; - listeners.push(listener); - this.listeners.set(channel, listeners); - } - - public removeListener(channel: string, listener: IpcListener): void { - const listeners = this.listeners.get(channel) ?? []; - this.listeners.set( - channel, - listeners.filter((candidate) => candidate !== listener), - ); - } - - public emit(channel: string, payload: unknown, input: { senderId?: number } = {}): void { - const event = { sender: { id: input.senderId ?? 10 } }; - for (const listener of this.listeners.get(channel) ?? []) { - listener(event, payload); - } - } - - public listenerCount(channel: string): number { - return this.listeners.get(channel)?.length ?? 0; - } -} +type ConsoleMessageListener = ( + event: unknown, + level: unknown, + message: unknown, + line: unknown, + sourceId: unknown, +) => void; class FakeWebContents { public readonly debugger = new FakeDebugger(); - public readonly consoleMessages: unknown[] = []; - public readonly destroyedListeners: Array<() => void> = []; + public readonly captures: Array<{ + rect: Rectangle | undefined; + options: { stayHidden?: boolean } | undefined; + }> = []; + public readonly invalidations: string[] = []; + private consoleMessageListener: ConsoleMessageListener | null = null; + private destroyedListener: (() => void) | null = null; public destroyed = false; - public constructor( - public readonly id: number, - public hostWebContents: FakeHostWebContents | null, - ) {} + public constructor(public readonly id: number) {} public getURL(): string { return "https://example.com"; @@ -125,362 +89,96 @@ class FakeWebContents { public reload(): void {} public async capturePage( - _rect?: Rectangle, - _options?: { stayHidden?: boolean }, + rect?: Rectangle, + options?: { stayHidden?: boolean }, ): Promise { + this.captures.push({ rect, options }); return new FakeImage(); } - public invalidate(): void {} - - public getBackgroundThrottling(): boolean { - return true; + public invalidate(): void { + this.invalidations.push("invalidate"); } - public setBackgroundThrottling(): void {} - - public on( - event: "console-message", - listener: ( - event: unknown, - level: unknown, - message: unknown, - line: unknown, - sourceId: unknown, - ) => void, - ): void { - this.consoleMessages.push({ event, listener }); + public on(event: "console-message", listener: ConsoleMessageListener): void { + expect(event).toBe("console-message"); + this.consoleMessageListener = listener; } public once(event: "destroyed", listener: () => void): void { expect(event).toBe("destroyed"); - this.destroyedListeners.push(listener); + this.destroyedListener = listener; + } + + public emitConsoleMessage(input: { + level: unknown; + message: unknown; + line: unknown; + sourceId: unknown; + }): void { + if (!this.consoleMessageListener) { + throw new Error("Console listener was not registered"); + } + this.consoleMessageListener({}, input.level, input.message, input.line, input.sourceId); + } + + public destroy(): void { + this.destroyed = true; + this.destroyedListener?.(); } } describe("browser automation IPC adapter", () => { - test("prepareForPixelCapture asks the embedder renderer and resolves the ack token", async () => { - const host = new FakeHostWebContents(10); - const contents = new FakeWebContents(20, host); - const ipc = new FakeIpcBridge(); - const tab = adaptWebContents(contents, "browser-a", { - ipc, - createRequestId: () => "prepare-1", + test("delegates viewport capture to the guest without a renderer prep bridge", async () => { + const contents = new FakeWebContents(20); + const tab = adaptWebContents(contents); + + const image = await tab.capturePage({ stayHidden: false }); + tab.invalidate(); + + expect(image.getSize()).toEqual({ width: 640, height: 480 }); + expect(contents.captures).toEqual([{ rect: undefined, options: { stayHidden: false } }]); + expect(contents.invalidations).toEqual(["invalidate"]); + }); + + test("collects console messages until the guest is destroyed", () => { + const contents = new FakeWebContents(21); + const tab = adaptWebContents(contents); + + contents.emitConsoleMessage({ + level: "warning", + message: "hello", + line: 12, + sourceId: "https://example.com/app.js", }); - const preparation = tab.prepareForPixelCapture(); - - expect(host.sentMessages).toEqual([ + expect(tab.getConsoleMessages?.()).toEqual([ { - channel: "paseo:browser:capture-prepare", - payload: { requestId: "prepare-1", browserId: "browser-a" }, + level: "warning", + message: "hello", + line: 12, + source: "https://example.com/app.js", + timestamp: expect.any(Number), }, ]); - ipc.emit("paseo:browser:capture-prepared", { - requestId: "other", - ok: true, - token: "wrong-token", - }); - expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1); - ipc.emit("paseo:browser:capture-prepared", { - requestId: "prepare-1", - ok: true, - token: "token-a", - }); + contents.destroy(); - await expect(preparation).resolves.toEqual({ token: "token-a" }); - expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0); + expect(tab.getConsoleMessages?.()).toEqual([]); }); - test("restorePixelCapture sends the capture token back to the embedder renderer", async () => { - const host = new FakeHostWebContents(10); - const contents = new FakeWebContents(20, host); - const ipc = new FakeIpcBridge(); - const requestIds = ["prepare-1", "restore-1"]; - const tab = adaptWebContents(contents, "browser-a", { - ipc, - createRequestId: () => { - const requestId = requestIds.shift(); - if (!requestId) { - throw new Error("Missing request id"); - } - return requestId; - }, + test("attaches the debugger before sending a CDP command", async () => { + const contents = new FakeWebContents(22); + const tab = adaptWebContents(contents); + + const result = await tab.sendDebugCommand?.("Page.captureScreenshot", { + format: "png", }); - const preparation = tab.prepareForPixelCapture(); - ipc.emit("paseo:browser:capture-prepared", { - requestId: "prepare-1", - ok: true, - token: "token-a", - }); - await expect(preparation).resolves.toEqual({ token: "token-a" }); - - const restored = tab.restorePixelCapture({ token: "token-a" }); - - expect(host.sentMessages).toEqual([ - { - channel: "paseo:browser:capture-prepare", - payload: { requestId: "prepare-1", browserId: "browser-a" }, - }, - { - channel: "paseo:browser:capture-restore", - payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" }, - }, + expect(result).toEqual({ ok: true }); + expect(contents.debugger.attachedProtocolVersions).toEqual(["1.3"]); + expect(contents.debugger.commands).toEqual([ + { command: "Page.captureScreenshot", params: { format: "png" } }, ]); - ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true }); - - await expect(restored).resolves.toBeUndefined(); - }); - - test("restorePixelCapture uses the host captured during preparation when the guest detaches", async () => { - const host = new FakeHostWebContents(10); - const contents = new FakeWebContents(20, host); - const ipc = new FakeIpcBridge(); - const requestIds = ["prepare-1", "restore-1"]; - const tab = adaptWebContents(contents, "browser-a", { - ipc, - createRequestId: () => { - const requestId = requestIds.shift(); - if (!requestId) { - throw new Error("Missing request id"); - } - return requestId; - }, - }); - - const preparationPromise = tab.prepareForPixelCapture(); - ipc.emit("paseo:browser:capture-prepared", { - requestId: "prepare-1", - ok: true, - token: "token-a", - }); - const preparation = await preparationPromise; - contents.hostWebContents = null; - - const restored = tab.restorePixelCapture(preparation); - - expect(host.sentMessages).toEqual([ - { - channel: "paseo:browser:capture-prepare", - payload: { requestId: "prepare-1", browserId: "browser-a" }, - }, - { - channel: "paseo:browser:capture-restore", - payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" }, - }, - ]); - ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true }); - - await expect(restored).resolves.toBeUndefined(); - await expect(tab.restorePixelCapture(preparation)).rejects.toThrow( - "Browser pixel capture preparation is no longer active.", - ); - }); - - test("prepareForPixelCapture rejects when the renderer reports preparation failure", async () => { - const host = new FakeHostWebContents(10); - const contents = new FakeWebContents(20, host); - const ipc = new FakeIpcBridge(); - const tab = adaptWebContents(contents, "browser-a", { - ipc, - createRequestId: () => "prepare-1", - }); - - const preparation = tab.prepareForPixelCapture(); - ipc.emit("paseo:browser:capture-prepared", { - requestId: "prepare-1", - ok: false, - message: "renderer could not prep", - }); - - await expect(preparation).rejects.toThrow("renderer could not prep"); - expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0); - }); - - test("prepareForPixelCapture waits for the guest host renderer before asking for prep", async () => { - vi.useFakeTimers(); - try { - const host = new FakeHostWebContents(10); - const contents = new FakeWebContents(20, null); - const ipc = new FakeIpcBridge(); - const tab = adaptWebContents(contents, "browser-a", { - ipc, - createRequestId: () => "prepare-1", - timeoutMs: 250, - }); - - const preparation = tab.prepareForPixelCapture(); - await vi.advanceTimersByTimeAsync(49); - - expect(host.sentMessages).toEqual([]); - - contents.hostWebContents = host; - await vi.advanceTimersByTimeAsync(1); - - expect(host.sentMessages).toEqual([ - { - channel: "paseo:browser:capture-prepare", - payload: { requestId: "prepare-1", browserId: "browser-a" }, - }, - ]); - ipc.emit("paseo:browser:capture-prepared", { - requestId: "prepare-1", - ok: true, - token: "token-a", - }); - - await expect(preparation).resolves.toEqual({ token: "token-a" }); - } finally { - vi.useRealTimers(); - } - }); - - test("prepareForPixelCapture retries when the renderer prep handler is not registered yet", async () => { - vi.useFakeTimers(); - try { - const host = new FakeHostWebContents(10); - const contents = new FakeWebContents(20, host); - const ipc = new FakeIpcBridge(); - const requestIds = ["prepare-1", "prepare-2"]; - const tab = adaptWebContents(contents, "browser-a", { - ipc, - createRequestId: () => { - const requestId = requestIds.shift(); - if (!requestId) { - throw new Error("Missing request id"); - } - return requestId; - }, - timeoutMs: 250, - }); - - const preparation = tab.prepareForPixelCapture(); - - expect(host.sentMessages).toEqual([ - { - channel: "paseo:browser:capture-prepare", - payload: { requestId: "prepare-1", browserId: "browser-a" }, - }, - ]); - ipc.emit("paseo:browser:capture-prepared", { - requestId: "prepare-1", - ok: false, - message: "Browser pixel capture preparation is unavailable.", - }); - - await vi.advanceTimersByTimeAsync(50); - - expect(host.sentMessages).toEqual([ - { - channel: "paseo:browser:capture-prepare", - payload: { requestId: "prepare-1", browserId: "browser-a" }, - }, - { - channel: "paseo:browser:capture-prepare", - payload: { requestId: "prepare-2", browserId: "browser-a" }, - }, - ]); - ipc.emit("paseo:browser:capture-prepared", { - requestId: "prepare-2", - ok: true, - token: "token-a", - }); - - await expect(preparation).resolves.toEqual({ token: "token-a" }); - } finally { - vi.useRealTimers(); - } - }); - - test("prepareForPixelCapture ignores matching responses from the wrong sender", async () => { - const host = new FakeHostWebContents(10); - const contents = new FakeWebContents(20, host); - const ipc = new FakeIpcBridge(); - const tab = adaptWebContents(contents, "browser-a", { - ipc, - createRequestId: () => "prepare-1", - }); - - const preparation = tab.prepareForPixelCapture(); - ipc.emit( - "paseo:browser:capture-prepared", - { - requestId: "prepare-1", - ok: true, - token: "spoofed-token", - }, - { senderId: 99 }, - ); - expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1); - - ipc.emit("paseo:browser:capture-prepared", { - requestId: "prepare-1", - ok: true, - token: "token-a", - }); - - await expect(preparation).resolves.toEqual({ token: "token-a" }); - }); - - test("prepareForPixelCapture cancels the renderer request when the ack times out", async () => { - vi.useFakeTimers(); - try { - const host = new FakeHostWebContents(10); - const contents = new FakeWebContents(20, host); - const ipc = new FakeIpcBridge(); - const tab = adaptWebContents(contents, "browser-a", { - ipc, - createRequestId: () => "prepare-1", - timeoutMs: 25, - }); - - const preparation = tab.prepareForPixelCapture(); - const rejection = expect(preparation).rejects.toThrow( - "Browser pixel capture prepare timed out.", - ); - await vi.advanceTimersByTimeAsync(25); - - await rejection; - expect(host.sentMessages).toEqual([ - { - channel: "paseo:browser:capture-prepare", - payload: { requestId: "prepare-1", browserId: "browser-a" }, - }, - { - channel: "paseo:browser:capture-cancel", - payload: { requestId: "prepare-1", browserId: "browser-a" }, - }, - ]); - expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0); - } finally { - vi.useRealTimers(); - } - }); - - test("prepareForPixelCapture reports prep_unavailable when the guest host renderer never appears", async () => { - vi.useFakeTimers(); - try { - const contents = new FakeWebContents(20, null); - const tab = adaptWebContents(contents, "browser-a", { timeoutMs: 25 }); - - const preparation = tab.prepareForPixelCapture(); - const rejection = expect(preparation).rejects.toThrow( - "Browser screenshot prep_unavailable: Browser host renderer is not available.", - ); - await vi.advanceTimersByTimeAsync(25); - - await rejection; - } finally { - vi.useRealTimers(); - } - }); - - test("prepareForPixelCapture rejects when the guest has no embedder renderer", async () => { - const contents = new FakeWebContents(20, null); - const tab = adaptWebContents(contents, "browser-a", { timeoutMs: 1 }); - - await expect(tab.prepareForPixelCapture()).rejects.toThrow("prep_unavailable"); }); }); diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 16925ee9e..00a56b28b 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -13,36 +13,13 @@ import { } from "../browser-webviews/index.js"; const MAX_CONSOLE_MESSAGES_PER_TAB = 200; -const PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS = 5_000; -const PIXEL_CAPTURE_PREP_RETRY_INTERVAL_MS = 50; -const PIXEL_CAPTURE_PREP_UNAVAILABLE_PREFIX = "Browser screenshot prep_unavailable:"; const consoleMessagesByContentsId = new Map(); const observedContentsIds = new Set(); -let nextPixelCaptureBridgeRequest = 0; interface IpcHandlerRegistry { handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void; } -interface IpcBridgeEvent { - sender?: { - id?: number; - }; -} - -type IpcListener = (event: IpcBridgeEvent, payload: unknown) => void; - -interface IpcCaptureBridge { - on(channel: string, listener: IpcListener): void; - removeListener(channel: string, listener: IpcListener): void; -} - -interface HostWebContents { - readonly id: number; - isDestroyed(): boolean; - send(channel: string, payload: unknown): void; -} - interface WebContentsDebugger { isAttached(): boolean; attach(protocolVersion?: string): void; @@ -65,7 +42,6 @@ interface ConsoleMessageEmitter { interface BrowserAutomationWebContents extends ConsoleMessageEmitter { readonly id: number; - readonly hostWebContents: HostWebContents | null; readonly debugger: WebContentsDebugger; getURL(): string; getTitle(): string; @@ -80,34 +56,10 @@ interface BrowserAutomationWebContents extends ConsoleMessageEmitter { reload(): void; capturePage(rect?: Rectangle, options?: { stayHidden?: boolean }): Promise; invalidate(): void; - getBackgroundThrottling(): boolean; - setBackgroundThrottling(allowed: boolean): void; } -type PixelCaptureBridgeKind = "prepare" | "restore"; - -interface PixelCaptureBridgeSuccess { - token?: string; -} - -interface PixelCaptureBridgeOptions { - ipc?: IpcCaptureBridge; - createRequestId?: () => string; - timeoutMs?: number; -} - -interface PreparedPixelCapture { - browserId: string; - host: HostWebContents; -} - -export function adaptWebContents( - contents: BrowserAutomationWebContents, - browserId: string, - options?: PixelCaptureBridgeOptions, -): TabContents { +export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents { observeConsoleMessages(contents); - const preparedPixelCapturesByToken = new Map(); return { id: contents.id, getURL: () => contents.getURL(), @@ -122,34 +74,7 @@ export function adaptWebContents( goForward: () => contents.goForward(), reload: () => contents.reload(), capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions), - prepareForPixelCapture: async () => { - const result = await preparePixelCaptureBridgeWithRetry({ contents, browserId, options }); - if (!result.token) { - throw new Error("Browser pixel capture preparation did not return a token."); - } - preparedPixelCapturesByToken.set(result.token, { browserId, host: result.host }); - return { token: result.token }; - }, - restorePixelCapture: async (preparation) => { - const prepared = preparedPixelCapturesByToken.get(preparation.token); - if (!prepared) { - throw new Error("Browser pixel capture preparation is no longer active."); - } - try { - await requestPixelCaptureBridge({ - host: prepared.host, - browserId: prepared.browserId, - kind: "restore", - options, - extraPayload: { token: preparation.token }, - }); - } finally { - preparedPixelCapturesByToken.delete(preparation.token); - } - }, invalidate: () => contents.invalidate(), - isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(), - setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed), getConsoleMessages: () => consoleMessagesByContentsId.get(contents.id) ?? [], sendDebugCommand: async (command: string, params?: Record) => { if (!contents.debugger.isAttached()) { @@ -160,204 +85,6 @@ export function adaptWebContents( }; } -function getPixelCaptureHost(contents: BrowserAutomationWebContents): HostWebContents | null { - const host = contents.hostWebContents; - if (!host || host.isDestroyed()) { - return null; - } - return host; -} - -async function preparePixelCaptureBridgeWithRetry(input: { - contents: BrowserAutomationWebContents; - browserId: string; - options: PixelCaptureBridgeOptions | undefined; -}): Promise { - const timeoutMs = input.options?.timeoutMs ?? PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS; - const deadline = Date.now() + timeoutMs; - let lastUnavailableReason = "Browser host renderer is not available."; - - while (Date.now() < deadline) { - const host = getPixelCaptureHost(input.contents); - if (!host) { - await delayUntilRetry(deadline); - continue; - } - - try { - const result = await requestPixelCaptureBridge({ - host, - browserId: input.browserId, - kind: "prepare", - options: withBridgeTimeout(input.options, Math.max(1, deadline - Date.now())), - }); - return { ...result, host }; - } catch (error) { - if (!isRetryablePrepareUnavailableError(error)) { - throw error; - } - lastUnavailableReason = errorMessage(error); - await delayUntilRetry(deadline); - } - } - - throw new Error(`${PIXEL_CAPTURE_PREP_UNAVAILABLE_PREFIX} ${lastUnavailableReason}`); -} - -function withBridgeTimeout( - options: PixelCaptureBridgeOptions | undefined, - timeoutMs: number, -): PixelCaptureBridgeOptions { - return { - ...options, - timeoutMs, - }; -} - -function isRetryablePrepareUnavailableError(error: unknown): boolean { - const message = errorMessage(error); - return ( - message.includes("Browser pixel capture preparation is unavailable.") || - message.includes("Browser host renderer is not available.") || - message.includes("is not mounted.") - ); -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -async function delayUntilRetry(deadline: number): Promise { - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) { - return; - } - await new Promise((resolve) => { - setTimeout(resolve, Math.min(PIXEL_CAPTURE_PREP_RETRY_INTERVAL_MS, remainingMs)); - }); -} - -function requestPixelCaptureBridge(input: { - host: HostWebContents; - browserId: string; - kind: PixelCaptureBridgeKind; - options: PixelCaptureBridgeOptions | undefined; - extraPayload?: { token: string }; -}): Promise { - const { host, browserId, kind, options, extraPayload } = input; - if (host.isDestroyed()) { - return Promise.reject(new Error("Browser host renderer is not available.")); - } - - const ipc = options?.ipc ?? ipcMain; - const requestId = - options?.createRequestId?.() ?? `browser-pixel-capture-${++nextPixelCaptureBridgeRequest}`; - const timeoutMs = options?.timeoutMs ?? PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS; - const requestChannel = - kind === "prepare" ? "paseo:browser:capture-prepare" : "paseo:browser:capture-restore"; - const responseChannel = - kind === "prepare" ? "paseo:browser:capture-prepared" : "paseo:browser:capture-restored"; - - return new Promise((resolve, reject) => { - let timeoutId: ReturnType | undefined; - const cleanup = () => { - if (timeoutId) { - clearTimeout(timeoutId); - } - ipc.removeListener(responseChannel, listener); - }; - const listener: IpcListener = (event, payload) => { - if (readSenderId(event) !== host.id) { - return; - } - const response = readPixelCaptureBridgeResponse(payload, requestId); - if (!response) { - return; - } - cleanup(); - if (response.ok) { - resolve(response); - } else { - reject(new Error(response.message)); - } - }; - - ipc.on(responseChannel, listener); - timeoutId = setTimeout(() => { - cleanup(); - sendPixelCaptureCancel(host, { - requestId, - browserId, - ...(extraPayload ? { token: extraPayload.token } : {}), - }); - reject(new Error(`Browser pixel capture ${kind} timed out.`)); - }, timeoutMs); - - try { - host.send(requestChannel, { - requestId, - browserId, - ...(extraPayload ? { token: extraPayload.token } : {}), - }); - } catch (error) { - cleanup(); - reject(error); - } - }); -} - -function sendPixelCaptureCancel( - host: HostWebContents, - payload: { browserId: string; requestId?: string; token?: string }, -): void { - if (host.isDestroyed()) { - return; - } - try { - host.send("paseo:browser:capture-cancel", payload); - } catch { - // The original prepare/restore request owns the user-visible error. - } -} - -function readSenderId(event: IpcBridgeEvent): number | null { - const senderId = event.sender?.id; - return typeof senderId === "number" ? senderId : null; -} - -function readPixelCaptureBridgeResponse( - payload: unknown, - requestId: string, -): ({ ok: true } & PixelCaptureBridgeSuccess) | { ok: false; message: string } | null { - if (!isRecord(payload)) { - return null; - } - const record = payload; - if (record.requestId !== requestId) { - return null; - } - if (record.ok === true) { - return { - ok: true, - ...(typeof record.token === "string" && record.token.length > 0 - ? { token: record.token } - : {}), - }; - } - if (record.ok === false) { - const message = - typeof record.message === "string" && record.message.length > 0 - ? record.message - : "Browser pixel capture bridge failed."; - return { ok: false, message }; - } - return null; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - function observeConsoleMessages(contents: BrowserAutomationWebContents): void { if (observedContentsIds.has(contents.id)) { return; @@ -398,7 +125,7 @@ function createRegistry(): BrowserRegistry { listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace, getTabContents(browserId: string): TabContents | null { const contents = getPaseoBrowserWebContents(browserId); - return contents ? adaptWebContents(contents, browserId) : null; + return contents ? adaptWebContents(contents) : null; }, getBrowserWorkspaceId: getPaseoBrowserWorkspaceId, getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId, diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index 28ef2c598..c89b696f8 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -7,12 +7,7 @@ import type { BrowserAutomationExecuteRequest, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { BrowserSnapshotEngine } from "./snapshot-engine.js"; -import type { - BrowserRegistry, - TabContents, - TabImage, - TabPixelCapturePreparation, -} from "./service.js"; +import type { BrowserRegistry, TabContents, TabImage } from "./service.js"; import { executeAutomationCommand } from "./service.js"; const BROWSER_A = "11111111-1111-4111-8111-111111111111"; @@ -41,7 +36,6 @@ class FakeTab implements TabContents { public readonly actions: string[] = []; public readonly capturedViewports: Array<{ stayHidden?: boolean }> = []; public readonly debugCommands: Array<{ command: string; params?: Record }> = []; - public readonly restoredPixelCaptureTokens: string[] = []; private readonly captureStartWaiters: Array<() => void> = []; private readonly deferredCaptures: Array<(image: TabImage) => void> = []; @@ -56,8 +50,6 @@ class FakeTab implements TabContents { public captureErrorMessage = "capture failed"; public viewportCaptureFailuresBeforeSuccess = 0; public deferCaptures = false; - public prepareNeverAcks = false; - public prepareErrorMessage: string | null = null; public fullPageScreenshotThrows = false; public fullPageScreenshotErrorMessage = "UnknownVizError"; public fullPageCaptureFailuresBeforeSuccess = 0; @@ -68,8 +60,6 @@ class FakeTab implements TabContents { public fullPageScreenshotData = "fullPagePng"; public documentNodeId = 1; public queriedNodeId = 2; - public backgroundThrottlingAllowed = true; - private nextPixelCapturePreparationId = 0; public constructor( public readonly id: number, @@ -153,36 +143,10 @@ class FakeTab implements TabContents { return new FakeImage(); } - public async prepareForPixelCapture(): Promise { - this.actions.push("prepare"); - if (this.prepareErrorMessage) { - throw new Error(this.prepareErrorMessage); - } - if (this.prepareNeverAcks) { - return new Promise(() => {}); - } - const token = `capture-${++this.nextPixelCapturePreparationId}`; - return { token }; - } - - public async restorePixelCapture(preparation: TabPixelCapturePreparation): Promise { - this.actions.push(`restore:${preparation.token}`); - this.restoredPixelCaptureTokens.push(preparation.token); - } - public invalidate(): void { this.actions.push("invalidate"); } - public isBackgroundThrottlingAllowed(): boolean { - return this.backgroundThrottlingAllowed; - } - - public setBackgroundThrottling(allowed: boolean): void { - this.backgroundThrottlingAllowed = allowed; - this.actions.push(`background:${allowed}`); - } - public getConsoleMessages(): BrowserAutomationConsoleLogEntry[] { return this.consoleMessages; } @@ -913,7 +877,7 @@ describe("executeAutomationCommand", () => { }); }); - test("screenshot serializes the painted viewport and restores throttling", async () => { + test("screenshot captures the painted viewport", async () => { const browser = new BrowserAutomationHarness(); const result = await browser.execute({ @@ -934,14 +898,7 @@ describe("executeAutomationCommand", () => { }, }); expect(browser.tab.capturedViewports).toEqual([{ stayHidden: false }]); - expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-1", - "background:true", - ]); + expect(browser.tab.actions).toEqual(["invalidate", "capture"]); }); test("screenshot returns no-frame when the viewport never paints", async () => { @@ -961,25 +918,17 @@ describe("executeAutomationCommand", () => { ok: false, error: { code: "screenshot_no_frame", - message: - "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, + message: "The tab has not painted yet. Retry the screenshot.", + retryable: true, }, }); - expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-1", - "background:true", - ]); + expect(browser.tab.actions).toEqual(["invalidate", "capture"]); } finally { vi.useRealTimers(); } }); - test("screenshot restores capture preparation when viewport capture fails with an ordinary error", async () => { + test("screenshot surfaces ordinary viewport capture errors", async () => { const browser = new BrowserAutomationHarness(); browser.tab.captureThrows = true; @@ -990,14 +939,7 @@ describe("executeAutomationCommand", () => { }), ).rejects.toThrow("capture failed"); - expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-1", - "background:true", - ]); + expect(browser.tab.actions).toEqual(["invalidate", "capture"]); }); test("screenshot retries UnknownVizError until the first viewport frame appears", async () => { @@ -1027,16 +969,12 @@ describe("executeAutomationCommand", () => { }); expect(browser.tab.capturedViewports).toHaveLength(3); expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", "invalidate", "capture", "invalidate", "capture", "invalidate", "capture", - "restore:capture-1", - "background:true", ]); } finally { vi.useRealTimers(); @@ -1061,74 +999,20 @@ describe("executeAutomationCommand", () => { ok: false, error: { code: "screenshot_no_frame", - message: - "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, + message: "The tab has not painted yet. Retry the screenshot.", + retryable: true, }, }); expect(browser.tab.capturedViewports.length).toBeGreaterThan(1); expect(browser.tab.actions.filter((action) => action === "invalidate")).toHaveLength( browser.tab.capturedViewports.length, ); - expect(browser.tab.restoredPixelCaptureTokens).toEqual(["capture-1"]); - expect(browser.tab.actions.at(-2)).toBe("restore:capture-1"); - expect(browser.tab.actions.at(-1)).toBe("background:true"); } finally { vi.useRealTimers(); } }); - test("screenshot returns no-frame when capture preparation does not ack", async () => { - vi.useFakeTimers(); - try { - const browser = new BrowserAutomationHarness(); - browser.tab.prepareNeverAcks = true; - - const resultPromise = browser.execute({ - command: "screenshot", - args: { browserId: BROWSER_A }, - }); - await vi.advanceTimersByTimeAsync(5_000); - - await expect(resultPromise).resolves.toEqual({ - requestId: "req-screenshot", - ok: false, - error: { - code: "screenshot_no_frame", - message: - "Browser screenshot prep_unavailable: the app renderer did not acknowledge capture preparation before the timeout.", - retryable: false, - }, - }); - expect(browser.tab.actions).toEqual(["prepare", "background:true"]); - } finally { - vi.useRealTimers(); - } - }); - - test("screenshot returns prep_unavailable and does not capture when preparation cannot be honored", async () => { - const browser = new BrowserAutomationHarness(); - browser.tab.prepareErrorMessage = - "Browser screenshot prep_unavailable: Browser host renderer is not available."; - - const result = await browser.execute({ - command: "screenshot", - args: { browserId: BROWSER_A }, - }); - - expect(result).toEqual({ - requestId: "req-screenshot", - ok: false, - error: { - code: "screenshot_no_frame", - message: "Browser screenshot prep_unavailable: Browser host renderer is not available.", - retryable: false, - }, - }); - expect(browser.tab.actions).toEqual(["prepare", "background:true"]); - }); - - test("overlapping screenshots serialize capture preparation and restore", async () => { + test("overlapping screenshots serialize through the shared capture queue", async () => { const browser = new BrowserAutomationHarness(); browser.tab.deferCaptures = true; @@ -1147,44 +1031,20 @@ describe("executeAutomationCommand", () => { ); await Promise.resolve(); - expect(browser.tab.actions).toEqual(["prepare", "background:false", "invalidate", "capture"]); + expect(browser.tab.actions).toEqual(["invalidate", "capture"]); browser.tab.finishNextCapture(); await browser.tab.waitForCaptureStart(2); - expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-1", - "background:true", - "prepare", - "background:false", - "invalidate", - "capture", - ]); + expect(browser.tab.actions).toEqual(["invalidate", "capture", "invalidate", "capture"]); browser.tab.finishNextCapture(); await expect(first).resolves.toMatchObject({ requestId: "req-screenshot", ok: true }); await expect(second).resolves.toMatchObject({ requestId: "req-screenshot-2", ok: true }); - expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-1", - "background:true", - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-2", - "background:true", - ]); + expect(browser.tab.actions).toEqual(["invalidate", "capture", "invalidate", "capture"]); }); - test("overlapping screenshots across browser tabs serialize capture preparation and restore", async () => { + test("overlapping screenshots across browser tabs serialize through the shared capture queue", async () => { const registry = new FakeRegistry(); const firstTab = new FakeTab(1, "https://a.test", "A"); const secondTab = new FakeTab(2, "https://b.test", "B"); @@ -1214,33 +1074,19 @@ describe("executeAutomationCommand", () => { ); await Promise.resolve(); - expect(firstTab.actions).toEqual(["prepare", "background:false", "invalidate", "capture"]); + expect(firstTab.actions).toEqual(["invalidate", "capture"]); expect(secondTab.actions).toEqual([]); firstTab.finishNextCapture(); await secondTab.waitForCaptureStart(1); - expect(firstTab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-1", - "background:true", - ]); - expect(secondTab.actions).toEqual(["prepare", "background:false", "invalidate", "capture"]); + expect(firstTab.actions).toEqual(["invalidate", "capture"]); + expect(secondTab.actions).toEqual(["invalidate", "capture"]); secondTab.finishNextCapture(); await expect(first).resolves.toMatchObject({ requestId: "req-screenshot", ok: true }); await expect(second).resolves.toMatchObject({ requestId: "req-screenshot-2", ok: true }); - expect(secondTab.actions).toEqual([ - "prepare", - "background:false", - "invalidate", - "capture", - "restore:capture-1", - "background:true", - ]); + expect(secondTab.actions).toEqual(["invalidate", "capture"]); }); test("screenshot with fullPage captures the page content area through CDP", async () => { @@ -1275,17 +1121,13 @@ describe("executeAutomationCommand", () => { }, ]); expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", "invalidate", "debug:Page.getLayoutMetrics", "debug:Page.captureScreenshot", - "restore:capture-1", - "background:true", ]); }); - test("screenshot with fullPage restores capture preparation when CDP returns no image", async () => { + test("screenshot with fullPage returns unsupported when CDP returns no image", async () => { const browser = new BrowserAutomationHarness(); browser.tab.fullPageScreenshotData = ""; @@ -1304,13 +1146,9 @@ describe("executeAutomationCommand", () => { }, }); expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", "invalidate", "debug:Page.getLayoutMetrics", "debug:Page.captureScreenshot", - "restore:capture-1", - "background:true", ]); }); @@ -1339,23 +1177,19 @@ describe("executeAutomationCommand", () => { }, }); expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", "invalidate", "debug:Page.getLayoutMetrics", "debug:Page.captureScreenshot", "invalidate", "debug:Page.getLayoutMetrics", "debug:Page.captureScreenshot", - "restore:capture-1", - "background:true", ]); } finally { vi.useRealTimers(); } }); - test("screenshot with fullPage restores capture preparation when CDP capture fails with an ordinary error", async () => { + test("screenshot with fullPage surfaces ordinary CDP capture errors", async () => { const browser = new BrowserAutomationHarness(); browser.tab.fullPageScreenshotThrows = true; browser.tab.fullPageScreenshotErrorMessage = "Debugger detached"; @@ -1367,13 +1201,9 @@ describe("executeAutomationCommand", () => { }), ).rejects.toThrow("Debugger detached"); expect(browser.tab.actions).toEqual([ - "prepare", - "background:false", "invalidate", "debug:Page.getLayoutMetrics", "debug:Page.captureScreenshot", - "restore:capture-1", - "background:true", ]); }); diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index 3bf4b7838..3a59bf447 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -24,19 +24,11 @@ export interface TabContents { goForward(): void; reload(): void; capturePage(options?: TabCapturePageOptions): Promise; - prepareForPixelCapture(): Promise; - restorePixelCapture(preparation: TabPixelCapturePreparation): Promise; invalidate(): void; - isBackgroundThrottlingAllowed(): boolean; - setBackgroundThrottling(allowed: boolean): void; getConsoleMessages?(): BrowserAutomationConsoleLogEntry[]; sendDebugCommand?(command: string, params?: Record): Promise; } -export interface TabPixelCapturePreparation { - token: string; -} - export interface TabImage { toPNG(): Uint8Array; getSize(): { width: number; height: number }; @@ -62,9 +54,7 @@ const DEFAULT_WAIT_TIMEOUT_MS = 5_000; const WAIT_POLL_INTERVAL_MS = 25; const PIXEL_CAPTURE_TIMEOUT_MS = 5_000; const PIXEL_CAPTURE_RETRY_INTERVAL_MS = 200; -const SCREENSHOT_NO_FRAME_MESSAGE = - "The browser tab has no painted frame. Focus the tab in the app, then try again."; -const SCREENSHOT_PREP_UNAVAILABLE_PREFIX = "Browser screenshot prep_unavailable:"; +const SCREENSHOT_NO_FRAME_MESSAGE = "The tab has not painted yet. Retry the screenshot."; const ALLOWED_PAGE_URL_PROTOCOLS = new Set(["http:", "https:"]); let pixelCaptureQueue: Promise = Promise.resolve(); @@ -92,7 +82,7 @@ function screenshotNoFrameFailure( requestId: string, error: ScreenshotNoFrameError, ): FailurePayload { - return fail(requestId, "screenshot_no_frame", error.message); + return fail(requestId, "screenshot_no_frame", error.message, true); } async function withPixelCaptureTimeout( @@ -138,26 +128,7 @@ async function runSerializedPixelCapture(capture: () => Promise): Promise< } } -async function prepareForPixelCapture(contents: TabContents): Promise { - try { - return await withPixelCaptureTimeout(contents.prepareForPixelCapture()); - } catch (error) { - throw screenshotPreparationError(error); - } -} - -async function restorePixelCapture( - contents: TabContents, - preparation: TabPixelCapturePreparation, -): Promise { - try { - await withPixelCaptureTimeout(contents.restorePixelCapture(preparation)); - } catch { - throw new ScreenshotNoFrameError(); - } -} - -async function capturePreparedPixelFrame( +async function capturePixelFrameWithRetry( contents: TabContents, capture: () => Promise, ): Promise { @@ -179,26 +150,6 @@ async function capturePreparedPixelFrame( throw new ScreenshotNoFrameError(); } -function screenshotPreparationError(error: unknown): ScreenshotNoFrameError { - const message = error instanceof Error ? error.message : String(error); - if (isScreenshotNoFrameError(error)) { - return new ScreenshotNoFrameError( - `${SCREENSHOT_PREP_UNAVAILABLE_PREFIX} the app renderer did not acknowledge capture preparation before the timeout.`, - ); - } - if (message.includes(SCREENSHOT_PREP_UNAVAILABLE_PREFIX)) { - return new ScreenshotNoFrameError(message); - } - if ( - message.includes("Browser pixel capture preparation is unavailable.") || - message.includes("Browser host renderer is not available.") || - message.includes("is not mounted.") - ) { - return new ScreenshotNoFrameError(`${SCREENSHOT_PREP_UNAVAILABLE_PREFIX} ${message}`); - } - return new ScreenshotNoFrameError(`${SCREENSHOT_PREP_UNAVAILABLE_PREFIX} ${message}`); -} - function isKnownNoFrameCaptureError(error: unknown): boolean { const message = error instanceof Error ? error.message : String(error); return ( @@ -208,34 +159,15 @@ function isKnownNoFrameCaptureError(error: unknown): boolean { ); } -async function runPreparedPixelCapture( +async function runPaintedPixelCapture( contents: TabContents, capture: () => Promise, ): Promise { - return runSerializedPixelCapture(async () => { - const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed(); - let preparation: TabPixelCapturePreparation | null = null; - try { - // Offscreen-parked webview guests have no compositor surface for - // capturePage/CDP to copy. The renderer must briefly make the host - // paintable before capture; see docs/browser-capture-harness.md. - preparation = await prepareForPixelCapture(contents); - contents.setBackgroundThrottling(false); - return await capturePreparedPixelFrame(contents, capture); - } finally { - try { - if (preparation) { - await restorePixelCapture(contents, preparation); - } - } finally { - contents.setBackgroundThrottling(previousBackgroundThrottling); - } - } - }); + return runSerializedPixelCapture(() => capturePixelFrameWithRetry(contents, capture)); } async function capturePaintedViewport(contents: TabContents): Promise { - return runPreparedPixelCapture(contents, () => contents.capturePage({ stayHidden: false })); + return runPaintedPixelCapture(contents, () => contents.capturePage({ stayHidden: false })); } function tabInfoFromContents( @@ -966,7 +898,7 @@ async function executeFullPageScreenshot( let width = 0; let height = 0; try { - screenshot = await runPreparedPixelCapture(target.contents, async () => { + screenshot = await runPaintedPixelCapture(target.contents, async () => { const metrics = await getCdpLayoutMetrics(target.contents); width = metrics.contentWidth; height = metrics.contentHeight; diff --git a/packages/desktop/src/features/browser-webviews/index.test.ts b/packages/desktop/src/features/browser-webviews/index.test.ts new file mode 100644 index 000000000..bec113075 --- /dev/null +++ b/packages/desktop/src/features/browser-webviews/index.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, test } from "vitest"; +import { getPaseoBrowserIdForWebContents, registerPaseoBrowserWebContents } from "./index.js"; + +class FakeRegisteredWebContents { + public readonly backgroundThrottlingCalls: boolean[] = []; + private destroyedListener: (() => void) | null = null; + private destroyed = false; + + public constructor(public readonly id: number) {} + + public isDestroyed(): boolean { + return this.destroyed; + } + + public setBackgroundThrottling(allowed: boolean): void { + this.backgroundThrottlingCalls.push(allowed); + } + + public once(event: "destroyed", listener: () => void): void { + expect(event).toBe("destroyed"); + this.destroyedListener = listener; + } + + public destroy(): void { + this.destroyed = true; + this.destroyedListener?.(); + } +} + +describe("registerPaseoBrowserWebContents", () => { + test("disables guest background throttling once when the webview is registered", () => { + const contents = new FakeRegisteredWebContents(9001); + + registerPaseoBrowserWebContents(contents, "browser-throttle"); + + expect(contents.backgroundThrottlingCalls).toEqual([false]); + expect(getPaseoBrowserIdForWebContents(contents)).toBe("browser-throttle"); + + contents.destroy(); + + expect(getPaseoBrowserIdForWebContents(contents)).toBeNull(); + expect(contents.backgroundThrottlingCalls).toEqual([false]); + }); +}); diff --git a/packages/desktop/src/features/browser-webviews/index.ts b/packages/desktop/src/features/browser-webviews/index.ts index ed7ad3fdf..6efbe8f93 100644 --- a/packages/desktop/src/features/browser-webviews/index.ts +++ b/packages/desktop/src/features/browser-webviews/index.ts @@ -11,6 +11,16 @@ export type { BrowserWorkspaceRegistration }; const browserRegistry = new PaseoBrowserWebviewRegistry(); +interface BrowserWebContentsIdentity { + readonly id: number; + isDestroyed(): boolean; +} + +interface RegisteredBrowserWebContents extends BrowserWebContentsIdentity { + setBackgroundThrottling(allowed: boolean): void; + once(event: "destroyed", listener: () => void): void; +} + function getBrowserIdFromWebviewPartition(partition: string | undefined): string | null { const prefix = "persist:paseo-browser-"; if (!partition?.startsWith(prefix)) { @@ -36,14 +46,20 @@ export function listRegisteredPaseoBrowserIds(): string[] { .filter((browserId) => getPaseoBrowserWebContents(browserId)); } -export function registerPaseoBrowserWebContents(contents: WebContents, browserId: string): void { +export function registerPaseoBrowserWebContents( + contents: RegisteredBrowserWebContents, + browserId: string, +): void { + contents.setBackgroundThrottling(false); browserRegistry.registerWebContents({ webContentsId: contents.id, browserId }); contents.once("destroyed", () => { browserRegistry.unregisterWebContents(contents.id); }); } -export function getPaseoBrowserIdForWebContents(contents: WebContents | null): string | null { +export function getPaseoBrowserIdForWebContents( + contents: BrowserWebContentsIdentity | null, +): string | null { if (!contents || contents.isDestroyed()) { return null; } diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index 31e739209..02fe9dc0e 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -1,117 +1,6 @@ import { contextBridge, ipcRenderer, webUtils } from "electron"; type EventHandler = (payload: unknown) => void; -type BrowserPixelCapturePrepareHandler = (input: { - requestId: string; - browserId: string; -}) => Promise<{ token: string }>; -type BrowserPixelCaptureRestoreHandler = (input: { token: string }) => Promise; -type BrowserPixelCaptureCancelHandler = (input: { - requestId?: string; - token?: string; -}) => Promise; - -let prepareForPixelCaptureHandler: BrowserPixelCapturePrepareHandler | null = null; -let restorePixelCaptureHandler: BrowserPixelCaptureRestoreHandler | null = null; -let cancelPixelCaptureHandler: BrowserPixelCaptureCancelHandler | null = null; -const canceledPixelCaptureRequestIds = new Set(); - -function readStringField(payload: unknown, key: string): string | null { - if (!isRecord(payload)) { - return null; - } - const value = payload[key]; - return typeof value === "string" && value.length > 0 ? value : null; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -ipcRenderer.on("paseo:browser:capture-prepare", async (_event, payload: unknown) => { - const requestId = readStringField(payload, "requestId"); - const browserId = readStringField(payload, "browserId"); - if (!requestId || !browserId || !prepareForPixelCaptureHandler) { - ipcRenderer.send("paseo:browser:capture-prepared", { - requestId: requestId ?? "unknown", - ok: false, - message: "Browser pixel capture preparation is unavailable.", - }); - return; - } - - try { - const preparation = await prepareForPixelCaptureHandler({ requestId, browserId }); - if (canceledPixelCaptureRequestIds.delete(requestId)) { - await restorePixelCaptureHandler?.({ token: preparation.token }); - ipcRenderer.send("paseo:browser:capture-prepared", { - requestId, - ok: false, - message: "Browser pixel capture preparation was canceled.", - }); - return; - } - ipcRenderer.send("paseo:browser:capture-prepared", { - requestId, - ok: true, - token: preparation.token, - }); - } catch (error) { - canceledPixelCaptureRequestIds.delete(requestId); - ipcRenderer.send("paseo:browser:capture-prepared", { - requestId, - ok: false, - message: errorMessage(error), - }); - } -}); - -ipcRenderer.on("paseo:browser:capture-cancel", async (_event, payload: unknown) => { - const requestId = readStringField(payload, "requestId"); - const token = readStringField(payload, "token"); - if (requestId) { - canceledPixelCaptureRequestIds.add(requestId); - } - if (!requestId && !token) { - return; - } - try { - await cancelPixelCaptureHandler?.({ - ...(requestId ? { requestId } : {}), - ...(token ? { token } : {}), - }); - } catch { - // The original prepare/restore request owns the user-visible error. - } -}); - -ipcRenderer.on("paseo:browser:capture-restore", async (_event, payload: unknown) => { - const requestId = readStringField(payload, "requestId"); - const token = readStringField(payload, "token"); - if (!requestId || !token || !restorePixelCaptureHandler) { - ipcRenderer.send("paseo:browser:capture-restored", { - requestId: requestId ?? "unknown", - ok: false, - message: "Browser pixel capture restore is unavailable.", - }); - return; - } - - try { - await restorePixelCaptureHandler({ token }); - ipcRenderer.send("paseo:browser:capture-restored", { requestId, ok: true }); - } catch (error) { - ipcRenderer.send("paseo:browser:capture-restored", { - requestId, - ok: false, - message: errorMessage(error), - }); - } -}); contextBridge.exposeInMainWorld("paseoDesktop", { platform: process.platform, @@ -201,29 +90,5 @@ contextBridge.exposeInMainWorld("paseoDesktop", { ) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect), copyElement: (payload: { text?: string; imageDataUrl?: string }) => ipcRenderer.invoke("paseo:browser:copy-element", payload), - onPrepareForPixelCapture: (handler: BrowserPixelCapturePrepareHandler): (() => void) => { - prepareForPixelCaptureHandler = handler; - return () => { - if (prepareForPixelCaptureHandler === handler) { - prepareForPixelCaptureHandler = null; - } - }; - }, - onRestorePixelCapture: (handler: BrowserPixelCaptureRestoreHandler): (() => void) => { - restorePixelCaptureHandler = handler; - return () => { - if (restorePixelCaptureHandler === handler) { - restorePixelCaptureHandler = null; - } - }; - }, - onCancelPixelCapture: (handler: BrowserPixelCaptureCancelHandler): (() => void) => { - cancelPixelCaptureHandler = handler; - return () => { - if (cancelPixelCaptureHandler === handler) { - cancelPixelCaptureHandler = null; - } - }; - }, }, }); diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts index edfa66555..c318e64a5 100644 --- a/packages/server/src/server/browser-tools/tools.test.ts +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -410,14 +410,14 @@ const brokerErrorCases = [ ok: false, error: { code: "screenshot_no_frame", - message: "The browser tab has no painted frame. Focus the tab in the app, then try again.", - retryable: false, + message: "The tab has not painted yet. Retry the screenshot.", + retryable: true, }, }, content: [ { type: "text", - text: "The browser tab has no painted frame. Focus the tab in the app, then try again.", + text: "The tab has not painted yet. Retry the screenshot.", }, ], context: {