diff --git a/packages/app/src/components/browser-webview-resident.browser.test.ts b/packages/app/src/components/browser-webview-resident.browser.test.ts index 015c533a6..cf5872314 100644 --- a/packages/app/src/components/browser-webview-resident.browser.test.ts +++ b/packages/app/src/components/browser-webview-resident.browser.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it } from "vitest"; import { + cancelResidentBrowserWebviewPixelCapture, clearResidentBrowserWebviewsForTests, ensureResidentBrowserWebview, prepareResidentBrowserWebviewForPixelCapture, @@ -121,4 +122,46 @@ describe("resident browser webviews", () => { expect(host?.style.width).toBe("1280px"); expect(host?.style.opacity).toBe("0"); }); + + it("cancels an in-flight pixel capture preparation by request id", async () => { + ensureResidentBrowserWebview({ + browserId: "browser-cancel", + url: "https://example.com", + }); + + const preparation = prepareResidentBrowserWebviewForPixelCapture({ + requestId: "prepare-1", + browserId: "browser-cancel", + }); + const host = document.getElementById("paseo-browser-resident-webviews"); + expect(host?.style.left).toBe("0px"); + expect(host?.style.opacity).toBe("1"); + + await cancelResidentBrowserWebviewPixelCapture({ requestId: "prepare-1" }); + + await expect(preparation).rejects.toThrow("Browser pixel capture preparation was canceled."); + expect(host?.style.left).toBe("-20000px"); + expect(host?.style.width).toBe("1280px"); + expect(host?.style.opacity).toBe("0"); + }); + + it("parks the resident host when a prepared browser tab is removed", async () => { + const webview = ensureResidentBrowserWebview({ + browserId: "browser-detached", + url: "https://example.com", + }); + const preparation = await prepareResidentBrowserWebviewForPixelCapture({ + browserId: "browser-detached", + }); + const host = document.getElementById("paseo-browser-resident-webviews"); + + removeResidentBrowserWebview("browser-detached"); + + expect(webview?.isConnected).toBe(false); + expect(host?.style.left).toBe("-20000px"); + expect(host?.style.width).toBe("1280px"); + expect(host?.style.opacity).toBe("0"); + await restoreResidentBrowserWebviewAfterPixelCapture(preparation); + expect(host?.style.left).toBe("-20000px"); + }); }); diff --git a/packages/app/src/components/browser-webview-resident.ts b/packages/app/src/components/browser-webview-resident.ts index dba212088..36b882134 100644 --- a/packages/app/src/components/browser-webview-resident.ts +++ b/packages/app/src/components/browser-webview-resident.ts @@ -6,7 +6,7 @@ const RESIDENT_VIEWPORT_WIDTH = 1280; const RESIDENT_VIEWPORT_HEIGHT = 800; const residentWebviewsByBrowserId = new Map(); -const activeCapturePreparations = new Map(); +const activeCapturePreparations = new Map(); let captureBridgeInstallCount = 0; let captureBridgeDisposer: (() => void) | null = null; @@ -16,6 +16,12 @@ interface BrowserWebviewElement extends HTMLElement { src: string; } +interface ActiveCapturePreparation { + browserId: string; + requestId?: string; + preparesResidentHost: boolean; +} + function trimNonEmpty(value: string | null | undefined): string | null { if (typeof value !== "string") { return null; @@ -120,6 +126,58 @@ async function waitForCapturePaint(webview: HTMLElement): Promise { webview.getBoundingClientRect(); } +function activeCaptureTokenFor(input: { token?: string; requestId?: string }): string | null { + const token = trimNonEmpty(input.token); + if (token && activeCapturePreparations.has(token)) { + return token; + } + const requestId = trimNonEmpty(input.requestId); + if (!requestId) { + return null; + } + for (const [candidateToken, preparation] of activeCapturePreparations.entries()) { + if (preparation.requestId === requestId) { + return candidateToken; + } + } + return null; +} + +function parkResidentHostIfIdle(): void { + if (hasActiveResidentHostPreparation()) { + return; + } + const host = readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID); + if (host instanceof HTMLElement) { + applyResidentHostParkingStyle(host); + } +} + +function releaseCapturePreparationToken(token: string): void { + const preparation = activeCapturePreparations.get(token); + if (!preparation) { + return; + } + activeCapturePreparations.delete(token); + if (preparation.preparesResidentHost) { + parkResidentHostIfIdle(); + } +} + +function releaseCapturePreparationsForBrowser(browserId: string): void { + for (const [token, preparation] of activeCapturePreparations.entries()) { + if (preparation.browserId === browserId) { + activeCapturePreparations.delete(token); + } + } + parkResidentHostIfIdle(); +} + +function releaseAllCapturePreparations(): void { + activeCapturePreparations.clear(); + parkResidentHostIfIdle(); +} + export function prepareBrowserWebview( webview: HTMLElement, input: { browserId: string; initialUrl?: string | null }, @@ -196,6 +254,7 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl export async function prepareResidentBrowserWebviewForPixelCapture(input: { browserId: string; + requestId?: string; }): Promise<{ token: string }> { const browserId = trimNonEmpty(input.browserId); if (!browserId) { @@ -214,19 +273,24 @@ export async function prepareResidentBrowserWebviewForPixelCapture(input: { const token = `capture-${++nextCapturePreparationId}`; const preparesResidentHost = webview.parentElement === host; - activeCapturePreparations.set(token, { preparesResidentHost }); + const requestId = trimNonEmpty(input.requestId); + activeCapturePreparations.set(token, { + browserId, + ...(requestId ? { requestId } : {}), + preparesResidentHost, + }); try { if (preparesResidentHost) { applyResidentHostCaptureStyle(host); applyResidentWebviewStyle(webview); } await waitForCapturePaint(webview); + if (!activeCapturePreparations.has(token)) { + throw new Error("Browser pixel capture preparation was canceled."); + } return { token }; } catch (error) { - activeCapturePreparations.delete(token); - if (!hasActiveResidentHostPreparation()) { - applyResidentHostParkingStyle(host); - } + releaseCapturePreparationToken(token); throw error; } } @@ -234,24 +298,18 @@ export async function prepareResidentBrowserWebviewForPixelCapture(input: { export async function restoreResidentBrowserWebviewAfterPixelCapture(input: { token: string; }): Promise { - const preparation = activeCapturePreparations.get(input.token); - if (!preparation) { - return; - } + releaseCapturePreparationToken(input.token); +} - activeCapturePreparations.delete(input.token); - if (!preparation.preparesResidentHost || hasActiveResidentHostPreparation()) { +export async function cancelResidentBrowserWebviewPixelCapture(input: { + requestId?: string; + token?: string; +}): Promise { + const token = activeCaptureTokenFor(input); + if (!token) { return; } - - const ownerDocument = readDocument(); - if (!ownerDocument) { - return; - } - const host = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID); - if (host instanceof HTMLElement) { - applyResidentHostParkingStyle(host); - } + releaseCapturePreparationToken(token); } export function installResidentBrowserCaptureBridge(): () => void { @@ -264,9 +322,13 @@ export function installResidentBrowserCaptureBridge(): () => void { const disposeRestore = browserBridge?.onRestorePixelCapture?.( restoreResidentBrowserWebviewAfterPixelCapture, ); + const disposeCancel = browserBridge?.onCancelPixelCapture?.( + cancelResidentBrowserWebviewPixelCapture, + ); captureBridgeDisposer = () => { disposePrepare?.(); disposeRestore?.(); + disposeCancel?.(); }; } @@ -277,6 +339,7 @@ export function installResidentBrowserCaptureBridge(): () => void { } captureBridgeDisposer?.(); captureBridgeDisposer = null; + releaseAllCapturePreparations(); }; } @@ -288,6 +351,7 @@ export function removeResidentBrowserWebview(browserId: string): void { const resident = residentWebviewsByBrowserId.get(normalizedBrowserId) ?? null; residentWebviewsByBrowserId.delete(normalizedBrowserId); + releaseCapturePreparationsForBrowser(normalizedBrowserId); resident?.remove(); } @@ -296,7 +360,7 @@ export function clearResidentBrowserWebviewsForTests(): void { webview.remove(); } residentWebviewsByBrowserId.clear(); - activeCapturePreparations.clear(); + releaseAllCapturePreparations(); nextCapturePreparationId = 0; readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove(); } diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index 1d57468c0..86245416b 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -149,11 +149,17 @@ export interface DesktopBrowserBridge { /** Copy element text and/or an image to the system clipboard from main. */ copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise; onPrepareForPixelCapture?: ( - handler: (input: { browserId: string }) => Promise, + handler: (input: { + requestId: string; + browserId: string; + }) => Promise, ) => () => void; onRestorePixelCapture?: ( handler: (input: DesktopBrowserPixelCapturePreparation) => Promise, ) => () => void; + onCancelPixelCapture?: ( + handler: (input: { requestId?: string; token?: string }) => Promise, + ) => () => void; } export interface DesktopInvokeBridge { diff --git a/packages/desktop/src/features/browser-automation/ipc.test.ts b/packages/desktop/src/features/browser-automation/ipc.test.ts index cf3fa7e33..f678c0f0e 100644 --- a/packages/desktop/src/features/browser-automation/ipc.test.ts +++ b/packages/desktop/src/features/browser-automation/ipc.test.ts @@ -1,5 +1,5 @@ import type { Rectangle } from "electron"; -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import type { TabImage } from "./service.js"; import { adaptWebContents } from "./ipc.js"; @@ -40,7 +40,13 @@ class FakeHostWebContents { } } -type IpcListener = (event: unknown, payload: unknown) => void; +interface FakeIpcEvent { + sender: { + id: number; + }; +} + +type IpcListener = (event: FakeIpcEvent, payload: unknown) => void; class FakeIpcBridge { private readonly listeners = new Map(); @@ -59,9 +65,10 @@ class FakeIpcBridge { ); } - public emit(channel: string, payload: unknown): void { + public emit(channel: string, payload: unknown, input: { senderId?: number } = {}): void { + const event = { sender: { id: input.senderId ?? 10 } }; for (const listener of this.listeners.get(channel) ?? []) { - listener({}, payload); + listener(event, payload); } } @@ -190,14 +197,33 @@ describe("browser automation IPC adapter", () => { const host = new FakeHostWebContents(10); const contents = new FakeWebContents(20, host); const ipc = new FakeIpcBridge(); + const requestIds = ["prepare-1", "restore-1"]; const tab = adaptWebContents(contents, "browser-a", { ipc, - createRequestId: () => "restore-1", + createRequestId: () => { + const requestId = requestIds.shift(); + if (!requestId) { + throw new Error("Missing request id"); + } + return requestId; + }, }); + const preparation = tab.prepareForPixelCapture(); + ipc.emit("paseo:browser:capture-prepared", { + requestId: "prepare-1", + ok: true, + token: "token-a", + }); + await expect(preparation).resolves.toEqual({ token: "token-a" }); + const restored = tab.restorePixelCapture({ token: "token-a" }); expect(host.sentMessages).toEqual([ + { + channel: "paseo:browser:capture-prepare", + payload: { requestId: "prepare-1", browserId: "browser-a" }, + }, { channel: "paseo:browser:capture-restore", payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" }, @@ -208,6 +234,51 @@ describe("browser automation IPC adapter", () => { await expect(restored).resolves.toBeUndefined(); }); + test("restorePixelCapture uses the host captured during preparation when the guest detaches", async () => { + const host = new FakeHostWebContents(10); + const contents = new FakeWebContents(20, host); + const ipc = new FakeIpcBridge(); + const requestIds = ["prepare-1", "restore-1"]; + const tab = adaptWebContents(contents, "browser-a", { + ipc, + createRequestId: () => { + const requestId = requestIds.shift(); + if (!requestId) { + throw new Error("Missing request id"); + } + return requestId; + }, + }); + + const preparationPromise = tab.prepareForPixelCapture(); + ipc.emit("paseo:browser:capture-prepared", { + requestId: "prepare-1", + ok: true, + token: "token-a", + }); + const preparation = await preparationPromise; + contents.hostWebContents = null; + + const restored = tab.restorePixelCapture(preparation); + + expect(host.sentMessages).toEqual([ + { + channel: "paseo:browser:capture-prepare", + payload: { requestId: "prepare-1", browserId: "browser-a" }, + }, + { + channel: "paseo:browser:capture-restore", + payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" }, + }, + ]); + ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true }); + + await expect(restored).resolves.toBeUndefined(); + await expect(tab.restorePixelCapture(preparation)).rejects.toThrow( + "Browser pixel capture preparation is no longer active.", + ); + }); + test("prepareForPixelCapture rejects when the renderer reports preparation failure", async () => { const host = new FakeHostWebContents(10); const contents = new FakeWebContents(20, host); @@ -228,6 +299,71 @@ describe("browser automation IPC adapter", () => { expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0); }); + test("prepareForPixelCapture ignores matching responses from the wrong sender", async () => { + const host = new FakeHostWebContents(10); + const contents = new FakeWebContents(20, host); + const ipc = new FakeIpcBridge(); + const tab = adaptWebContents(contents, "browser-a", { + ipc, + createRequestId: () => "prepare-1", + }); + + const preparation = tab.prepareForPixelCapture(); + ipc.emit( + "paseo:browser:capture-prepared", + { + requestId: "prepare-1", + ok: true, + token: "spoofed-token", + }, + { senderId: 99 }, + ); + expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1); + + ipc.emit("paseo:browser:capture-prepared", { + requestId: "prepare-1", + ok: true, + token: "token-a", + }); + + await expect(preparation).resolves.toEqual({ token: "token-a" }); + }); + + test("prepareForPixelCapture cancels the renderer request when the ack times out", async () => { + vi.useFakeTimers(); + try { + const host = new FakeHostWebContents(10); + const contents = new FakeWebContents(20, host); + const ipc = new FakeIpcBridge(); + const tab = adaptWebContents(contents, "browser-a", { + ipc, + createRequestId: () => "prepare-1", + timeoutMs: 25, + }); + + const preparation = tab.prepareForPixelCapture(); + const rejection = expect(preparation).rejects.toThrow( + "Browser pixel capture prepare timed out.", + ); + await vi.advanceTimersByTimeAsync(25); + + await rejection; + expect(host.sentMessages).toEqual([ + { + channel: "paseo:browser:capture-prepare", + payload: { requestId: "prepare-1", browserId: "browser-a" }, + }, + { + channel: "paseo:browser:capture-cancel", + payload: { requestId: "prepare-1", browserId: "browser-a" }, + }, + ]); + expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + test("prepareForPixelCapture rejects when the guest has no embedder renderer", async () => { const contents = new FakeWebContents(20, null); const tab = adaptWebContents(contents, "browser-a"); diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 14050f676..478c2eaab 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -22,7 +22,13 @@ interface IpcHandlerRegistry { handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void; } -type IpcListener = (event: unknown, payload: unknown) => void; +interface IpcBridgeEvent { + sender?: { + id?: number; + }; +} + +type IpcListener = (event: IpcBridgeEvent, payload: unknown) => void; interface IpcCaptureBridge { on(channel: string, listener: IpcListener): void; @@ -88,12 +94,18 @@ interface PixelCaptureBridgeOptions { timeoutMs?: number; } +interface PreparedPixelCapture { + browserId: string; + host: HostWebContents; +} + export function adaptWebContents( contents: BrowserAutomationWebContents, browserId: string, options?: PixelCaptureBridgeOptions, ): TabContents { observeConsoleMessages(contents); + const preparedPixelCapturesByToken = new Map(); return { id: contents.id, getURL: () => contents.getURL(), @@ -109,16 +121,30 @@ export function adaptWebContents( reload: () => contents.reload(), capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions), prepareForPixelCapture: async () => { - const result = await requestPixelCaptureBridge(contents, browserId, "prepare", options); + const host = getPixelCaptureHost(contents); + const result = await requestPixelCaptureBridge({ host, browserId, kind: "prepare", options }); if (!result.token) { throw new Error("Browser pixel capture preparation did not return a token."); } + preparedPixelCapturesByToken.set(result.token, { browserId, host }); return { token: result.token }; }, restorePixelCapture: async (preparation) => { - await requestPixelCaptureBridge(contents, browserId, "restore", options, { - token: preparation.token, - }); + const prepared = preparedPixelCapturesByToken.get(preparation.token); + if (!prepared) { + throw new Error("Browser pixel capture preparation is no longer active."); + } + try { + await requestPixelCaptureBridge({ + host: prepared.host, + browserId: prepared.browserId, + kind: "restore", + options, + extraPayload: { token: preparation.token }, + }); + } finally { + preparedPixelCapturesByToken.delete(preparation.token); + } }, invalidate: () => contents.invalidate(), isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(), @@ -133,15 +159,23 @@ export function adaptWebContents( }; } -function requestPixelCaptureBridge( - contents: BrowserAutomationWebContents, - browserId: string, - kind: PixelCaptureBridgeKind, - options: PixelCaptureBridgeOptions | undefined, - extraPayload?: { token: string }, -): Promise { +function getPixelCaptureHost(contents: BrowserAutomationWebContents): HostWebContents { const host = contents.hostWebContents; if (!host || host.isDestroyed()) { + throw new Error("Browser host renderer is not available."); + } + return host; +} + +function requestPixelCaptureBridge(input: { + host: HostWebContents; + browserId: string; + kind: PixelCaptureBridgeKind; + options: PixelCaptureBridgeOptions | undefined; + extraPayload?: { token: string }; +}): Promise { + const { host, browserId, kind, options, extraPayload } = input; + if (host.isDestroyed()) { return Promise.reject(new Error("Browser host renderer is not available.")); } @@ -162,7 +196,10 @@ function requestPixelCaptureBridge( } ipc.removeListener(responseChannel, listener); }; - const listener: IpcListener = (_event, payload) => { + const listener: IpcListener = (event, payload) => { + if (readSenderId(event) !== host.id) { + return; + } const response = readPixelCaptureBridgeResponse(payload, requestId); if (!response) { return; @@ -178,6 +215,11 @@ function requestPixelCaptureBridge( ipc.on(responseChannel, listener); timeoutId = setTimeout(() => { cleanup(); + sendPixelCaptureCancel(host, { + requestId, + browserId, + ...(extraPayload ? { token: extraPayload.token } : {}), + }); reject(new Error(`Browser pixel capture ${kind} timed out.`)); }, timeoutMs); @@ -194,6 +236,25 @@ function requestPixelCaptureBridge( }); } +function sendPixelCaptureCancel( + host: HostWebContents, + payload: { browserId: string; requestId?: string; token?: string }, +): void { + if (host.isDestroyed()) { + return; + } + try { + host.send("paseo:browser:capture-cancel", payload); + } catch { + // The original prepare/restore request owns the user-visible error. + } +} + +function readSenderId(event: IpcBridgeEvent): number | null { + const senderId = event.sender?.id; + return typeof senderId === "number" ? senderId : null; +} + function readPixelCaptureBridgeResponse( payload: unknown, requestId: string, diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index e04d43f9f..2bccd4453 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -159,7 +159,7 @@ async function runPreparedPixelCapture( try { // Offscreen-parked webview guests have no compositor surface for // capturePage/CDP to copy. The renderer must briefly make the host - // paintable before capture; see ~/.paseo/plans/browser-capture-harness-results.md. + // paintable before capture; see docs/browser-capture-harness.md. preparation = await prepareForPixelCapture(contents); contents.setBackgroundThrottling(false); contents.invalidate(); diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index 3785c6ec9..31e739209 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -2,12 +2,19 @@ import { contextBridge, ipcRenderer, webUtils } from "electron"; type EventHandler = (payload: unknown) => void; type BrowserPixelCapturePrepareHandler = (input: { + requestId: string; browserId: string; }) => Promise<{ token: string }>; type BrowserPixelCaptureRestoreHandler = (input: { token: string }) => Promise; +type BrowserPixelCaptureCancelHandler = (input: { + requestId?: string; + token?: string; +}) => Promise; let prepareForPixelCaptureHandler: BrowserPixelCapturePrepareHandler | null = null; let restorePixelCaptureHandler: BrowserPixelCaptureRestoreHandler | null = null; +let cancelPixelCaptureHandler: BrowserPixelCaptureCancelHandler | null = null; +const canceledPixelCaptureRequestIds = new Set(); function readStringField(payload: unknown, key: string): string | null { if (!isRecord(payload)) { @@ -38,13 +45,23 @@ ipcRenderer.on("paseo:browser:capture-prepare", async (_event, payload: unknown) } try { - const preparation = await prepareForPixelCaptureHandler({ browserId }); + const preparation = await prepareForPixelCaptureHandler({ requestId, browserId }); + if (canceledPixelCaptureRequestIds.delete(requestId)) { + await restorePixelCaptureHandler?.({ token: preparation.token }); + ipcRenderer.send("paseo:browser:capture-prepared", { + requestId, + ok: false, + message: "Browser pixel capture preparation was canceled.", + }); + return; + } ipcRenderer.send("paseo:browser:capture-prepared", { requestId, ok: true, token: preparation.token, }); } catch (error) { + canceledPixelCaptureRequestIds.delete(requestId); ipcRenderer.send("paseo:browser:capture-prepared", { requestId, ok: false, @@ -53,6 +70,25 @@ ipcRenderer.on("paseo:browser:capture-prepare", async (_event, payload: unknown) } }); +ipcRenderer.on("paseo:browser:capture-cancel", async (_event, payload: unknown) => { + const requestId = readStringField(payload, "requestId"); + const token = readStringField(payload, "token"); + if (requestId) { + canceledPixelCaptureRequestIds.add(requestId); + } + if (!requestId && !token) { + return; + } + try { + await cancelPixelCaptureHandler?.({ + ...(requestId ? { requestId } : {}), + ...(token ? { token } : {}), + }); + } catch { + // The original prepare/restore request owns the user-visible error. + } +}); + ipcRenderer.on("paseo:browser:capture-restore", async (_event, payload: unknown) => { const requestId = readStringField(payload, "requestId"); const token = readStringField(payload, "token"); @@ -181,5 +217,13 @@ contextBridge.exposeInMainWorld("paseoDesktop", { } }; }, + onCancelPixelCapture: (handler: BrowserPixelCaptureCancelHandler): (() => void) => { + cancelPixelCaptureHandler = handler; + return () => { + if (cancelPixelCaptureHandler === handler) { + cancelPixelCaptureHandler = null; + } + }; + }, }, });