mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-14 13:22:37 +00:00
Fix browser screenshot capture for parked webviews
This commit is contained in:
@@ -21,34 +21,35 @@ This is an npm workspace monorepo:
|
||||
|
||||
At the start of non-trivial work, list `docs/` and skim anything relevant to the task. When you learn something meta worth preserving — a gotcha, a convention, a workflow, a piece of system context that will outlive the current task — update an existing doc or propose a new one. Code-level facts belong in inline comments next to the code; system, process, and gotcha-level facts belong in `docs/`.
|
||||
|
||||
| Doc | What's in it |
|
||||
| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
|
||||
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
|
||||
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
|
||||
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
|
||||
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
|
||||
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
|
||||
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
|
||||
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
|
||||
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
|
||||
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
|
||||
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
|
||||
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
|
||||
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
|
||||
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
|
||||
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
|
||||
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
|
||||
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
|
||||
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
|
||||
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
|
||||
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
|
||||
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
|
||||
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
|
||||
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
|
||||
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
|
||||
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
|
||||
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
|
||||
| Doc | What's in it |
|
||||
| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
|
||||
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
|
||||
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
|
||||
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
|
||||
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
|
||||
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
|
||||
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
|
||||
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
|
||||
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
|
||||
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
|
||||
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
|
||||
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
|
||||
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
|
||||
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
|
||||
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
|
||||
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
|
||||
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
|
||||
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
|
||||
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
|
||||
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
|
||||
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
|
||||
| [docs/browser-capture-harness.md](docs/browser-capture-harness.md) | Real-Electron browser screenshot harness and compositor-surface gotcha |
|
||||
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
|
||||
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
|
||||
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
|
||||
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
|
||||
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
|
||||
|
||||
## Quick start
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Browser Capture Harness
|
||||
|
||||
The desktop capture harness is the real-Electron verification path for browser screenshots.
|
||||
It validates the compositor behavior that unit tests cannot see:
|
||||
|
||||
- the resident automation `<webview>` starts in the production parking state;
|
||||
- the parked guest has no copyable viewport frame;
|
||||
- the resident webview guest is sized to 1280x800 logical pixels;
|
||||
- the app-style prep sequence moves the host to a paintable 1x1 clipped state, waits two animation frames and a layout read, then both viewport `capturePage` and full-page CDP screenshots return real pixels;
|
||||
- restore returns the host to offscreen parking after every capture.
|
||||
|
||||
Run it with the repo Electron:
|
||||
|
||||
```bash
|
||||
npm run capture-harness --workspace=@getpaseo/desktop
|
||||
```
|
||||
|
||||
The harness writes PNG evidence and `results.json` to:
|
||||
|
||||
```text
|
||||
packages/desktop/capture-harness/out/
|
||||
```
|
||||
|
||||
A passing run prints `PASS` lines for guest sizing, the expected parked-capture failure,
|
||||
five viewport prep captures, five full-page prep captures, and final completion. The PNG
|
||||
sizes may be device-pixel scaled; on a Retina display the 1280x800 logical viewport is
|
||||
usually saved as 2560x1600.
|
||||
|
||||
## Mechanism
|
||||
|
||||
Electron captures copy from the guest web contents' compositor surface. A resident
|
||||
webview parked at `left:-20000px` and `opacity:0` does not have a copyable surface, and
|
||||
`capturePage({ stayHidden:false })` or CDP `Page.captureScreenshot` cannot rescue it.
|
||||
|
||||
Before pixel capture, the app renderer temporarily makes the resident host paintable:
|
||||
`left:0`, `top:0`, `opacity:1`, `pointer-events:none`, host size `1x1`, and
|
||||
`overflow:hidden`, with the full-size 1280x800 webview inside. Main captures only after
|
||||
the renderer acknowledges two animation frames plus a `getBoundingClientRect()` read, and
|
||||
the renderer restores parking in a `finally`.
|
||||
@@ -1,6 +1,9 @@
|
||||
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
|
||||
import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
|
||||
import {
|
||||
ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault,
|
||||
installResidentBrowserCaptureBridge,
|
||||
} from "@/components/browser-webview-resident";
|
||||
import { createWorkspaceBrowser } from "@/stores/browser-store";
|
||||
import {
|
||||
buildWorkspaceTabPersistenceKey,
|
||||
@@ -40,7 +43,8 @@ export function mountBrowserAutomationHandler(
|
||||
options: BrowserAutomationHandlerOptions,
|
||||
): () => void {
|
||||
const getHost = options.getHost ?? getDesktopHost;
|
||||
return options.client.on("browser.automation.execute.request", (request) => {
|
||||
const uninstallCaptureBridge = installResidentBrowserCaptureBridge();
|
||||
const unsubscribe = options.client.on("browser.automation.execute.request", (request) => {
|
||||
void handleBrowserAutomationRequest({
|
||||
client: options.client,
|
||||
getHost,
|
||||
@@ -56,6 +60,10 @@ export function mountBrowserAutomationHandler(
|
||||
: {}),
|
||||
});
|
||||
});
|
||||
return () => {
|
||||
unsubscribe();
|
||||
uninstallCaptureBridge();
|
||||
};
|
||||
}
|
||||
|
||||
export function mountBrowserAutomationDaemonClientHandler(
|
||||
|
||||
@@ -2,8 +2,10 @@ import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
clearResidentBrowserWebviewsForTests,
|
||||
ensureResidentBrowserWebview,
|
||||
prepareResidentBrowserWebviewForPixelCapture,
|
||||
releaseResidentBrowserWebview,
|
||||
removeResidentBrowserWebview,
|
||||
restoreResidentBrowserWebviewAfterPixelCapture,
|
||||
takeResidentBrowserWebview,
|
||||
} from "./browser-webview-resident";
|
||||
|
||||
@@ -22,6 +24,7 @@ describe("resident browser webviews", () => {
|
||||
|
||||
expect(host.children).toHaveLength(0);
|
||||
expect(webview.isConnected).toBe(true);
|
||||
expect(webview.style.display).toBe("inline-flex");
|
||||
expect(webview.style.width).toBe("1280px");
|
||||
expect(webview.style.height).toBe("800px");
|
||||
|
||||
@@ -57,4 +60,65 @@ describe("resident browser webviews", () => {
|
||||
expect(webview?.isConnected).toBe(false);
|
||||
expect(takeResidentBrowserWebview("browser-closed")).toBeNull();
|
||||
});
|
||||
|
||||
it("temporarily makes resident webviews paintable for pixel capture", async () => {
|
||||
const webview = ensureResidentBrowserWebview({
|
||||
browserId: "browser-capture",
|
||||
url: "https://example.com",
|
||||
});
|
||||
if (!webview) {
|
||||
throw new Error("Expected resident browser webview");
|
||||
}
|
||||
|
||||
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-capture",
|
||||
});
|
||||
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||
|
||||
expect(preparation.token).toBe("capture-1");
|
||||
expect(host?.style.left).toBe("0px");
|
||||
expect(host?.style.top).toBe("0px");
|
||||
expect(host?.style.width).toBe("1px");
|
||||
expect(host?.style.height).toBe("1px");
|
||||
expect(host?.style.overflow).toBe("hidden");
|
||||
expect(host?.style.opacity).toBe("1");
|
||||
expect(host?.style.pointerEvents).toBe("none");
|
||||
expect(webview.style.display).toBe("inline-flex");
|
||||
expect(webview.style.width).toBe("1280px");
|
||||
expect(webview.style.height).toBe("800px");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
|
||||
|
||||
expect(host?.style.left).toBe("-20000px");
|
||||
expect(host?.style.width).toBe("1280px");
|
||||
expect(host?.style.height).toBe("800px");
|
||||
expect(host?.style.opacity).toBe("0");
|
||||
});
|
||||
|
||||
it("keeps the resident host paintable until every capture token is restored", async () => {
|
||||
ensureResidentBrowserWebview({
|
||||
browserId: "browser-overlap",
|
||||
url: "https://example.com",
|
||||
});
|
||||
|
||||
const first = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-overlap",
|
||||
});
|
||||
const second = await prepareResidentBrowserWebviewForPixelCapture({
|
||||
browserId: "browser-overlap",
|
||||
});
|
||||
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(first);
|
||||
|
||||
expect(host?.style.left).toBe("0px");
|
||||
expect(host?.style.width).toBe("1px");
|
||||
expect(host?.style.opacity).toBe("1");
|
||||
|
||||
await restoreResidentBrowserWebviewAfterPixelCapture(second);
|
||||
|
||||
expect(host?.style.left).toBe("-20000px");
|
||||
expect(host?.style.width).toBe("1280px");
|
||||
expect(host?.style.opacity).toBe("0");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
import { getDesktopHost } from "@/desktop/host";
|
||||
|
||||
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
|
||||
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
|
||||
const RESIDENT_VIEWPORT_WIDTH = 1280;
|
||||
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
||||
|
||||
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
|
||||
const activeCapturePreparations = new Map<string, { preparesResidentHost: boolean }>();
|
||||
|
||||
let captureBridgeInstallCount = 0;
|
||||
let captureBridgeDisposer: (() => void) | null = null;
|
||||
let nextCapturePreparationId = 0;
|
||||
|
||||
interface BrowserWebviewElement extends HTMLElement {
|
||||
src: string;
|
||||
@@ -21,14 +28,7 @@ function readDocument(): Document | null {
|
||||
return typeof document === "undefined" ? null : document;
|
||||
}
|
||||
|
||||
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
||||
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const host = ownerDocument.createElement("div");
|
||||
host.id = RESIDENT_BROWSER_HOST_ID;
|
||||
function applyResidentHostParkingStyle(host: HTMLElement): void {
|
||||
host.setAttribute("aria-hidden", "true");
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "-20000px";
|
||||
@@ -38,27 +38,88 @@ function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "0";
|
||||
host.style.pointerEvents = "none";
|
||||
host.style.zIndex = "";
|
||||
host.style.clipPath = "";
|
||||
host.style.visibility = "";
|
||||
host.style.transform = "";
|
||||
}
|
||||
|
||||
function applyResidentHostCaptureStyle(host: HTMLElement): void {
|
||||
host.setAttribute("aria-hidden", "true");
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "0";
|
||||
host.style.top = "0";
|
||||
host.style.width = "1px";
|
||||
host.style.height = "1px";
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "1";
|
||||
host.style.pointerEvents = "none";
|
||||
host.style.zIndex = "1";
|
||||
host.style.clipPath = "";
|
||||
host.style.visibility = "";
|
||||
host.style.transform = "";
|
||||
}
|
||||
|
||||
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
||||
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
const host = ownerDocument.createElement("div");
|
||||
host.id = RESIDENT_BROWSER_HOST_ID;
|
||||
applyResidentHostParkingStyle(host);
|
||||
ownerDocument.body.appendChild(host);
|
||||
return host;
|
||||
}
|
||||
|
||||
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
|
||||
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
|
||||
if (!(element instanceof HTMLElement)) {
|
||||
continue;
|
||||
}
|
||||
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
|
||||
return element as HTMLElement;
|
||||
return element;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function applyResidentWebviewStyle(webview: HTMLElement): void {
|
||||
webview.style.display = "block";
|
||||
webview.style.display = "inline-flex";
|
||||
webview.style.flex = "0 0 auto";
|
||||
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
webview.style.border = "0";
|
||||
webview.style.background = "transparent";
|
||||
}
|
||||
|
||||
function hasActiveResidentHostPreparation(): boolean {
|
||||
for (const preparation of activeCapturePreparations.values()) {
|
||||
if (preparation.preparesResidentHost) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function nextAnimationFrame(): Promise<void> {
|
||||
if (typeof requestAnimationFrame !== "function") {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
requestAnimationFrame(() => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForCapturePaint(webview: HTMLElement): Promise<void> {
|
||||
await nextAnimationFrame();
|
||||
await nextAnimationFrame();
|
||||
webview.getBoundingClientRect();
|
||||
}
|
||||
|
||||
export function prepareBrowserWebview(
|
||||
webview: HTMLElement,
|
||||
input: { browserId: string; initialUrl?: string | null },
|
||||
@@ -133,6 +194,92 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl
|
||||
getResidentBrowserHost(ownerDocument).appendChild(webview);
|
||||
}
|
||||
|
||||
export async function prepareResidentBrowserWebviewForPixelCapture(input: {
|
||||
browserId: string;
|
||||
}): Promise<{ token: string }> {
|
||||
const browserId = trimNonEmpty(input.browserId);
|
||||
if (!browserId) {
|
||||
throw new Error("Browser id is required for pixel capture preparation.");
|
||||
}
|
||||
const ownerDocument = readDocument();
|
||||
if (!ownerDocument) {
|
||||
throw new Error("Browser pixel capture preparation requires a document.");
|
||||
}
|
||||
|
||||
const host = getResidentBrowserHost(ownerDocument);
|
||||
const webview = findBrowserWebview(browserId, ownerDocument);
|
||||
if (!webview) {
|
||||
throw new Error(`Browser webview ${browserId} is not mounted.`);
|
||||
}
|
||||
|
||||
const token = `capture-${++nextCapturePreparationId}`;
|
||||
const preparesResidentHost = webview.parentElement === host;
|
||||
activeCapturePreparations.set(token, { preparesResidentHost });
|
||||
try {
|
||||
if (preparesResidentHost) {
|
||||
applyResidentHostCaptureStyle(host);
|
||||
applyResidentWebviewStyle(webview);
|
||||
}
|
||||
await waitForCapturePaint(webview);
|
||||
return { token };
|
||||
} catch (error) {
|
||||
activeCapturePreparations.delete(token);
|
||||
if (!hasActiveResidentHostPreparation()) {
|
||||
applyResidentHostParkingStyle(host);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function restoreResidentBrowserWebviewAfterPixelCapture(input: {
|
||||
token: string;
|
||||
}): Promise<void> {
|
||||
const preparation = activeCapturePreparations.get(input.token);
|
||||
if (!preparation) {
|
||||
return;
|
||||
}
|
||||
|
||||
activeCapturePreparations.delete(input.token);
|
||||
if (!preparation.preparesResidentHost || hasActiveResidentHostPreparation()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ownerDocument = readDocument();
|
||||
if (!ownerDocument) {
|
||||
return;
|
||||
}
|
||||
const host = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||
if (host instanceof HTMLElement) {
|
||||
applyResidentHostParkingStyle(host);
|
||||
}
|
||||
}
|
||||
|
||||
export function installResidentBrowserCaptureBridge(): () => void {
|
||||
captureBridgeInstallCount += 1;
|
||||
if (!captureBridgeDisposer) {
|
||||
const browserBridge = getDesktopHost()?.browser;
|
||||
const disposePrepare = browserBridge?.onPrepareForPixelCapture?.(
|
||||
prepareResidentBrowserWebviewForPixelCapture,
|
||||
);
|
||||
const disposeRestore = browserBridge?.onRestorePixelCapture?.(
|
||||
restoreResidentBrowserWebviewAfterPixelCapture,
|
||||
);
|
||||
captureBridgeDisposer = () => {
|
||||
disposePrepare?.();
|
||||
disposeRestore?.();
|
||||
};
|
||||
}
|
||||
|
||||
return () => {
|
||||
captureBridgeInstallCount = Math.max(0, captureBridgeInstallCount - 1);
|
||||
if (captureBridgeInstallCount > 0) {
|
||||
return;
|
||||
}
|
||||
captureBridgeDisposer?.();
|
||||
captureBridgeDisposer = null;
|
||||
};
|
||||
}
|
||||
|
||||
export function removeResidentBrowserWebview(browserId: string): void {
|
||||
const normalizedBrowserId = trimNonEmpty(browserId);
|
||||
if (!normalizedBrowserId) {
|
||||
@@ -149,5 +296,7 @@ export function clearResidentBrowserWebviewsForTests(): void {
|
||||
webview.remove();
|
||||
}
|
||||
residentWebviewsByBrowserId.clear();
|
||||
activeCapturePreparations.clear();
|
||||
nextCapturePreparationId = 0;
|
||||
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
|
||||
}
|
||||
|
||||
@@ -121,6 +121,10 @@ export interface DesktopBrowserShortcutEvent {
|
||||
action: "focus-url";
|
||||
}
|
||||
|
||||
export interface DesktopBrowserPixelCapturePreparation {
|
||||
token: string;
|
||||
}
|
||||
|
||||
export interface DesktopBrowserNewTabRequestEvent {
|
||||
sourceBrowserId: string;
|
||||
url: string;
|
||||
@@ -144,6 +148,12 @@ export interface DesktopBrowserBridge {
|
||||
) => Promise<string | null>;
|
||||
/** Copy element text and/or an image to the system clipboard from main. */
|
||||
copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
|
||||
onPrepareForPixelCapture?: (
|
||||
handler: (input: { browserId: string }) => Promise<DesktopBrowserPixelCapturePreparation>,
|
||||
) => () => void;
|
||||
onRestorePixelCapture?: (
|
||||
handler: (input: DesktopBrowserPixelCapturePreparation) => Promise<void>,
|
||||
) => () => void;
|
||||
}
|
||||
|
||||
export interface DesktopInvokeBridge {
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
out/
|
||||
fatal-error.txt
|
||||
@@ -0,0 +1,74 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Paseo Capture Harness Target</title>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
width: 1280px;
|
||||
min-height: 1600px;
|
||||
margin: 0;
|
||||
background: #ff00ff;
|
||||
overflow: auto;
|
||||
font-family: Arial, Helvetica, sans-serif;
|
||||
}
|
||||
|
||||
.label {
|
||||
position: absolute;
|
||||
left: 48px;
|
||||
top: 48px;
|
||||
color: #ffffff;
|
||||
font-size: 72px;
|
||||
font-weight: 800;
|
||||
line-height: 1;
|
||||
text-shadow:
|
||||
5px 5px 0 #000000,
|
||||
-2px -2px 0 #000000;
|
||||
}
|
||||
|
||||
.sub {
|
||||
position: absolute;
|
||||
left: 54px;
|
||||
top: 150px;
|
||||
color: #000000;
|
||||
background: #ffffff;
|
||||
font-size: 34px;
|
||||
font-weight: 700;
|
||||
padding: 10px 18px;
|
||||
}
|
||||
|
||||
.block {
|
||||
position: absolute;
|
||||
left: 56px;
|
||||
top: 220px;
|
||||
width: 360px;
|
||||
height: 92px;
|
||||
background: linear-gradient(
|
||||
90deg,
|
||||
#000000 0 25%,
|
||||
#ffffff 25% 50%,
|
||||
#000000 50% 75%,
|
||||
#ffffff 75% 100%
|
||||
);
|
||||
border: 8px solid #000000;
|
||||
}
|
||||
|
||||
.bottom {
|
||||
position: absolute;
|
||||
left: 48px;
|
||||
top: 1370px;
|
||||
color: #ffffff;
|
||||
font-size: 60px;
|
||||
font-weight: 800;
|
||||
text-shadow: 5px 5px 0 #000000;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="label">PASEO CAPTURE HARNESS</div>
|
||||
<div class="sub">LOCAL MAGENTA TARGET 1280x800 VIEWPORT</div>
|
||||
<div class="block"></div>
|
||||
<div class="bottom">FULL PAGE MARKER AT Y=1370</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,166 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Paseo Browser Capture Harness</title>
|
||||
<style>
|
||||
html,
|
||||
body {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
background: #202020;
|
||||
overflow: hidden;
|
||||
font-family: Arial, sans-serif;
|
||||
}
|
||||
|
||||
#paseo-browser-resident-webviews {
|
||||
position: fixed;
|
||||
left: -20000px;
|
||||
top: 0;
|
||||
width: 1280px;
|
||||
height: 800px;
|
||||
overflow: hidden;
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
#target-webview {
|
||||
display: inline-flex;
|
||||
flex: 0 0 auto;
|
||||
width: 1280px;
|
||||
height: 800px;
|
||||
border: 0;
|
||||
background: transparent;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="paseo-browser-resident-webviews" aria-hidden="true"></div>
|
||||
|
||||
<script>
|
||||
const RESIDENT_VIEWPORT_WIDTH = 1280;
|
||||
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
||||
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
const activeTokens = new Set();
|
||||
let nextTokenId = 0;
|
||||
|
||||
const webview = document.createElement("webview");
|
||||
webview.id = "target-webview";
|
||||
webview.setAttribute("data-paseo-browser-id", "capture-harness");
|
||||
webview.setAttribute("partition", "persist:paseo-capture-harness");
|
||||
webview.setAttribute("allowpopups", "true");
|
||||
webview.setAttribute("spellcheck", "false");
|
||||
webview.setAttribute("autosize", "on");
|
||||
webview.src = params.get("targetUrl") || "bright.html";
|
||||
host.appendChild(webview);
|
||||
|
||||
function applyParking() {
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "-20000px";
|
||||
host.style.top = "0";
|
||||
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "0";
|
||||
host.style.pointerEvents = "none";
|
||||
host.style.zIndex = "";
|
||||
host.style.clipPath = "";
|
||||
host.style.visibility = "";
|
||||
host.style.transform = "";
|
||||
webview.style.display = "inline-flex";
|
||||
webview.style.flex = "0 0 auto";
|
||||
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||
webview.style.border = "0";
|
||||
webview.style.background = "transparent";
|
||||
}
|
||||
|
||||
function applyCapturePrep() {
|
||||
host.style.position = "fixed";
|
||||
host.style.left = "0";
|
||||
host.style.top = "0";
|
||||
host.style.width = "1px";
|
||||
host.style.height = "1px";
|
||||
host.style.overflow = "hidden";
|
||||
host.style.opacity = "1";
|
||||
host.style.pointerEvents = "none";
|
||||
host.style.zIndex = "1";
|
||||
host.style.clipPath = "";
|
||||
host.style.visibility = "";
|
||||
host.style.transform = "";
|
||||
}
|
||||
|
||||
function waitFrames(count) {
|
||||
return new Promise((resolve) => {
|
||||
function step(remaining) {
|
||||
if (remaining <= 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
requestAnimationFrame(() => step(remaining - 1));
|
||||
}
|
||||
step(count);
|
||||
});
|
||||
}
|
||||
|
||||
function state() {
|
||||
const hostRect = host.getBoundingClientRect();
|
||||
const webviewRect = webview.getBoundingClientRect();
|
||||
return {
|
||||
hostStyle: {
|
||||
left: host.style.left,
|
||||
top: host.style.top,
|
||||
width: host.style.width,
|
||||
height: host.style.height,
|
||||
overflow: host.style.overflow,
|
||||
opacity: host.style.opacity,
|
||||
pointerEvents: host.style.pointerEvents,
|
||||
},
|
||||
hostRect: {
|
||||
x: hostRect.x,
|
||||
y: hostRect.y,
|
||||
width: hostRect.width,
|
||||
height: hostRect.height,
|
||||
},
|
||||
webviewRect: {
|
||||
x: webviewRect.x,
|
||||
y: webviewRect.y,
|
||||
width: webviewRect.width,
|
||||
height: webviewRect.height,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
window.captureHarness = {
|
||||
async waitForFrames(count = 2) {
|
||||
await waitFrames(count);
|
||||
return state();
|
||||
},
|
||||
async prepareForPixelCapture() {
|
||||
const token = `capture-${++nextTokenId}`;
|
||||
activeTokens.add(token);
|
||||
applyCapturePrep();
|
||||
await waitFrames(2);
|
||||
webview.getBoundingClientRect();
|
||||
return { token, state: state() };
|
||||
},
|
||||
async restorePixelCapture(token) {
|
||||
activeTokens.delete(token);
|
||||
if (activeTokens.size === 0) {
|
||||
applyParking();
|
||||
}
|
||||
await waitFrames(2);
|
||||
return state();
|
||||
},
|
||||
async restoreParking() {
|
||||
activeTokens.clear();
|
||||
applyParking();
|
||||
await waitFrames(2);
|
||||
return state();
|
||||
},
|
||||
};
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,378 @@
|
||||
const fs = require("node:fs");
|
||||
const fsp = require("node:fs/promises");
|
||||
const path = require("node:path");
|
||||
const { app, BrowserWindow, nativeImage } = require("electron");
|
||||
|
||||
const ROOT = __dirname;
|
||||
const OUT_DIR = process.env.PASEO_CAPTURE_HARNESS_OUT_DIR || path.join(ROOT, "out");
|
||||
const VIEWPORT_WIDTH = 1280;
|
||||
const VIEWPORT_HEIGHT = 800;
|
||||
const FULL_PAGE_HEIGHT = 1600;
|
||||
const CAPTURE_TIMEOUT_MS = 5000;
|
||||
const REPEAT_COUNT = 5;
|
||||
|
||||
function fileUrl(filePath) {
|
||||
return new URL(`file://${filePath}`).toString();
|
||||
}
|
||||
|
||||
function ensureDirSync(dir) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
function withTimeout(promise, label) {
|
||||
let timeoutId;
|
||||
const timeout = new Promise((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
reject(new Error(`${label} timed out after ${CAPTURE_TIMEOUT_MS}ms`));
|
||||
}, CAPTURE_TIMEOUT_MS);
|
||||
});
|
||||
return Promise.race([promise, timeout]).finally(() => {
|
||||
clearTimeout(timeoutId);
|
||||
});
|
||||
}
|
||||
|
||||
function pixelOffset(width, x, y) {
|
||||
return (y * width + x) * 4;
|
||||
}
|
||||
|
||||
function isBrightMagenta(bitmap, offset) {
|
||||
const c0 = bitmap[offset];
|
||||
const c1 = bitmap[offset + 1];
|
||||
const c2 = bitmap[offset + 2];
|
||||
return c0 > 200 && c1 < 90 && c2 > 200;
|
||||
}
|
||||
|
||||
function analyzeImage(image, expected, guestMetrics) {
|
||||
if (!image || image.isEmpty()) {
|
||||
return {
|
||||
width: 0,
|
||||
height: 0,
|
||||
brightRatio: 0,
|
||||
textNonUniform: false,
|
||||
pass: false,
|
||||
};
|
||||
}
|
||||
|
||||
const size = image.getSize();
|
||||
const width = size.width;
|
||||
const height = size.height;
|
||||
const bitmap = image.toBitmap();
|
||||
const totalPixels = width * height;
|
||||
let brightPixels = 0;
|
||||
for (let offset = 0; offset < bitmap.length; offset += 4) {
|
||||
if (isBrightMagenta(bitmap, offset)) {
|
||||
brightPixels += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const crop = {
|
||||
left: Math.min(40, Math.max(0, width - 1)),
|
||||
top: Math.min(40, Math.max(0, height - 1)),
|
||||
right: Math.min(width, 940),
|
||||
bottom: Math.min(height, 260),
|
||||
};
|
||||
let cropPixels = 0;
|
||||
let cropNonBright = 0;
|
||||
let luminanceSum = 0;
|
||||
let luminanceSqSum = 0;
|
||||
const quantized = new Set();
|
||||
for (let y = crop.top; y < crop.bottom; y += 1) {
|
||||
for (let x = crop.left; x < crop.right; x += 1) {
|
||||
const offset = pixelOffset(width, x, y);
|
||||
cropPixels += 1;
|
||||
if (!isBrightMagenta(bitmap, offset)) {
|
||||
cropNonBright += 1;
|
||||
}
|
||||
const r = bitmap[offset + 2];
|
||||
const g = bitmap[offset + 1];
|
||||
const b = bitmap[offset];
|
||||
const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
||||
luminanceSum += luma;
|
||||
luminanceSqSum += luma * luma;
|
||||
quantized.add(`${r >> 5},${g >> 5},${b >> 5},${bitmap[offset + 3] >> 6}`);
|
||||
}
|
||||
}
|
||||
|
||||
const devicePixelRatio =
|
||||
typeof guestMetrics.devicePixelRatio === "number" && guestMetrics.devicePixelRatio > 0
|
||||
? guestMetrics.devicePixelRatio
|
||||
: 1;
|
||||
const sizeTargets = [
|
||||
{ width: expected.width, height: expected.height },
|
||||
{
|
||||
width: Math.round(expected.width * devicePixelRatio),
|
||||
height: Math.round(expected.height * devicePixelRatio),
|
||||
},
|
||||
];
|
||||
const matchedSize = sizeTargets.some(
|
||||
(target) => Math.abs(width - target.width) <= 2 && Math.abs(height - target.height) <= 2,
|
||||
);
|
||||
const luminanceMean = cropPixels ? luminanceSum / cropPixels : 0;
|
||||
const luminanceVariance = cropPixels
|
||||
? luminanceSqSum / cropPixels - luminanceMean * luminanceMean
|
||||
: 0;
|
||||
const brightRatio = totalPixels ? brightPixels / totalPixels : 0;
|
||||
const textNonUniform =
|
||||
cropPixels > 0 &&
|
||||
cropNonBright / cropPixels > 0.02 &&
|
||||
quantized.size >= 4 &&
|
||||
luminanceVariance > 100;
|
||||
|
||||
return {
|
||||
width,
|
||||
height,
|
||||
logicalWidthAtDpr: width / devicePixelRatio,
|
||||
logicalHeightAtDpr: height / devicePixelRatio,
|
||||
brightRatio,
|
||||
textNonUniform,
|
||||
matchedSize,
|
||||
pass: matchedSize && brightRatio >= expected.minBrightRatio && textNonUniform,
|
||||
};
|
||||
}
|
||||
|
||||
function pass(message) {
|
||||
console.log(`PASS ${message}`);
|
||||
}
|
||||
|
||||
function fail(message) {
|
||||
console.log(`FAIL ${message}`);
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
async function saveImage(image, outputPath) {
|
||||
await fsp.writeFile(outputPath, image.toPNG());
|
||||
}
|
||||
|
||||
async function waitForGuestLoad(contents) {
|
||||
await new Promise((resolve) => {
|
||||
if (!contents.isLoading()) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
contents.once("did-finish-load", resolve);
|
||||
contents.once("did-fail-load", resolve);
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
async function renderer(win, expression) {
|
||||
return await win.webContents.executeJavaScript(expression, true);
|
||||
}
|
||||
|
||||
async function readGuestMetrics(contents) {
|
||||
return await contents.executeJavaScript(
|
||||
`({
|
||||
innerWidth: window.innerWidth,
|
||||
innerHeight: window.innerHeight,
|
||||
devicePixelRatio: window.devicePixelRatio,
|
||||
documentClientWidth: document.documentElement.clientWidth,
|
||||
documentClientHeight: document.documentElement.clientHeight,
|
||||
scrollWidth: document.documentElement.scrollWidth,
|
||||
scrollHeight: document.documentElement.scrollHeight,
|
||||
visualViewport: window.visualViewport ? {
|
||||
width: window.visualViewport.width,
|
||||
height: window.visualViewport.height,
|
||||
scale: window.visualViewport.scale
|
||||
} : null
|
||||
})`,
|
||||
true,
|
||||
);
|
||||
}
|
||||
|
||||
async function capturePageSequence(contents) {
|
||||
const previousBackgroundThrottling = contents.getBackgroundThrottling();
|
||||
contents.setBackgroundThrottling(false);
|
||||
try {
|
||||
contents.invalidate();
|
||||
return await withTimeout(contents.capturePage(undefined, { stayHidden: false }), "capturePage");
|
||||
} finally {
|
||||
contents.setBackgroundThrottling(previousBackgroundThrottling);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureFullPage(contents) {
|
||||
let attachedHere = false;
|
||||
if (!contents.debugger.isAttached()) {
|
||||
contents.debugger.attach("1.3");
|
||||
attachedHere = true;
|
||||
}
|
||||
try {
|
||||
const metrics = await contents.debugger.sendCommand("Page.getLayoutMetrics");
|
||||
const contentSize = metrics.cssContentSize ||
|
||||
metrics.contentSize || {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: VIEWPORT_WIDTH,
|
||||
height: FULL_PAGE_HEIGHT,
|
||||
};
|
||||
const clip = {
|
||||
x: Math.floor(contentSize.x || 0),
|
||||
y: Math.floor(contentSize.y || 0),
|
||||
width: Math.ceil(contentSize.width || VIEWPORT_WIDTH),
|
||||
height: Math.ceil(contentSize.height || FULL_PAGE_HEIGHT),
|
||||
scale: 1,
|
||||
};
|
||||
const result = await withTimeout(
|
||||
contents.debugger.sendCommand("Page.captureScreenshot", {
|
||||
format: "png",
|
||||
captureBeyondViewport: true,
|
||||
clip,
|
||||
}),
|
||||
"CDP Page.captureScreenshot",
|
||||
);
|
||||
return nativeImage.createFromBuffer(Buffer.from(result.data, "base64"));
|
||||
} finally {
|
||||
if (attachedHere && contents.debugger.isAttached()) {
|
||||
contents.debugger.detach();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureWithPrep({ win, contents, mode, repeatIndex, guestMetrics }) {
|
||||
const preparation = await renderer(win, "window.captureHarness.prepareForPixelCapture()");
|
||||
const outputPath = path.join(OUT_DIR, `${mode}-prep-${repeatIndex}.png`);
|
||||
try {
|
||||
const image =
|
||||
mode === "viewport" ? await capturePageSequence(contents) : await captureFullPage(contents);
|
||||
await saveImage(image, outputPath);
|
||||
const expected =
|
||||
mode === "viewport"
|
||||
? { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 }
|
||||
: { width: VIEWPORT_WIDTH, height: FULL_PAGE_HEIGHT, minBrightRatio: 0.55 };
|
||||
const analysis = analyzeImage(image, expected, guestMetrics);
|
||||
const size = `${analysis.width}x${analysis.height}`;
|
||||
const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`;
|
||||
const bright = analysis.brightRatio.toFixed(4);
|
||||
if (!analysis.pass) {
|
||||
fail(
|
||||
`${mode} prep ${repeatIndex}/${REPEAT_COUNT} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
|
||||
);
|
||||
}
|
||||
pass(
|
||||
`${mode} prep ${repeatIndex}/${REPEAT_COUNT} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
|
||||
);
|
||||
return analysis;
|
||||
} finally {
|
||||
const restoredState = await renderer(
|
||||
win,
|
||||
`window.captureHarness.restorePixelCapture(${JSON.stringify(preparation.token)})`,
|
||||
);
|
||||
const style = restoredState.hostStyle;
|
||||
if (style.left !== "-20000px" || style.opacity !== "0") {
|
||||
fail(`restore left=${style.left} opacity=${style.opacity}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
ensureDirSync(OUT_DIR);
|
||||
|
||||
let resolveGuest;
|
||||
const guestPromise = new Promise((resolve) => {
|
||||
resolveGuest = resolve;
|
||||
});
|
||||
const win = new BrowserWindow({
|
||||
width: 1000,
|
||||
height: 700,
|
||||
show: true,
|
||||
backgroundColor: "#202020",
|
||||
webPreferences: {
|
||||
webviewTag: true,
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
sandbox: false,
|
||||
},
|
||||
});
|
||||
|
||||
win.webContents.on("will-attach-webview", (_event, webPreferences) => {
|
||||
webPreferences.nodeIntegration = false;
|
||||
webPreferences.contextIsolation = true;
|
||||
});
|
||||
win.webContents.on("did-attach-webview", (_event, contents) => {
|
||||
resolveGuest(contents);
|
||||
});
|
||||
|
||||
await win.loadFile(path.join(ROOT, "index.html"), {
|
||||
query: { targetUrl: fileUrl(path.join(ROOT, "bright.html")) },
|
||||
});
|
||||
const guest = await withTimeout(guestPromise, "did-attach-webview");
|
||||
await waitForGuestLoad(guest);
|
||||
await renderer(win, "window.captureHarness.waitForFrames(2)");
|
||||
const guestMetrics = await readGuestMetrics(guest);
|
||||
|
||||
if (guestMetrics.innerWidth !== VIEWPORT_WIDTH || guestMetrics.innerHeight !== VIEWPORT_HEIGHT) {
|
||||
fail(
|
||||
`guest viewport sizing inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} expected=${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
|
||||
);
|
||||
}
|
||||
pass(
|
||||
`guest viewport sizing inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} dpr=${guestMetrics.devicePixelRatio}`,
|
||||
);
|
||||
|
||||
await renderer(win, "window.captureHarness.restoreParking()");
|
||||
try {
|
||||
const image = await capturePageSequence(guest);
|
||||
const analysis = analyzeImage(
|
||||
image,
|
||||
{ width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 },
|
||||
guestMetrics,
|
||||
);
|
||||
fail(
|
||||
`parked webview unexpectedly captured size=${analysis.width}x${analysis.height} bright=${analysis.brightRatio.toFixed(4)}`,
|
||||
);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
pass(`parked webview has no copyable viewport frame error=${message}`);
|
||||
}
|
||||
|
||||
const results = [];
|
||||
for (let index = 1; index <= REPEAT_COUNT; index += 1) {
|
||||
results.push(
|
||||
await captureWithPrep({
|
||||
win,
|
||||
contents: guest,
|
||||
mode: "viewport",
|
||||
repeatIndex: index,
|
||||
guestMetrics,
|
||||
}),
|
||||
);
|
||||
}
|
||||
for (let index = 1; index <= REPEAT_COUNT; index += 1) {
|
||||
results.push(
|
||||
await captureWithPrep({
|
||||
win,
|
||||
contents: guest,
|
||||
mode: "full-page",
|
||||
repeatIndex: index,
|
||||
guestMetrics,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
await fsp.writeFile(
|
||||
path.join(OUT_DIR, "results.json"),
|
||||
`${JSON.stringify({ generatedAt: new Date().toISOString(), guestMetrics, results }, null, 2)}\n`,
|
||||
);
|
||||
pass(`capture harness complete output=${OUT_DIR}`);
|
||||
|
||||
if (!win.isDestroyed()) {
|
||||
win.close();
|
||||
}
|
||||
}
|
||||
|
||||
app
|
||||
.whenReady()
|
||||
.then(main)
|
||||
.then(() => app.quit())
|
||||
.catch(async (error) => {
|
||||
console.error(error);
|
||||
try {
|
||||
await fsp.writeFile(
|
||||
path.join(OUT_DIR, "fatal-error.txt"),
|
||||
`${error && error.stack ? error.stack : String(error)}\n`,
|
||||
);
|
||||
} catch {
|
||||
// Ignore reporting failures during shutdown.
|
||||
}
|
||||
app.exit(1);
|
||||
});
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd)
|
||||
|
||||
exec "$REPO_ROOT/node_modules/.bin/electron" "$SCRIPT_DIR/main.js"
|
||||
@@ -17,6 +17,7 @@
|
||||
"scripts": {
|
||||
"build": "npm --prefix ../.. run build:server:clean && npm run build:main && electron-builder --config electron-builder.yml",
|
||||
"build:main": "tsc -p tsconfig.json",
|
||||
"capture-harness": "./capture-harness/run.sh",
|
||||
"dev": "./scripts/dev.sh",
|
||||
"dev:win": "powershell ./scripts/dev.ps1",
|
||||
"test": "vitest run",
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
import type { Rectangle } from "electron";
|
||||
import { describe, expect, test } from "vitest";
|
||||
import type { TabImage } from "./service.js";
|
||||
import { adaptWebContents } from "./ipc.js";
|
||||
|
||||
class FakeImage implements TabImage {
|
||||
public toPNG(): Uint8Array {
|
||||
return new Uint8Array([137, 80, 78, 71]);
|
||||
}
|
||||
|
||||
public getSize(): { width: number; height: number } {
|
||||
return { width: 640, height: 480 };
|
||||
}
|
||||
}
|
||||
|
||||
class FakeDebugger {
|
||||
public isAttached(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public attach(): void {}
|
||||
|
||||
public async sendCommand(): Promise<unknown> {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
class FakeHostWebContents {
|
||||
public readonly sentMessages: Array<{ channel: string; payload: unknown }> = [];
|
||||
public destroyed = false;
|
||||
|
||||
public constructor(public readonly id: number) {}
|
||||
|
||||
public isDestroyed(): boolean {
|
||||
return this.destroyed;
|
||||
}
|
||||
|
||||
public send(channel: string, payload: unknown): void {
|
||||
this.sentMessages.push({ channel, payload });
|
||||
}
|
||||
}
|
||||
|
||||
type IpcListener = (event: unknown, payload: unknown) => void;
|
||||
|
||||
class FakeIpcBridge {
|
||||
private readonly listeners = new Map<string, IpcListener[]>();
|
||||
|
||||
public on(channel: string, listener: IpcListener): void {
|
||||
const listeners = this.listeners.get(channel) ?? [];
|
||||
listeners.push(listener);
|
||||
this.listeners.set(channel, listeners);
|
||||
}
|
||||
|
||||
public removeListener(channel: string, listener: IpcListener): void {
|
||||
const listeners = this.listeners.get(channel) ?? [];
|
||||
this.listeners.set(
|
||||
channel,
|
||||
listeners.filter((candidate) => candidate !== listener),
|
||||
);
|
||||
}
|
||||
|
||||
public emit(channel: string, payload: unknown): void {
|
||||
for (const listener of this.listeners.get(channel) ?? []) {
|
||||
listener({}, payload);
|
||||
}
|
||||
}
|
||||
|
||||
public listenerCount(channel: string): number {
|
||||
return this.listeners.get(channel)?.length ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
class FakeWebContents {
|
||||
public readonly debugger = new FakeDebugger();
|
||||
public readonly consoleMessages: unknown[] = [];
|
||||
public readonly destroyedListeners: Array<() => void> = [];
|
||||
public destroyed = false;
|
||||
|
||||
public constructor(
|
||||
public readonly id: number,
|
||||
public hostWebContents: FakeHostWebContents | null,
|
||||
) {}
|
||||
|
||||
public getURL(): string {
|
||||
return "https://example.com";
|
||||
}
|
||||
|
||||
public getTitle(): string {
|
||||
return "Example";
|
||||
}
|
||||
|
||||
public canGoBack(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public canGoForward(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public isLoading(): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
public isDestroyed(): boolean {
|
||||
return this.destroyed;
|
||||
}
|
||||
|
||||
public async executeJavaScript(): Promise<unknown> {
|
||||
return null;
|
||||
}
|
||||
|
||||
public async loadURL(): Promise<void> {}
|
||||
|
||||
public goBack(): void {}
|
||||
|
||||
public goForward(): void {}
|
||||
|
||||
public reload(): void {}
|
||||
|
||||
public async capturePage(
|
||||
_rect?: Rectangle,
|
||||
_options?: { stayHidden?: boolean },
|
||||
): Promise<TabImage> {
|
||||
return new FakeImage();
|
||||
}
|
||||
|
||||
public invalidate(): void {}
|
||||
|
||||
public getBackgroundThrottling(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
public setBackgroundThrottling(): void {}
|
||||
|
||||
public on(
|
||||
event: "console-message",
|
||||
listener: (
|
||||
event: unknown,
|
||||
level: unknown,
|
||||
message: unknown,
|
||||
line: unknown,
|
||||
sourceId: unknown,
|
||||
) => void,
|
||||
): void {
|
||||
this.consoleMessages.push({ event, listener });
|
||||
}
|
||||
|
||||
public once(event: "destroyed", listener: () => void): void {
|
||||
expect(event).toBe("destroyed");
|
||||
this.destroyedListeners.push(listener);
|
||||
}
|
||||
}
|
||||
|
||||
describe("browser automation IPC adapter", () => {
|
||||
test("prepareForPixelCapture asks the embedder renderer and resolves the ack token", async () => {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => "prepare-1",
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-prepare",
|
||||
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||
},
|
||||
]);
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "other",
|
||||
ok: true,
|
||||
token: "wrong-token",
|
||||
});
|
||||
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1);
|
||||
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: true,
|
||||
token: "token-a",
|
||||
});
|
||||
|
||||
await expect(preparation).resolves.toEqual({ token: "token-a" });
|
||||
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
|
||||
});
|
||||
|
||||
test("restorePixelCapture sends the capture token back to the embedder renderer", async () => {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => "restore-1",
|
||||
});
|
||||
|
||||
const restored = tab.restorePixelCapture({ token: "token-a" });
|
||||
|
||||
expect(host.sentMessages).toEqual([
|
||||
{
|
||||
channel: "paseo:browser:capture-restore",
|
||||
payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" },
|
||||
},
|
||||
]);
|
||||
ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true });
|
||||
|
||||
await expect(restored).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture rejects when the renderer reports preparation failure", async () => {
|
||||
const host = new FakeHostWebContents(10);
|
||||
const contents = new FakeWebContents(20, host);
|
||||
const ipc = new FakeIpcBridge();
|
||||
const tab = adaptWebContents(contents, "browser-a", {
|
||||
ipc,
|
||||
createRequestId: () => "prepare-1",
|
||||
});
|
||||
|
||||
const preparation = tab.prepareForPixelCapture();
|
||||
ipc.emit("paseo:browser:capture-prepared", {
|
||||
requestId: "prepare-1",
|
||||
ok: false,
|
||||
message: "renderer could not prep",
|
||||
});
|
||||
|
||||
await expect(preparation).rejects.toThrow("renderer could not prep");
|
||||
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
|
||||
});
|
||||
|
||||
test("prepareForPixelCapture rejects when the guest has no embedder renderer", async () => {
|
||||
const contents = new FakeWebContents(20, null);
|
||||
const tab = adaptWebContents(contents, "browser-a");
|
||||
|
||||
await expect(tab.prepareForPixelCapture()).rejects.toThrow(
|
||||
"Browser host renderer is not available.",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,8 @@
|
||||
import type { WebContents } from "electron";
|
||||
import type { Rectangle } from "electron";
|
||||
import { ipcMain } from "electron";
|
||||
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { TabContents, BrowserRegistry } from "./service.js";
|
||||
import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
|
||||
import { executeAutomationCommand } from "./service.js";
|
||||
import {
|
||||
listRegisteredPaseoBrowserIds,
|
||||
@@ -13,14 +13,86 @@ import {
|
||||
} from "../browser-webviews/index.js";
|
||||
|
||||
const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
|
||||
const PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS = 5_000;
|
||||
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
|
||||
const observedContentsIds = new Set<number>();
|
||||
let nextPixelCaptureBridgeRequest = 0;
|
||||
|
||||
interface IpcHandlerRegistry {
|
||||
handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
|
||||
}
|
||||
|
||||
function adaptWebContents(contents: WebContents): TabContents {
|
||||
type IpcListener = (event: unknown, payload: unknown) => void;
|
||||
|
||||
interface IpcCaptureBridge {
|
||||
on(channel: string, listener: IpcListener): void;
|
||||
removeListener(channel: string, listener: IpcListener): void;
|
||||
}
|
||||
|
||||
interface HostWebContents {
|
||||
readonly id: number;
|
||||
isDestroyed(): boolean;
|
||||
send(channel: string, payload: unknown): void;
|
||||
}
|
||||
|
||||
interface WebContentsDebugger {
|
||||
isAttached(): boolean;
|
||||
attach(protocolVersion?: string): void;
|
||||
sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface ConsoleMessageEmitter {
|
||||
on(
|
||||
event: "console-message",
|
||||
listener: (
|
||||
event: unknown,
|
||||
level: unknown,
|
||||
message: unknown,
|
||||
line: unknown,
|
||||
sourceId: unknown,
|
||||
) => void,
|
||||
): void;
|
||||
once(event: "destroyed", listener: () => void): void;
|
||||
}
|
||||
|
||||
interface BrowserAutomationWebContents extends ConsoleMessageEmitter {
|
||||
readonly id: number;
|
||||
readonly hostWebContents: HostWebContents | null;
|
||||
readonly debugger: WebContentsDebugger;
|
||||
getURL(): string;
|
||||
getTitle(): string;
|
||||
canGoBack(): boolean;
|
||||
canGoForward(): boolean;
|
||||
isLoading(): boolean;
|
||||
isDestroyed(): boolean;
|
||||
executeJavaScript(code: string): Promise<unknown>;
|
||||
loadURL(url: string): Promise<void>;
|
||||
goBack(): void;
|
||||
goForward(): void;
|
||||
reload(): void;
|
||||
capturePage(rect?: Rectangle, options?: { stayHidden?: boolean }): Promise<TabImage>;
|
||||
invalidate(): void;
|
||||
getBackgroundThrottling(): boolean;
|
||||
setBackgroundThrottling(allowed: boolean): void;
|
||||
}
|
||||
|
||||
type PixelCaptureBridgeKind = "prepare" | "restore";
|
||||
|
||||
interface PixelCaptureBridgeSuccess {
|
||||
token?: string;
|
||||
}
|
||||
|
||||
interface PixelCaptureBridgeOptions {
|
||||
ipc?: IpcCaptureBridge;
|
||||
createRequestId?: () => string;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export function adaptWebContents(
|
||||
contents: BrowserAutomationWebContents,
|
||||
browserId: string,
|
||||
options?: PixelCaptureBridgeOptions,
|
||||
): TabContents {
|
||||
observeConsoleMessages(contents);
|
||||
return {
|
||||
id: contents.id,
|
||||
@@ -35,7 +107,19 @@ function adaptWebContents(contents: WebContents): TabContents {
|
||||
goBack: () => contents.goBack(),
|
||||
goForward: () => contents.goForward(),
|
||||
reload: () => contents.reload(),
|
||||
capturePage: (options) => contents.capturePage(undefined, options),
|
||||
capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
|
||||
prepareForPixelCapture: async () => {
|
||||
const result = await requestPixelCaptureBridge(contents, browserId, "prepare", options);
|
||||
if (!result.token) {
|
||||
throw new Error("Browser pixel capture preparation did not return a token.");
|
||||
}
|
||||
return { token: result.token };
|
||||
},
|
||||
restorePixelCapture: async (preparation) => {
|
||||
await requestPixelCaptureBridge(contents, browserId, "restore", options, {
|
||||
token: preparation.token,
|
||||
});
|
||||
},
|
||||
invalidate: () => contents.invalidate(),
|
||||
isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(),
|
||||
setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed),
|
||||
@@ -49,7 +133,101 @@ function adaptWebContents(contents: WebContents): TabContents {
|
||||
};
|
||||
}
|
||||
|
||||
function observeConsoleMessages(contents: WebContents): void {
|
||||
function requestPixelCaptureBridge(
|
||||
contents: BrowserAutomationWebContents,
|
||||
browserId: string,
|
||||
kind: PixelCaptureBridgeKind,
|
||||
options: PixelCaptureBridgeOptions | undefined,
|
||||
extraPayload?: { token: string },
|
||||
): Promise<PixelCaptureBridgeSuccess> {
|
||||
const host = contents.hostWebContents;
|
||||
if (!host || host.isDestroyed()) {
|
||||
return Promise.reject(new Error("Browser host renderer is not available."));
|
||||
}
|
||||
|
||||
const ipc = options?.ipc ?? ipcMain;
|
||||
const requestId =
|
||||
options?.createRequestId?.() ?? `browser-pixel-capture-${++nextPixelCaptureBridgeRequest}`;
|
||||
const timeoutMs = options?.timeoutMs ?? PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS;
|
||||
const requestChannel =
|
||||
kind === "prepare" ? "paseo:browser:capture-prepare" : "paseo:browser:capture-restore";
|
||||
const responseChannel =
|
||||
kind === "prepare" ? "paseo:browser:capture-prepared" : "paseo:browser:capture-restored";
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
const cleanup = () => {
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
ipc.removeListener(responseChannel, listener);
|
||||
};
|
||||
const listener: IpcListener = (_event, payload) => {
|
||||
const response = readPixelCaptureBridgeResponse(payload, requestId);
|
||||
if (!response) {
|
||||
return;
|
||||
}
|
||||
cleanup();
|
||||
if (response.ok) {
|
||||
resolve(response);
|
||||
} else {
|
||||
reject(new Error(response.message));
|
||||
}
|
||||
};
|
||||
|
||||
ipc.on(responseChannel, listener);
|
||||
timeoutId = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error(`Browser pixel capture ${kind} timed out.`));
|
||||
}, timeoutMs);
|
||||
|
||||
try {
|
||||
host.send(requestChannel, {
|
||||
requestId,
|
||||
browserId,
|
||||
...(extraPayload ? { token: extraPayload.token } : {}),
|
||||
});
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function readPixelCaptureBridgeResponse(
|
||||
payload: unknown,
|
||||
requestId: string,
|
||||
): ({ ok: true } & PixelCaptureBridgeSuccess) | { ok: false; message: string } | null {
|
||||
if (!isRecord(payload)) {
|
||||
return null;
|
||||
}
|
||||
const record = payload;
|
||||
if (record.requestId !== requestId) {
|
||||
return null;
|
||||
}
|
||||
if (record.ok === true) {
|
||||
return {
|
||||
ok: true,
|
||||
...(typeof record.token === "string" && record.token.length > 0
|
||||
? { token: record.token }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
if (record.ok === false) {
|
||||
const message =
|
||||
typeof record.message === "string" && record.message.length > 0
|
||||
? record.message
|
||||
: "Browser pixel capture bridge failed.";
|
||||
return { ok: false, message };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
|
||||
if (observedContentsIds.has(contents.id)) {
|
||||
return;
|
||||
}
|
||||
@@ -89,7 +267,7 @@ function createRegistry(): BrowserRegistry {
|
||||
listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace,
|
||||
getTabContents(browserId: string): TabContents | null {
|
||||
const contents = getPaseoBrowserWebContents(browserId);
|
||||
return contents ? adaptWebContents(contents) : null;
|
||||
return contents ? adaptWebContents(contents, browserId) : null;
|
||||
},
|
||||
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
|
||||
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,
|
||||
|
||||
@@ -7,7 +7,12 @@ import type {
|
||||
BrowserAutomationExecuteRequest,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
|
||||
import type { BrowserRegistry, TabContents, TabImage } from "./service.js";
|
||||
import type {
|
||||
BrowserRegistry,
|
||||
TabContents,
|
||||
TabImage,
|
||||
TabPixelCapturePreparation,
|
||||
} from "./service.js";
|
||||
import { executeAutomationCommand } from "./service.js";
|
||||
|
||||
const BROWSER_A = "11111111-1111-4111-8111-111111111111";
|
||||
@@ -36,6 +41,9 @@ class FakeTab implements TabContents {
|
||||
public readonly actions: string[] = [];
|
||||
public readonly capturedViewports: Array<{ stayHidden?: boolean }> = [];
|
||||
public readonly debugCommands: Array<{ command: string; params?: Record<string, unknown> }> = [];
|
||||
public readonly restoredPixelCaptureTokens: string[] = [];
|
||||
private readonly captureStartWaiters: Array<() => void> = [];
|
||||
private readonly deferredCaptures: Array<(image: TabImage) => void> = [];
|
||||
|
||||
public destroyed = false;
|
||||
public bodyText = "";
|
||||
@@ -44,6 +52,9 @@ class FakeTab implements TabContents {
|
||||
public networkEntries: unknown[] = [];
|
||||
public consoleMessages: BrowserAutomationConsoleLogEntry[] = [];
|
||||
public captureNeverPaints = false;
|
||||
public captureThrows = false;
|
||||
public deferCaptures = false;
|
||||
public prepareNeverAcks = false;
|
||||
public layoutMetrics = {
|
||||
cssLayoutViewport: { clientWidth: 390, clientHeight: 844 },
|
||||
cssContentSize: { width: 390, height: 1200 },
|
||||
@@ -52,6 +63,7 @@ class FakeTab implements TabContents {
|
||||
public documentNodeId = 1;
|
||||
public queriedNodeId = 2;
|
||||
public backgroundThrottlingAllowed = true;
|
||||
private nextPixelCapturePreparationId = 0;
|
||||
|
||||
public constructor(
|
||||
public readonly id: number,
|
||||
@@ -116,12 +128,35 @@ class FakeTab implements TabContents {
|
||||
public async capturePage(options?: { stayHidden?: boolean }): Promise<TabImage> {
|
||||
this.capturedViewports.push(options ?? {});
|
||||
this.actions.push("capture");
|
||||
this.resolveCaptureStartWaiters();
|
||||
if (this.captureThrows) {
|
||||
throw new Error("capture failed");
|
||||
}
|
||||
if (this.captureNeverPaints) {
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
if (this.deferCaptures) {
|
||||
return new Promise<TabImage>((resolve) => {
|
||||
this.deferredCaptures.push(resolve);
|
||||
});
|
||||
}
|
||||
return new FakeImage();
|
||||
}
|
||||
|
||||
public async prepareForPixelCapture(): Promise<TabPixelCapturePreparation> {
|
||||
this.actions.push("prepare");
|
||||
if (this.prepareNeverAcks) {
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
const token = `capture-${++this.nextPixelCapturePreparationId}`;
|
||||
return { token };
|
||||
}
|
||||
|
||||
public async restorePixelCapture(preparation: TabPixelCapturePreparation): Promise<void> {
|
||||
this.actions.push(`restore:${preparation.token}`);
|
||||
this.restoredPixelCaptureTokens.push(preparation.token);
|
||||
}
|
||||
|
||||
public invalidate(): void {
|
||||
this.actions.push("invalidate");
|
||||
}
|
||||
@@ -143,6 +178,7 @@ class FakeTab implements TabContents {
|
||||
command: string,
|
||||
params?: Record<string, unknown>,
|
||||
): Promise<unknown> {
|
||||
this.actions.push(`debug:${command}`);
|
||||
this.debugCommands.push({ command, ...(params ? { params } : {}) });
|
||||
if (command === "Page.getLayoutMetrics") {
|
||||
return this.layoutMetrics;
|
||||
@@ -158,6 +194,33 @@ class FakeTab implements TabContents {
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
public waitForCaptureStart(count: number): Promise<void> {
|
||||
if (this.capturedViewports.length >= count) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
this.captureStartWaiters.push(() => {
|
||||
if (this.capturedViewports.length >= count) {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public finishNextCapture(): void {
|
||||
const resolve = this.deferredCaptures.shift();
|
||||
if (!resolve) {
|
||||
throw new Error("No deferred capture is waiting");
|
||||
}
|
||||
resolve(new FakeImage());
|
||||
}
|
||||
|
||||
private resolveCaptureStartWaiters(): void {
|
||||
for (const waiter of this.captureStartWaiters.splice(0)) {
|
||||
waiter();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class FakeRegistry implements BrowserRegistry {
|
||||
@@ -852,9 +915,11 @@ describe("executeAutomationCommand", () => {
|
||||
});
|
||||
expect(browser.tab.capturedViewports).toEqual([{ stayHidden: false }]);
|
||||
expect(browser.tab.actions).toEqual([
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"capture",
|
||||
"restore:capture-1",
|
||||
"background:true",
|
||||
]);
|
||||
});
|
||||
@@ -882,9 +947,11 @@ describe("executeAutomationCommand", () => {
|
||||
},
|
||||
});
|
||||
expect(browser.tab.actions).toEqual([
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"capture",
|
||||
"restore:capture-1",
|
||||
"background:true",
|
||||
]);
|
||||
} finally {
|
||||
@@ -892,6 +959,111 @@ describe("executeAutomationCommand", () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("screenshot restores capture preparation when viewport capture fails", async () => {
|
||||
const browser = new BrowserAutomationHarness();
|
||||
browser.tab.captureThrows = true;
|
||||
|
||||
await expect(
|
||||
browser.execute({
|
||||
command: "screenshot",
|
||||
args: { browserId: BROWSER_A },
|
||||
}),
|
||||
).rejects.toThrow("capture failed");
|
||||
|
||||
expect(browser.tab.actions).toEqual([
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"capture",
|
||||
"restore:capture-1",
|
||||
"background:true",
|
||||
]);
|
||||
});
|
||||
|
||||
test("screenshot returns no-frame when capture preparation does not ack", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const browser = new BrowserAutomationHarness();
|
||||
browser.tab.prepareNeverAcks = true;
|
||||
|
||||
const resultPromise = browser.execute({
|
||||
command: "screenshot",
|
||||
args: { browserId: BROWSER_A },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
|
||||
await expect(resultPromise).resolves.toEqual({
|
||||
requestId: "req-screenshot",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "screenshot_no_frame",
|
||||
message:
|
||||
"The browser tab has no painted frame. Focus the tab in the app, then try again.",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(browser.tab.actions).toEqual(["prepare", "background:true"]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("overlapping screenshots serialize capture preparation and restore", async () => {
|
||||
const browser = new BrowserAutomationHarness();
|
||||
browser.tab.deferCaptures = true;
|
||||
|
||||
const first = browser.execute({
|
||||
command: "screenshot",
|
||||
args: { browserId: BROWSER_A },
|
||||
});
|
||||
await browser.tab.waitForCaptureStart(1);
|
||||
|
||||
const second = browser.execute(
|
||||
{
|
||||
command: "screenshot",
|
||||
args: { browserId: BROWSER_A },
|
||||
},
|
||||
{ requestId: "req-screenshot-2" },
|
||||
);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(browser.tab.actions).toEqual(["prepare", "background:false", "invalidate", "capture"]);
|
||||
|
||||
browser.tab.finishNextCapture();
|
||||
await browser.tab.waitForCaptureStart(2);
|
||||
|
||||
expect(browser.tab.actions).toEqual([
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"capture",
|
||||
"restore:capture-1",
|
||||
"background:true",
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"capture",
|
||||
]);
|
||||
|
||||
browser.tab.finishNextCapture();
|
||||
await expect(first).resolves.toMatchObject({ requestId: "req-screenshot", ok: true });
|
||||
await expect(second).resolves.toMatchObject({ requestId: "req-screenshot-2", ok: true });
|
||||
expect(browser.tab.actions).toEqual([
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"capture",
|
||||
"restore:capture-1",
|
||||
"background:true",
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"capture",
|
||||
"restore:capture-2",
|
||||
"background:true",
|
||||
]);
|
||||
});
|
||||
|
||||
test("screenshot with fullPage captures the page content area through CDP", async () => {
|
||||
const browser = new BrowserAutomationHarness();
|
||||
|
||||
@@ -923,6 +1095,44 @@ describe("executeAutomationCommand", () => {
|
||||
},
|
||||
},
|
||||
]);
|
||||
expect(browser.tab.actions).toEqual([
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"debug:Page.getLayoutMetrics",
|
||||
"debug:Page.captureScreenshot",
|
||||
"restore:capture-1",
|
||||
"background:true",
|
||||
]);
|
||||
});
|
||||
|
||||
test("screenshot with fullPage restores capture preparation when CDP returns no image", async () => {
|
||||
const browser = new BrowserAutomationHarness();
|
||||
browser.tab.fullPageScreenshotData = "";
|
||||
|
||||
const result = await browser.execute({
|
||||
command: "screenshot",
|
||||
args: { browserId: BROWSER_A, fullPage: true },
|
||||
});
|
||||
|
||||
expect(result).toEqual({
|
||||
requestId: "req-screenshot",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: "browser_screenshot fullPage returned no data",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(browser.tab.actions).toEqual([
|
||||
"prepare",
|
||||
"background:false",
|
||||
"invalidate",
|
||||
"debug:Page.getLayoutMetrics",
|
||||
"debug:Page.captureScreenshot",
|
||||
"restore:capture-1",
|
||||
"background:true",
|
||||
]);
|
||||
});
|
||||
|
||||
test("upload resolves workspace files before setting them on the file input", async () => {
|
||||
|
||||
@@ -24,6 +24,8 @@ export interface TabContents {
|
||||
goForward(): void;
|
||||
reload(): void;
|
||||
capturePage(options?: TabCapturePageOptions): Promise<TabImage>;
|
||||
prepareForPixelCapture(): Promise<TabPixelCapturePreparation>;
|
||||
restorePixelCapture(preparation: TabPixelCapturePreparation): Promise<void>;
|
||||
invalidate(): void;
|
||||
isBackgroundThrottlingAllowed(): boolean;
|
||||
setBackgroundThrottling(allowed: boolean): void;
|
||||
@@ -31,6 +33,10 @@ export interface TabContents {
|
||||
sendDebugCommand?(command: string, params?: Record<string, unknown>): Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface TabPixelCapturePreparation {
|
||||
token: string;
|
||||
}
|
||||
|
||||
export interface TabImage {
|
||||
toPNG(): Uint8Array;
|
||||
getSize(): { width: number; height: number };
|
||||
@@ -124,19 +130,56 @@ async function runSerializedPixelCapture<T>(
|
||||
}
|
||||
}
|
||||
|
||||
async function capturePaintedViewport(contents: TabContents): Promise<TabImage> {
|
||||
async function prepareForPixelCapture(contents: TabContents): Promise<TabPixelCapturePreparation> {
|
||||
try {
|
||||
return await withPixelCaptureTimeout(contents.prepareForPixelCapture());
|
||||
} catch {
|
||||
throw new ScreenshotNoFrameError();
|
||||
}
|
||||
}
|
||||
|
||||
async function restorePixelCapture(
|
||||
contents: TabContents,
|
||||
preparation: TabPixelCapturePreparation,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await withPixelCaptureTimeout(contents.restorePixelCapture(preparation));
|
||||
} catch {
|
||||
throw new ScreenshotNoFrameError();
|
||||
}
|
||||
}
|
||||
|
||||
async function runPreparedPixelCapture<T>(
|
||||
contents: TabContents,
|
||||
capture: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
return runSerializedPixelCapture(contents, async () => {
|
||||
const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed();
|
||||
contents.setBackgroundThrottling(false);
|
||||
let preparation: TabPixelCapturePreparation | null = null;
|
||||
try {
|
||||
// Offscreen-parked webview guests have no compositor surface for
|
||||
// capturePage/CDP to copy. The renderer must briefly make the host
|
||||
// paintable before capture; see ~/.paseo/plans/browser-capture-harness-results.md.
|
||||
preparation = await prepareForPixelCapture(contents);
|
||||
contents.setBackgroundThrottling(false);
|
||||
contents.invalidate();
|
||||
return await withPixelCaptureTimeout(contents.capturePage({ stayHidden: false }));
|
||||
return await withPixelCaptureTimeout(capture());
|
||||
} finally {
|
||||
contents.setBackgroundThrottling(previousBackgroundThrottling);
|
||||
try {
|
||||
if (preparation) {
|
||||
await restorePixelCapture(contents, preparation);
|
||||
}
|
||||
} finally {
|
||||
contents.setBackgroundThrottling(previousBackgroundThrottling);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function capturePaintedViewport(contents: TabContents): Promise<TabImage> {
|
||||
return runPreparedPixelCapture(contents, () => contents.capturePage({ stayHidden: false }));
|
||||
}
|
||||
|
||||
function tabInfoFromContents(
|
||||
browserId: string,
|
||||
contents: TabContents,
|
||||
@@ -860,18 +903,21 @@ async function executeFullPageScreenshot(
|
||||
if (!target.contents.sendDebugCommand) {
|
||||
return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP");
|
||||
}
|
||||
const metrics = await getCdpLayoutMetrics(target.contents);
|
||||
const width = metrics.contentWidth;
|
||||
const height = metrics.contentHeight;
|
||||
const sendDebugCommand = target.contents.sendDebugCommand.bind(target.contents);
|
||||
let screenshot: CdpCaptureScreenshotResult;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
try {
|
||||
screenshot = (await withPixelCaptureTimeout(
|
||||
target.contents.sendDebugCommand("Page.captureScreenshot", {
|
||||
screenshot = await runPreparedPixelCapture(target.contents, async () => {
|
||||
const metrics = await getCdpLayoutMetrics(target.contents);
|
||||
width = metrics.contentWidth;
|
||||
height = metrics.contentHeight;
|
||||
return (await sendDebugCommand("Page.captureScreenshot", {
|
||||
format: "png",
|
||||
captureBeyondViewport: true,
|
||||
clip: { x: 0, y: 0, width, height, scale: 1 },
|
||||
}),
|
||||
)) as CdpCaptureScreenshotResult;
|
||||
})) as CdpCaptureScreenshotResult;
|
||||
});
|
||||
} catch (error) {
|
||||
if (isScreenshotNoFrameError(error)) {
|
||||
return screenshotNoFrameFailure(requestId);
|
||||
|
||||
@@ -1,6 +1,81 @@
|
||||
import { contextBridge, ipcRenderer, webUtils } from "electron";
|
||||
|
||||
type EventHandler = (payload: unknown) => void;
|
||||
type BrowserPixelCapturePrepareHandler = (input: {
|
||||
browserId: string;
|
||||
}) => Promise<{ token: string }>;
|
||||
type BrowserPixelCaptureRestoreHandler = (input: { token: string }) => Promise<void>;
|
||||
|
||||
let prepareForPixelCaptureHandler: BrowserPixelCapturePrepareHandler | null = null;
|
||||
let restorePixelCaptureHandler: BrowserPixelCaptureRestoreHandler | null = null;
|
||||
|
||||
function readStringField(payload: unknown, key: string): string | null {
|
||||
if (!isRecord(payload)) {
|
||||
return null;
|
||||
}
|
||||
const value = payload[key];
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
ipcRenderer.on("paseo:browser:capture-prepare", async (_event, payload: unknown) => {
|
||||
const requestId = readStringField(payload, "requestId");
|
||||
const browserId = readStringField(payload, "browserId");
|
||||
if (!requestId || !browserId || !prepareForPixelCaptureHandler) {
|
||||
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||
requestId: requestId ?? "unknown",
|
||||
ok: false,
|
||||
message: "Browser pixel capture preparation is unavailable.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const preparation = await prepareForPixelCaptureHandler({ browserId });
|
||||
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||
requestId,
|
||||
ok: true,
|
||||
token: preparation.token,
|
||||
});
|
||||
} catch (error) {
|
||||
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||
requestId,
|
||||
ok: false,
|
||||
message: errorMessage(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
ipcRenderer.on("paseo:browser:capture-restore", async (_event, payload: unknown) => {
|
||||
const requestId = readStringField(payload, "requestId");
|
||||
const token = readStringField(payload, "token");
|
||||
if (!requestId || !token || !restorePixelCaptureHandler) {
|
||||
ipcRenderer.send("paseo:browser:capture-restored", {
|
||||
requestId: requestId ?? "unknown",
|
||||
ok: false,
|
||||
message: "Browser pixel capture restore is unavailable.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await restorePixelCaptureHandler({ token });
|
||||
ipcRenderer.send("paseo:browser:capture-restored", { requestId, ok: true });
|
||||
} catch (error) {
|
||||
ipcRenderer.send("paseo:browser:capture-restored", {
|
||||
requestId,
|
||||
ok: false,
|
||||
message: errorMessage(error),
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
platform: process.platform,
|
||||
@@ -90,5 +165,21 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect),
|
||||
copyElement: (payload: { text?: string; imageDataUrl?: string }) =>
|
||||
ipcRenderer.invoke("paseo:browser:copy-element", payload),
|
||||
onPrepareForPixelCapture: (handler: BrowserPixelCapturePrepareHandler): (() => void) => {
|
||||
prepareForPixelCaptureHandler = handler;
|
||||
return () => {
|
||||
if (prepareForPixelCaptureHandler === handler) {
|
||||
prepareForPixelCaptureHandler = null;
|
||||
}
|
||||
};
|
||||
},
|
||||
onRestorePixelCapture: (handler: BrowserPixelCaptureRestoreHandler): (() => void) => {
|
||||
restorePixelCaptureHandler = handler;
|
||||
return () => {
|
||||
if (restorePixelCaptureHandler === handler) {
|
||||
restorePixelCaptureHandler = null;
|
||||
}
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user