mirror of
https://github.com/getpaseo/paseo.git
synced 2026-08-14 12:23:16 +00:00
Fix browser screenshot capture for parked webviews
This commit is contained in:
57
CLAUDE.md
57
CLAUDE.md
@@ -21,34 +21,35 @@ This is an npm workspace monorepo:
|
|||||||
|
|
||||||
At the start of non-trivial work, list `docs/` and skim anything relevant to the task. When you learn something meta worth preserving — a gotcha, a convention, a workflow, a piece of system context that will outlive the current task — update an existing doc or propose a new one. Code-level facts belong in inline comments next to the code; system, process, and gotcha-level facts belong in `docs/`.
|
At the start of non-trivial work, list `docs/` and skim anything relevant to the task. When you learn something meta worth preserving — a gotcha, a convention, a workflow, a piece of system context that will outlive the current task — update an existing doc or propose a new one. Code-level facts belong in inline comments next to the code; system, process, and gotcha-level facts belong in `docs/`.
|
||||||
|
|
||||||
| Doc | What's in it |
|
| Doc | What's in it |
|
||||||
| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
|
| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
|
| [docs/product.md](docs/product.md) | What Paseo is, who it's for, where it's going |
|
||||||
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
|
| [docs/architecture.md](docs/architecture.md) | System design, package layering, WebSocket protocol, agent lifecycle, data flow |
|
||||||
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
|
| [docs/agent-lifecycle.md](docs/agent-lifecycle.md) | Agent states, parent/child relationships, archive semantics, tabs vs archive, subagents track |
|
||||||
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
|
| [docs/data-model.md](docs/data-model.md) | File-based JSON persistence, Zod schemas, atomic writes, no migrations |
|
||||||
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
|
| [docs/glossary.md](docs/glossary.md) | Authoritative terminology — UI label wins, no synonyms |
|
||||||
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
|
| [docs/coding-standards.md](docs/coding-standards.md) | Type hygiene, error handling, state design, React patterns, file organization |
|
||||||
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
|
| [docs/design.md](docs/design.md) | Theme tokens — colors, fonts, spacing, radii, icons |
|
||||||
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
|
| [docs/hover.md](docs/hover.md) | Hover — the canonical pattern (plain View + onPointerEnter/Leave, separate inner Pressable) and the three ways agents break it |
|
||||||
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
|
| [docs/unistyles.md](docs/unistyles.md) | Unistyles gotchas — `useUnistyles()` is forbidden, alternatives in order |
|
||||||
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
|
| [docs/floating-panels.md](docs/floating-panels.md) | Anchored popovers — Portal/Modal escape for Android, lifecycle gates, keyboard-shared-value, status-bar offset, the flash |
|
||||||
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
|
| [docs/expo-router.md](docs/expo-router.md) | Expo Router route ownership, startup restore, and native blank-screen gotchas |
|
||||||
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
|
| [docs/file-icons.md](docs/file-icons.md) | Material icon theme integration for the file explorer |
|
||||||
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
|
| [docs/providers.md](docs/providers.md) | Adding a new agent provider end-to-end |
|
||||||
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
|
| [docs/custom-providers.md](docs/custom-providers.md) | Custom provider config: Z.AI, Alibaba/Qwen, ACP agents, profiles, custom binaries |
|
||||||
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
|
| [docs/service-proxy.md](docs/service-proxy.md) | Service proxy: exposing workspace scripts at public URLs, DNS setup, reverse proxy config |
|
||||||
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
|
| [docs/development.md](docs/development.md) | Dev server, build sync gotchas, CLI reference, agent state, Playwright MCP |
|
||||||
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
|
| [docs/rpc-namespacing.md](docs/rpc-namespacing.md) | WebSocket RPC naming convention — dotted namespaces and `.request`/`.response` pairs |
|
||||||
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
|
| [docs/terminal-performance.md](docs/terminal-performance.md) | Terminal latency pipeline, coalescing/backpressure invariants, benchmark + perf spec usage |
|
||||||
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
|
| [docs/testing.md](docs/testing.md) | TDD workflow, determinism, real dependencies over mocks, test organization |
|
||||||
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
|
| [docs/mobile-testing.md](docs/mobile-testing.md) | Maestro and mobile test workflows |
|
||||||
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
|
| [docs/ad-hoc-daemon-testing.md](docs/ad-hoc-daemon-testing.md) | Isolated in-process daemon test harness |
|
||||||
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
|
| [docs/browser-capture-harness.md](docs/browser-capture-harness.md) | Real-Electron browser screenshot harness and compositor-surface gotcha |
|
||||||
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
|
| [docs/android.md](docs/android.md) | App variants, local/cloud builds, EAS workflows |
|
||||||
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
|
| [docs/docker.md](docs/docker.md) | Running the daemon and bundled web UI in Docker, volumes, agent images, security |
|
||||||
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
|
| [docs/release.md](docs/release.md) | Release playbook, draft releases, completion checklist |
|
||||||
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
|
| [docs/terminal-activity.md](docs/terminal-activity.md) | Terminal activity indicators — source-agnostic tracker, agent hook reporting, adding a new hook provider |
|
||||||
|
| [SECURITY.md](SECURITY.md) | Relay threat model, E2E encryption, DNS rebinding, agent auth |
|
||||||
|
|
||||||
## Quick start
|
## Quick start
|
||||||
|
|
||||||
|
|||||||
39
docs/browser-capture-harness.md
Normal file
39
docs/browser-capture-harness.md
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
# Browser Capture Harness
|
||||||
|
|
||||||
|
The desktop capture harness is the real-Electron verification path for browser screenshots.
|
||||||
|
It validates the compositor behavior that unit tests cannot see:
|
||||||
|
|
||||||
|
- the resident automation `<webview>` starts in the production parking state;
|
||||||
|
- the parked guest has no copyable viewport frame;
|
||||||
|
- the resident webview guest is sized to 1280x800 logical pixels;
|
||||||
|
- the app-style prep sequence moves the host to a paintable 1x1 clipped state, waits two animation frames and a layout read, then both viewport `capturePage` and full-page CDP screenshots return real pixels;
|
||||||
|
- restore returns the host to offscreen parking after every capture.
|
||||||
|
|
||||||
|
Run it with the repo Electron:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run capture-harness --workspace=@getpaseo/desktop
|
||||||
|
```
|
||||||
|
|
||||||
|
The harness writes PNG evidence and `results.json` to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
packages/desktop/capture-harness/out/
|
||||||
|
```
|
||||||
|
|
||||||
|
A passing run prints `PASS` lines for guest sizing, the expected parked-capture failure,
|
||||||
|
five viewport prep captures, five full-page prep captures, and final completion. The PNG
|
||||||
|
sizes may be device-pixel scaled; on a Retina display the 1280x800 logical viewport is
|
||||||
|
usually saved as 2560x1600.
|
||||||
|
|
||||||
|
## Mechanism
|
||||||
|
|
||||||
|
Electron captures copy from the guest web contents' compositor surface. A resident
|
||||||
|
webview parked at `left:-20000px` and `opacity:0` does not have a copyable surface, and
|
||||||
|
`capturePage({ stayHidden:false })` or CDP `Page.captureScreenshot` cannot rescue it.
|
||||||
|
|
||||||
|
Before pixel capture, the app renderer temporarily makes the resident host paintable:
|
||||||
|
`left:0`, `top:0`, `opacity:1`, `pointer-events:none`, host size `1x1`, and
|
||||||
|
`overflow:hidden`, with the full-size 1280x800 webview inside. Main captures only after
|
||||||
|
the renderer acknowledges two animation frames plus a `getBoundingClientRect()` read, and
|
||||||
|
the renderer restores parking in a `finally`.
|
||||||
@@ -1,6 +1,9 @@
|
|||||||
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||||
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
|
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
|
||||||
import { ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault } from "@/components/browser-webview-resident";
|
import {
|
||||||
|
ensureResidentBrowserWebview as ensureResidentBrowserWebviewDefault,
|
||||||
|
installResidentBrowserCaptureBridge,
|
||||||
|
} from "@/components/browser-webview-resident";
|
||||||
import { createWorkspaceBrowser } from "@/stores/browser-store";
|
import { createWorkspaceBrowser } from "@/stores/browser-store";
|
||||||
import {
|
import {
|
||||||
buildWorkspaceTabPersistenceKey,
|
buildWorkspaceTabPersistenceKey,
|
||||||
@@ -40,7 +43,8 @@ export function mountBrowserAutomationHandler(
|
|||||||
options: BrowserAutomationHandlerOptions,
|
options: BrowserAutomationHandlerOptions,
|
||||||
): () => void {
|
): () => void {
|
||||||
const getHost = options.getHost ?? getDesktopHost;
|
const getHost = options.getHost ?? getDesktopHost;
|
||||||
return options.client.on("browser.automation.execute.request", (request) => {
|
const uninstallCaptureBridge = installResidentBrowserCaptureBridge();
|
||||||
|
const unsubscribe = options.client.on("browser.automation.execute.request", (request) => {
|
||||||
void handleBrowserAutomationRequest({
|
void handleBrowserAutomationRequest({
|
||||||
client: options.client,
|
client: options.client,
|
||||||
getHost,
|
getHost,
|
||||||
@@ -56,6 +60,10 @@ export function mountBrowserAutomationHandler(
|
|||||||
: {}),
|
: {}),
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
return () => {
|
||||||
|
unsubscribe();
|
||||||
|
uninstallCaptureBridge();
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function mountBrowserAutomationDaemonClientHandler(
|
export function mountBrowserAutomationDaemonClientHandler(
|
||||||
|
|||||||
@@ -2,8 +2,10 @@ import { afterEach, describe, expect, it } from "vitest";
|
|||||||
import {
|
import {
|
||||||
clearResidentBrowserWebviewsForTests,
|
clearResidentBrowserWebviewsForTests,
|
||||||
ensureResidentBrowserWebview,
|
ensureResidentBrowserWebview,
|
||||||
|
prepareResidentBrowserWebviewForPixelCapture,
|
||||||
releaseResidentBrowserWebview,
|
releaseResidentBrowserWebview,
|
||||||
removeResidentBrowserWebview,
|
removeResidentBrowserWebview,
|
||||||
|
restoreResidentBrowserWebviewAfterPixelCapture,
|
||||||
takeResidentBrowserWebview,
|
takeResidentBrowserWebview,
|
||||||
} from "./browser-webview-resident";
|
} from "./browser-webview-resident";
|
||||||
|
|
||||||
@@ -22,6 +24,7 @@ describe("resident browser webviews", () => {
|
|||||||
|
|
||||||
expect(host.children).toHaveLength(0);
|
expect(host.children).toHaveLength(0);
|
||||||
expect(webview.isConnected).toBe(true);
|
expect(webview.isConnected).toBe(true);
|
||||||
|
expect(webview.style.display).toBe("inline-flex");
|
||||||
expect(webview.style.width).toBe("1280px");
|
expect(webview.style.width).toBe("1280px");
|
||||||
expect(webview.style.height).toBe("800px");
|
expect(webview.style.height).toBe("800px");
|
||||||
|
|
||||||
@@ -57,4 +60,65 @@ describe("resident browser webviews", () => {
|
|||||||
expect(webview?.isConnected).toBe(false);
|
expect(webview?.isConnected).toBe(false);
|
||||||
expect(takeResidentBrowserWebview("browser-closed")).toBeNull();
|
expect(takeResidentBrowserWebview("browser-closed")).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("temporarily makes resident webviews paintable for pixel capture", async () => {
|
||||||
|
const webview = ensureResidentBrowserWebview({
|
||||||
|
browserId: "browser-capture",
|
||||||
|
url: "https://example.com",
|
||||||
|
});
|
||||||
|
if (!webview) {
|
||||||
|
throw new Error("Expected resident browser webview");
|
||||||
|
}
|
||||||
|
|
||||||
|
const preparation = await prepareResidentBrowserWebviewForPixelCapture({
|
||||||
|
browserId: "browser-capture",
|
||||||
|
});
|
||||||
|
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||||
|
|
||||||
|
expect(preparation.token).toBe("capture-1");
|
||||||
|
expect(host?.style.left).toBe("0px");
|
||||||
|
expect(host?.style.top).toBe("0px");
|
||||||
|
expect(host?.style.width).toBe("1px");
|
||||||
|
expect(host?.style.height).toBe("1px");
|
||||||
|
expect(host?.style.overflow).toBe("hidden");
|
||||||
|
expect(host?.style.opacity).toBe("1");
|
||||||
|
expect(host?.style.pointerEvents).toBe("none");
|
||||||
|
expect(webview.style.display).toBe("inline-flex");
|
||||||
|
expect(webview.style.width).toBe("1280px");
|
||||||
|
expect(webview.style.height).toBe("800px");
|
||||||
|
|
||||||
|
await restoreResidentBrowserWebviewAfterPixelCapture(preparation);
|
||||||
|
|
||||||
|
expect(host?.style.left).toBe("-20000px");
|
||||||
|
expect(host?.style.width).toBe("1280px");
|
||||||
|
expect(host?.style.height).toBe("800px");
|
||||||
|
expect(host?.style.opacity).toBe("0");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the resident host paintable until every capture token is restored", async () => {
|
||||||
|
ensureResidentBrowserWebview({
|
||||||
|
browserId: "browser-overlap",
|
||||||
|
url: "https://example.com",
|
||||||
|
});
|
||||||
|
|
||||||
|
const first = await prepareResidentBrowserWebviewForPixelCapture({
|
||||||
|
browserId: "browser-overlap",
|
||||||
|
});
|
||||||
|
const second = await prepareResidentBrowserWebviewForPixelCapture({
|
||||||
|
browserId: "browser-overlap",
|
||||||
|
});
|
||||||
|
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||||
|
|
||||||
|
await restoreResidentBrowserWebviewAfterPixelCapture(first);
|
||||||
|
|
||||||
|
expect(host?.style.left).toBe("0px");
|
||||||
|
expect(host?.style.width).toBe("1px");
|
||||||
|
expect(host?.style.opacity).toBe("1");
|
||||||
|
|
||||||
|
await restoreResidentBrowserWebviewAfterPixelCapture(second);
|
||||||
|
|
||||||
|
expect(host?.style.left).toBe("-20000px");
|
||||||
|
expect(host?.style.width).toBe("1280px");
|
||||||
|
expect(host?.style.opacity).toBe("0");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,9 +1,16 @@
|
|||||||
|
import { getDesktopHost } from "@/desktop/host";
|
||||||
|
|
||||||
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
|
const RESIDENT_BROWSER_HOST_ID = "paseo-browser-resident-webviews";
|
||||||
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
|
const BROWSER_ID_ATTRIBUTE = "data-paseo-browser-id";
|
||||||
const RESIDENT_VIEWPORT_WIDTH = 1280;
|
const RESIDENT_VIEWPORT_WIDTH = 1280;
|
||||||
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
||||||
|
|
||||||
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
|
const residentWebviewsByBrowserId = new Map<string, HTMLElement>();
|
||||||
|
const activeCapturePreparations = new Map<string, { preparesResidentHost: boolean }>();
|
||||||
|
|
||||||
|
let captureBridgeInstallCount = 0;
|
||||||
|
let captureBridgeDisposer: (() => void) | null = null;
|
||||||
|
let nextCapturePreparationId = 0;
|
||||||
|
|
||||||
interface BrowserWebviewElement extends HTMLElement {
|
interface BrowserWebviewElement extends HTMLElement {
|
||||||
src: string;
|
src: string;
|
||||||
@@ -21,14 +28,7 @@ function readDocument(): Document | null {
|
|||||||
return typeof document === "undefined" ? null : document;
|
return typeof document === "undefined" ? null : document;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
function applyResidentHostParkingStyle(host: HTMLElement): void {
|
||||||
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
|
||||||
if (existing) {
|
|
||||||
return existing;
|
|
||||||
}
|
|
||||||
|
|
||||||
const host = ownerDocument.createElement("div");
|
|
||||||
host.id = RESIDENT_BROWSER_HOST_ID;
|
|
||||||
host.setAttribute("aria-hidden", "true");
|
host.setAttribute("aria-hidden", "true");
|
||||||
host.style.position = "fixed";
|
host.style.position = "fixed";
|
||||||
host.style.left = "-20000px";
|
host.style.left = "-20000px";
|
||||||
@@ -38,27 +38,88 @@ function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
|||||||
host.style.overflow = "hidden";
|
host.style.overflow = "hidden";
|
||||||
host.style.opacity = "0";
|
host.style.opacity = "0";
|
||||||
host.style.pointerEvents = "none";
|
host.style.pointerEvents = "none";
|
||||||
|
host.style.zIndex = "";
|
||||||
|
host.style.clipPath = "";
|
||||||
|
host.style.visibility = "";
|
||||||
|
host.style.transform = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyResidentHostCaptureStyle(host: HTMLElement): void {
|
||||||
|
host.setAttribute("aria-hidden", "true");
|
||||||
|
host.style.position = "fixed";
|
||||||
|
host.style.left = "0";
|
||||||
|
host.style.top = "0";
|
||||||
|
host.style.width = "1px";
|
||||||
|
host.style.height = "1px";
|
||||||
|
host.style.overflow = "hidden";
|
||||||
|
host.style.opacity = "1";
|
||||||
|
host.style.pointerEvents = "none";
|
||||||
|
host.style.zIndex = "1";
|
||||||
|
host.style.clipPath = "";
|
||||||
|
host.style.visibility = "";
|
||||||
|
host.style.transform = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function getResidentBrowserHost(ownerDocument: Document): HTMLElement {
|
||||||
|
const existing = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||||
|
if (existing) {
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
|
const host = ownerDocument.createElement("div");
|
||||||
|
host.id = RESIDENT_BROWSER_HOST_ID;
|
||||||
|
applyResidentHostParkingStyle(host);
|
||||||
ownerDocument.body.appendChild(host);
|
ownerDocument.body.appendChild(host);
|
||||||
return host;
|
return host;
|
||||||
}
|
}
|
||||||
|
|
||||||
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
|
function findBrowserWebview(browserId: string, ownerDocument: Document): HTMLElement | null {
|
||||||
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
|
for (const element of ownerDocument.querySelectorAll(`[${BROWSER_ID_ATTRIBUTE}]`)) {
|
||||||
|
if (!(element instanceof HTMLElement)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
|
if (element.getAttribute(BROWSER_ID_ATTRIBUTE) === browserId) {
|
||||||
return element as HTMLElement;
|
return element;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyResidentWebviewStyle(webview: HTMLElement): void {
|
function applyResidentWebviewStyle(webview: HTMLElement): void {
|
||||||
webview.style.display = "block";
|
webview.style.display = "inline-flex";
|
||||||
|
webview.style.flex = "0 0 auto";
|
||||||
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||||
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||||
webview.style.border = "0";
|
webview.style.border = "0";
|
||||||
webview.style.background = "transparent";
|
webview.style.background = "transparent";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hasActiveResidentHostPreparation(): boolean {
|
||||||
|
for (const preparation of activeCapturePreparations.values()) {
|
||||||
|
if (preparation.preparesResidentHost) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nextAnimationFrame(): Promise<void> {
|
||||||
|
if (typeof requestAnimationFrame !== "function") {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
requestAnimationFrame(() => {
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForCapturePaint(webview: HTMLElement): Promise<void> {
|
||||||
|
await nextAnimationFrame();
|
||||||
|
await nextAnimationFrame();
|
||||||
|
webview.getBoundingClientRect();
|
||||||
|
}
|
||||||
|
|
||||||
export function prepareBrowserWebview(
|
export function prepareBrowserWebview(
|
||||||
webview: HTMLElement,
|
webview: HTMLElement,
|
||||||
input: { browserId: string; initialUrl?: string | null },
|
input: { browserId: string; initialUrl?: string | null },
|
||||||
@@ -133,6 +194,92 @@ export function releaseResidentBrowserWebview(browserId: string, webview: HTMLEl
|
|||||||
getResidentBrowserHost(ownerDocument).appendChild(webview);
|
getResidentBrowserHost(ownerDocument).appendChild(webview);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function prepareResidentBrowserWebviewForPixelCapture(input: {
|
||||||
|
browserId: string;
|
||||||
|
}): Promise<{ token: string }> {
|
||||||
|
const browserId = trimNonEmpty(input.browserId);
|
||||||
|
if (!browserId) {
|
||||||
|
throw new Error("Browser id is required for pixel capture preparation.");
|
||||||
|
}
|
||||||
|
const ownerDocument = readDocument();
|
||||||
|
if (!ownerDocument) {
|
||||||
|
throw new Error("Browser pixel capture preparation requires a document.");
|
||||||
|
}
|
||||||
|
|
||||||
|
const host = getResidentBrowserHost(ownerDocument);
|
||||||
|
const webview = findBrowserWebview(browserId, ownerDocument);
|
||||||
|
if (!webview) {
|
||||||
|
throw new Error(`Browser webview ${browserId} is not mounted.`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const token = `capture-${++nextCapturePreparationId}`;
|
||||||
|
const preparesResidentHost = webview.parentElement === host;
|
||||||
|
activeCapturePreparations.set(token, { preparesResidentHost });
|
||||||
|
try {
|
||||||
|
if (preparesResidentHost) {
|
||||||
|
applyResidentHostCaptureStyle(host);
|
||||||
|
applyResidentWebviewStyle(webview);
|
||||||
|
}
|
||||||
|
await waitForCapturePaint(webview);
|
||||||
|
return { token };
|
||||||
|
} catch (error) {
|
||||||
|
activeCapturePreparations.delete(token);
|
||||||
|
if (!hasActiveResidentHostPreparation()) {
|
||||||
|
applyResidentHostParkingStyle(host);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function restoreResidentBrowserWebviewAfterPixelCapture(input: {
|
||||||
|
token: string;
|
||||||
|
}): Promise<void> {
|
||||||
|
const preparation = activeCapturePreparations.get(input.token);
|
||||||
|
if (!preparation) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
activeCapturePreparations.delete(input.token);
|
||||||
|
if (!preparation.preparesResidentHost || hasActiveResidentHostPreparation()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ownerDocument = readDocument();
|
||||||
|
if (!ownerDocument) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const host = ownerDocument.getElementById(RESIDENT_BROWSER_HOST_ID);
|
||||||
|
if (host instanceof HTMLElement) {
|
||||||
|
applyResidentHostParkingStyle(host);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function installResidentBrowserCaptureBridge(): () => void {
|
||||||
|
captureBridgeInstallCount += 1;
|
||||||
|
if (!captureBridgeDisposer) {
|
||||||
|
const browserBridge = getDesktopHost()?.browser;
|
||||||
|
const disposePrepare = browserBridge?.onPrepareForPixelCapture?.(
|
||||||
|
prepareResidentBrowserWebviewForPixelCapture,
|
||||||
|
);
|
||||||
|
const disposeRestore = browserBridge?.onRestorePixelCapture?.(
|
||||||
|
restoreResidentBrowserWebviewAfterPixelCapture,
|
||||||
|
);
|
||||||
|
captureBridgeDisposer = () => {
|
||||||
|
disposePrepare?.();
|
||||||
|
disposeRestore?.();
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
captureBridgeInstallCount = Math.max(0, captureBridgeInstallCount - 1);
|
||||||
|
if (captureBridgeInstallCount > 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
captureBridgeDisposer?.();
|
||||||
|
captureBridgeDisposer = null;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function removeResidentBrowserWebview(browserId: string): void {
|
export function removeResidentBrowserWebview(browserId: string): void {
|
||||||
const normalizedBrowserId = trimNonEmpty(browserId);
|
const normalizedBrowserId = trimNonEmpty(browserId);
|
||||||
if (!normalizedBrowserId) {
|
if (!normalizedBrowserId) {
|
||||||
@@ -149,5 +296,7 @@ export function clearResidentBrowserWebviewsForTests(): void {
|
|||||||
webview.remove();
|
webview.remove();
|
||||||
}
|
}
|
||||||
residentWebviewsByBrowserId.clear();
|
residentWebviewsByBrowserId.clear();
|
||||||
|
activeCapturePreparations.clear();
|
||||||
|
nextCapturePreparationId = 0;
|
||||||
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
|
readDocument()?.getElementById(RESIDENT_BROWSER_HOST_ID)?.remove();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -121,6 +121,10 @@ export interface DesktopBrowserShortcutEvent {
|
|||||||
action: "focus-url";
|
action: "focus-url";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface DesktopBrowserPixelCapturePreparation {
|
||||||
|
token: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface DesktopBrowserNewTabRequestEvent {
|
export interface DesktopBrowserNewTabRequestEvent {
|
||||||
sourceBrowserId: string;
|
sourceBrowserId: string;
|
||||||
url: string;
|
url: string;
|
||||||
@@ -144,6 +148,12 @@ export interface DesktopBrowserBridge {
|
|||||||
) => Promise<string | null>;
|
) => Promise<string | null>;
|
||||||
/** Copy element text and/or an image to the system clipboard from main. */
|
/** Copy element text and/or an image to the system clipboard from main. */
|
||||||
copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
|
copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
|
||||||
|
onPrepareForPixelCapture?: (
|
||||||
|
handler: (input: { browserId: string }) => Promise<DesktopBrowserPixelCapturePreparation>,
|
||||||
|
) => () => void;
|
||||||
|
onRestorePixelCapture?: (
|
||||||
|
handler: (input: DesktopBrowserPixelCapturePreparation) => Promise<void>,
|
||||||
|
) => () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface DesktopInvokeBridge {
|
export interface DesktopInvokeBridge {
|
||||||
|
|||||||
2
packages/desktop/capture-harness/.gitignore
vendored
Normal file
2
packages/desktop/capture-harness/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
out/
|
||||||
|
fatal-error.txt
|
||||||
74
packages/desktop/capture-harness/bright.html
Normal file
74
packages/desktop/capture-harness/bright.html
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>Paseo Capture Harness Target</title>
|
||||||
|
<style>
|
||||||
|
html,
|
||||||
|
body {
|
||||||
|
width: 1280px;
|
||||||
|
min-height: 1600px;
|
||||||
|
margin: 0;
|
||||||
|
background: #ff00ff;
|
||||||
|
overflow: auto;
|
||||||
|
font-family: Arial, Helvetica, sans-serif;
|
||||||
|
}
|
||||||
|
|
||||||
|
.label {
|
||||||
|
position: absolute;
|
||||||
|
left: 48px;
|
||||||
|
top: 48px;
|
||||||
|
color: #ffffff;
|
||||||
|
font-size: 72px;
|
||||||
|
font-weight: 800;
|
||||||
|
line-height: 1;
|
||||||
|
text-shadow:
|
||||||
|
5px 5px 0 #000000,
|
||||||
|
-2px -2px 0 #000000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.sub {
|
||||||
|
position: absolute;
|
||||||
|
left: 54px;
|
||||||
|
top: 150px;
|
||||||
|
color: #000000;
|
||||||
|
background: #ffffff;
|
||||||
|
font-size: 34px;
|
||||||
|
font-weight: 700;
|
||||||
|
padding: 10px 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.block {
|
||||||
|
position: absolute;
|
||||||
|
left: 56px;
|
||||||
|
top: 220px;
|
||||||
|
width: 360px;
|
||||||
|
height: 92px;
|
||||||
|
background: linear-gradient(
|
||||||
|
90deg,
|
||||||
|
#000000 0 25%,
|
||||||
|
#ffffff 25% 50%,
|
||||||
|
#000000 50% 75%,
|
||||||
|
#ffffff 75% 100%
|
||||||
|
);
|
||||||
|
border: 8px solid #000000;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bottom {
|
||||||
|
position: absolute;
|
||||||
|
left: 48px;
|
||||||
|
top: 1370px;
|
||||||
|
color: #ffffff;
|
||||||
|
font-size: 60px;
|
||||||
|
font-weight: 800;
|
||||||
|
text-shadow: 5px 5px 0 #000000;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="label">PASEO CAPTURE HARNESS</div>
|
||||||
|
<div class="sub">LOCAL MAGENTA TARGET 1280x800 VIEWPORT</div>
|
||||||
|
<div class="block"></div>
|
||||||
|
<div class="bottom">FULL PAGE MARKER AT Y=1370</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
166
packages/desktop/capture-harness/index.html
Normal file
166
packages/desktop/capture-harness/index.html
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>Paseo Browser Capture Harness</title>
|
||||||
|
<style>
|
||||||
|
html,
|
||||||
|
body {
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
margin: 0;
|
||||||
|
background: #202020;
|
||||||
|
overflow: hidden;
|
||||||
|
font-family: Arial, sans-serif;
|
||||||
|
}
|
||||||
|
|
||||||
|
#paseo-browser-resident-webviews {
|
||||||
|
position: fixed;
|
||||||
|
left: -20000px;
|
||||||
|
top: 0;
|
||||||
|
width: 1280px;
|
||||||
|
height: 800px;
|
||||||
|
overflow: hidden;
|
||||||
|
opacity: 0;
|
||||||
|
pointer-events: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
#target-webview {
|
||||||
|
display: inline-flex;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
width: 1280px;
|
||||||
|
height: 800px;
|
||||||
|
border: 0;
|
||||||
|
background: transparent;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div id="paseo-browser-resident-webviews" aria-hidden="true"></div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
const RESIDENT_VIEWPORT_WIDTH = 1280;
|
||||||
|
const RESIDENT_VIEWPORT_HEIGHT = 800;
|
||||||
|
const host = document.getElementById("paseo-browser-resident-webviews");
|
||||||
|
const params = new URLSearchParams(window.location.search);
|
||||||
|
const activeTokens = new Set();
|
||||||
|
let nextTokenId = 0;
|
||||||
|
|
||||||
|
const webview = document.createElement("webview");
|
||||||
|
webview.id = "target-webview";
|
||||||
|
webview.setAttribute("data-paseo-browser-id", "capture-harness");
|
||||||
|
webview.setAttribute("partition", "persist:paseo-capture-harness");
|
||||||
|
webview.setAttribute("allowpopups", "true");
|
||||||
|
webview.setAttribute("spellcheck", "false");
|
||||||
|
webview.setAttribute("autosize", "on");
|
||||||
|
webview.src = params.get("targetUrl") || "bright.html";
|
||||||
|
host.appendChild(webview);
|
||||||
|
|
||||||
|
function applyParking() {
|
||||||
|
host.style.position = "fixed";
|
||||||
|
host.style.left = "-20000px";
|
||||||
|
host.style.top = "0";
|
||||||
|
host.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||||
|
host.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||||
|
host.style.overflow = "hidden";
|
||||||
|
host.style.opacity = "0";
|
||||||
|
host.style.pointerEvents = "none";
|
||||||
|
host.style.zIndex = "";
|
||||||
|
host.style.clipPath = "";
|
||||||
|
host.style.visibility = "";
|
||||||
|
host.style.transform = "";
|
||||||
|
webview.style.display = "inline-flex";
|
||||||
|
webview.style.flex = "0 0 auto";
|
||||||
|
webview.style.width = `${RESIDENT_VIEWPORT_WIDTH}px`;
|
||||||
|
webview.style.height = `${RESIDENT_VIEWPORT_HEIGHT}px`;
|
||||||
|
webview.style.border = "0";
|
||||||
|
webview.style.background = "transparent";
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyCapturePrep() {
|
||||||
|
host.style.position = "fixed";
|
||||||
|
host.style.left = "0";
|
||||||
|
host.style.top = "0";
|
||||||
|
host.style.width = "1px";
|
||||||
|
host.style.height = "1px";
|
||||||
|
host.style.overflow = "hidden";
|
||||||
|
host.style.opacity = "1";
|
||||||
|
host.style.pointerEvents = "none";
|
||||||
|
host.style.zIndex = "1";
|
||||||
|
host.style.clipPath = "";
|
||||||
|
host.style.visibility = "";
|
||||||
|
host.style.transform = "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function waitFrames(count) {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
function step(remaining) {
|
||||||
|
if (remaining <= 0) {
|
||||||
|
resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
requestAnimationFrame(() => step(remaining - 1));
|
||||||
|
}
|
||||||
|
step(count);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function state() {
|
||||||
|
const hostRect = host.getBoundingClientRect();
|
||||||
|
const webviewRect = webview.getBoundingClientRect();
|
||||||
|
return {
|
||||||
|
hostStyle: {
|
||||||
|
left: host.style.left,
|
||||||
|
top: host.style.top,
|
||||||
|
width: host.style.width,
|
||||||
|
height: host.style.height,
|
||||||
|
overflow: host.style.overflow,
|
||||||
|
opacity: host.style.opacity,
|
||||||
|
pointerEvents: host.style.pointerEvents,
|
||||||
|
},
|
||||||
|
hostRect: {
|
||||||
|
x: hostRect.x,
|
||||||
|
y: hostRect.y,
|
||||||
|
width: hostRect.width,
|
||||||
|
height: hostRect.height,
|
||||||
|
},
|
||||||
|
webviewRect: {
|
||||||
|
x: webviewRect.x,
|
||||||
|
y: webviewRect.y,
|
||||||
|
width: webviewRect.width,
|
||||||
|
height: webviewRect.height,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
window.captureHarness = {
|
||||||
|
async waitForFrames(count = 2) {
|
||||||
|
await waitFrames(count);
|
||||||
|
return state();
|
||||||
|
},
|
||||||
|
async prepareForPixelCapture() {
|
||||||
|
const token = `capture-${++nextTokenId}`;
|
||||||
|
activeTokens.add(token);
|
||||||
|
applyCapturePrep();
|
||||||
|
await waitFrames(2);
|
||||||
|
webview.getBoundingClientRect();
|
||||||
|
return { token, state: state() };
|
||||||
|
},
|
||||||
|
async restorePixelCapture(token) {
|
||||||
|
activeTokens.delete(token);
|
||||||
|
if (activeTokens.size === 0) {
|
||||||
|
applyParking();
|
||||||
|
}
|
||||||
|
await waitFrames(2);
|
||||||
|
return state();
|
||||||
|
},
|
||||||
|
async restoreParking() {
|
||||||
|
activeTokens.clear();
|
||||||
|
applyParking();
|
||||||
|
await waitFrames(2);
|
||||||
|
return state();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
378
packages/desktop/capture-harness/main.js
Normal file
378
packages/desktop/capture-harness/main.js
Normal file
@@ -0,0 +1,378 @@
|
|||||||
|
const fs = require("node:fs");
|
||||||
|
const fsp = require("node:fs/promises");
|
||||||
|
const path = require("node:path");
|
||||||
|
const { app, BrowserWindow, nativeImage } = require("electron");
|
||||||
|
|
||||||
|
const ROOT = __dirname;
|
||||||
|
const OUT_DIR = process.env.PASEO_CAPTURE_HARNESS_OUT_DIR || path.join(ROOT, "out");
|
||||||
|
const VIEWPORT_WIDTH = 1280;
|
||||||
|
const VIEWPORT_HEIGHT = 800;
|
||||||
|
const FULL_PAGE_HEIGHT = 1600;
|
||||||
|
const CAPTURE_TIMEOUT_MS = 5000;
|
||||||
|
const REPEAT_COUNT = 5;
|
||||||
|
|
||||||
|
function fileUrl(filePath) {
|
||||||
|
return new URL(`file://${filePath}`).toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureDirSync(dir) {
|
||||||
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
function withTimeout(promise, label) {
|
||||||
|
let timeoutId;
|
||||||
|
const timeout = new Promise((_, reject) => {
|
||||||
|
timeoutId = setTimeout(() => {
|
||||||
|
reject(new Error(`${label} timed out after ${CAPTURE_TIMEOUT_MS}ms`));
|
||||||
|
}, CAPTURE_TIMEOUT_MS);
|
||||||
|
});
|
||||||
|
return Promise.race([promise, timeout]).finally(() => {
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function pixelOffset(width, x, y) {
|
||||||
|
return (y * width + x) * 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isBrightMagenta(bitmap, offset) {
|
||||||
|
const c0 = bitmap[offset];
|
||||||
|
const c1 = bitmap[offset + 1];
|
||||||
|
const c2 = bitmap[offset + 2];
|
||||||
|
return c0 > 200 && c1 < 90 && c2 > 200;
|
||||||
|
}
|
||||||
|
|
||||||
|
function analyzeImage(image, expected, guestMetrics) {
|
||||||
|
if (!image || image.isEmpty()) {
|
||||||
|
return {
|
||||||
|
width: 0,
|
||||||
|
height: 0,
|
||||||
|
brightRatio: 0,
|
||||||
|
textNonUniform: false,
|
||||||
|
pass: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const size = image.getSize();
|
||||||
|
const width = size.width;
|
||||||
|
const height = size.height;
|
||||||
|
const bitmap = image.toBitmap();
|
||||||
|
const totalPixels = width * height;
|
||||||
|
let brightPixels = 0;
|
||||||
|
for (let offset = 0; offset < bitmap.length; offset += 4) {
|
||||||
|
if (isBrightMagenta(bitmap, offset)) {
|
||||||
|
brightPixels += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const crop = {
|
||||||
|
left: Math.min(40, Math.max(0, width - 1)),
|
||||||
|
top: Math.min(40, Math.max(0, height - 1)),
|
||||||
|
right: Math.min(width, 940),
|
||||||
|
bottom: Math.min(height, 260),
|
||||||
|
};
|
||||||
|
let cropPixels = 0;
|
||||||
|
let cropNonBright = 0;
|
||||||
|
let luminanceSum = 0;
|
||||||
|
let luminanceSqSum = 0;
|
||||||
|
const quantized = new Set();
|
||||||
|
for (let y = crop.top; y < crop.bottom; y += 1) {
|
||||||
|
for (let x = crop.left; x < crop.right; x += 1) {
|
||||||
|
const offset = pixelOffset(width, x, y);
|
||||||
|
cropPixels += 1;
|
||||||
|
if (!isBrightMagenta(bitmap, offset)) {
|
||||||
|
cropNonBright += 1;
|
||||||
|
}
|
||||||
|
const r = bitmap[offset + 2];
|
||||||
|
const g = bitmap[offset + 1];
|
||||||
|
const b = bitmap[offset];
|
||||||
|
const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b;
|
||||||
|
luminanceSum += luma;
|
||||||
|
luminanceSqSum += luma * luma;
|
||||||
|
quantized.add(`${r >> 5},${g >> 5},${b >> 5},${bitmap[offset + 3] >> 6}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const devicePixelRatio =
|
||||||
|
typeof guestMetrics.devicePixelRatio === "number" && guestMetrics.devicePixelRatio > 0
|
||||||
|
? guestMetrics.devicePixelRatio
|
||||||
|
: 1;
|
||||||
|
const sizeTargets = [
|
||||||
|
{ width: expected.width, height: expected.height },
|
||||||
|
{
|
||||||
|
width: Math.round(expected.width * devicePixelRatio),
|
||||||
|
height: Math.round(expected.height * devicePixelRatio),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const matchedSize = sizeTargets.some(
|
||||||
|
(target) => Math.abs(width - target.width) <= 2 && Math.abs(height - target.height) <= 2,
|
||||||
|
);
|
||||||
|
const luminanceMean = cropPixels ? luminanceSum / cropPixels : 0;
|
||||||
|
const luminanceVariance = cropPixels
|
||||||
|
? luminanceSqSum / cropPixels - luminanceMean * luminanceMean
|
||||||
|
: 0;
|
||||||
|
const brightRatio = totalPixels ? brightPixels / totalPixels : 0;
|
||||||
|
const textNonUniform =
|
||||||
|
cropPixels > 0 &&
|
||||||
|
cropNonBright / cropPixels > 0.02 &&
|
||||||
|
quantized.size >= 4 &&
|
||||||
|
luminanceVariance > 100;
|
||||||
|
|
||||||
|
return {
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
logicalWidthAtDpr: width / devicePixelRatio,
|
||||||
|
logicalHeightAtDpr: height / devicePixelRatio,
|
||||||
|
brightRatio,
|
||||||
|
textNonUniform,
|
||||||
|
matchedSize,
|
||||||
|
pass: matchedSize && brightRatio >= expected.minBrightRatio && textNonUniform,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function pass(message) {
|
||||||
|
console.log(`PASS ${message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function fail(message) {
|
||||||
|
console.log(`FAIL ${message}`);
|
||||||
|
throw new Error(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveImage(image, outputPath) {
|
||||||
|
await fsp.writeFile(outputPath, image.toPNG());
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForGuestLoad(contents) {
|
||||||
|
await new Promise((resolve) => {
|
||||||
|
if (!contents.isLoading()) {
|
||||||
|
resolve();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
contents.once("did-finish-load", resolve);
|
||||||
|
contents.once("did-fail-load", resolve);
|
||||||
|
});
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function renderer(win, expression) {
|
||||||
|
return await win.webContents.executeJavaScript(expression, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readGuestMetrics(contents) {
|
||||||
|
return await contents.executeJavaScript(
|
||||||
|
`({
|
||||||
|
innerWidth: window.innerWidth,
|
||||||
|
innerHeight: window.innerHeight,
|
||||||
|
devicePixelRatio: window.devicePixelRatio,
|
||||||
|
documentClientWidth: document.documentElement.clientWidth,
|
||||||
|
documentClientHeight: document.documentElement.clientHeight,
|
||||||
|
scrollWidth: document.documentElement.scrollWidth,
|
||||||
|
scrollHeight: document.documentElement.scrollHeight,
|
||||||
|
visualViewport: window.visualViewport ? {
|
||||||
|
width: window.visualViewport.width,
|
||||||
|
height: window.visualViewport.height,
|
||||||
|
scale: window.visualViewport.scale
|
||||||
|
} : null
|
||||||
|
})`,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function capturePageSequence(contents) {
|
||||||
|
const previousBackgroundThrottling = contents.getBackgroundThrottling();
|
||||||
|
contents.setBackgroundThrottling(false);
|
||||||
|
try {
|
||||||
|
contents.invalidate();
|
||||||
|
return await withTimeout(contents.capturePage(undefined, { stayHidden: false }), "capturePage");
|
||||||
|
} finally {
|
||||||
|
contents.setBackgroundThrottling(previousBackgroundThrottling);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureFullPage(contents) {
|
||||||
|
let attachedHere = false;
|
||||||
|
if (!contents.debugger.isAttached()) {
|
||||||
|
contents.debugger.attach("1.3");
|
||||||
|
attachedHere = true;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const metrics = await contents.debugger.sendCommand("Page.getLayoutMetrics");
|
||||||
|
const contentSize = metrics.cssContentSize ||
|
||||||
|
metrics.contentSize || {
|
||||||
|
x: 0,
|
||||||
|
y: 0,
|
||||||
|
width: VIEWPORT_WIDTH,
|
||||||
|
height: FULL_PAGE_HEIGHT,
|
||||||
|
};
|
||||||
|
const clip = {
|
||||||
|
x: Math.floor(contentSize.x || 0),
|
||||||
|
y: Math.floor(contentSize.y || 0),
|
||||||
|
width: Math.ceil(contentSize.width || VIEWPORT_WIDTH),
|
||||||
|
height: Math.ceil(contentSize.height || FULL_PAGE_HEIGHT),
|
||||||
|
scale: 1,
|
||||||
|
};
|
||||||
|
const result = await withTimeout(
|
||||||
|
contents.debugger.sendCommand("Page.captureScreenshot", {
|
||||||
|
format: "png",
|
||||||
|
captureBeyondViewport: true,
|
||||||
|
clip,
|
||||||
|
}),
|
||||||
|
"CDP Page.captureScreenshot",
|
||||||
|
);
|
||||||
|
return nativeImage.createFromBuffer(Buffer.from(result.data, "base64"));
|
||||||
|
} finally {
|
||||||
|
if (attachedHere && contents.debugger.isAttached()) {
|
||||||
|
contents.debugger.detach();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureWithPrep({ win, contents, mode, repeatIndex, guestMetrics }) {
|
||||||
|
const preparation = await renderer(win, "window.captureHarness.prepareForPixelCapture()");
|
||||||
|
const outputPath = path.join(OUT_DIR, `${mode}-prep-${repeatIndex}.png`);
|
||||||
|
try {
|
||||||
|
const image =
|
||||||
|
mode === "viewport" ? await capturePageSequence(contents) : await captureFullPage(contents);
|
||||||
|
await saveImage(image, outputPath);
|
||||||
|
const expected =
|
||||||
|
mode === "viewport"
|
||||||
|
? { width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 }
|
||||||
|
: { width: VIEWPORT_WIDTH, height: FULL_PAGE_HEIGHT, minBrightRatio: 0.55 };
|
||||||
|
const analysis = analyzeImage(image, expected, guestMetrics);
|
||||||
|
const size = `${analysis.width}x${analysis.height}`;
|
||||||
|
const logicalSize = `${analysis.logicalWidthAtDpr}x${analysis.logicalHeightAtDpr}`;
|
||||||
|
const bright = analysis.brightRatio.toFixed(4);
|
||||||
|
if (!analysis.pass) {
|
||||||
|
fail(
|
||||||
|
`${mode} prep ${repeatIndex}/${REPEAT_COUNT} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
pass(
|
||||||
|
`${mode} prep ${repeatIndex}/${REPEAT_COUNT} size=${size} logical=${logicalSize} bright=${bright} text=${analysis.textNonUniform} file=${outputPath}`,
|
||||||
|
);
|
||||||
|
return analysis;
|
||||||
|
} finally {
|
||||||
|
const restoredState = await renderer(
|
||||||
|
win,
|
||||||
|
`window.captureHarness.restorePixelCapture(${JSON.stringify(preparation.token)})`,
|
||||||
|
);
|
||||||
|
const style = restoredState.hostStyle;
|
||||||
|
if (style.left !== "-20000px" || style.opacity !== "0") {
|
||||||
|
fail(`restore left=${style.left} opacity=${style.opacity}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
ensureDirSync(OUT_DIR);
|
||||||
|
|
||||||
|
let resolveGuest;
|
||||||
|
const guestPromise = new Promise((resolve) => {
|
||||||
|
resolveGuest = resolve;
|
||||||
|
});
|
||||||
|
const win = new BrowserWindow({
|
||||||
|
width: 1000,
|
||||||
|
height: 700,
|
||||||
|
show: true,
|
||||||
|
backgroundColor: "#202020",
|
||||||
|
webPreferences: {
|
||||||
|
webviewTag: true,
|
||||||
|
contextIsolation: true,
|
||||||
|
nodeIntegration: false,
|
||||||
|
sandbox: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
win.webContents.on("will-attach-webview", (_event, webPreferences) => {
|
||||||
|
webPreferences.nodeIntegration = false;
|
||||||
|
webPreferences.contextIsolation = true;
|
||||||
|
});
|
||||||
|
win.webContents.on("did-attach-webview", (_event, contents) => {
|
||||||
|
resolveGuest(contents);
|
||||||
|
});
|
||||||
|
|
||||||
|
await win.loadFile(path.join(ROOT, "index.html"), {
|
||||||
|
query: { targetUrl: fileUrl(path.join(ROOT, "bright.html")) },
|
||||||
|
});
|
||||||
|
const guest = await withTimeout(guestPromise, "did-attach-webview");
|
||||||
|
await waitForGuestLoad(guest);
|
||||||
|
await renderer(win, "window.captureHarness.waitForFrames(2)");
|
||||||
|
const guestMetrics = await readGuestMetrics(guest);
|
||||||
|
|
||||||
|
if (guestMetrics.innerWidth !== VIEWPORT_WIDTH || guestMetrics.innerHeight !== VIEWPORT_HEIGHT) {
|
||||||
|
fail(
|
||||||
|
`guest viewport sizing inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} expected=${VIEWPORT_WIDTH}x${VIEWPORT_HEIGHT}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
pass(
|
||||||
|
`guest viewport sizing inner=${guestMetrics.innerWidth}x${guestMetrics.innerHeight} dpr=${guestMetrics.devicePixelRatio}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await renderer(win, "window.captureHarness.restoreParking()");
|
||||||
|
try {
|
||||||
|
const image = await capturePageSequence(guest);
|
||||||
|
const analysis = analyzeImage(
|
||||||
|
image,
|
||||||
|
{ width: VIEWPORT_WIDTH, height: VIEWPORT_HEIGHT, minBrightRatio: 0.65 },
|
||||||
|
guestMetrics,
|
||||||
|
);
|
||||||
|
fail(
|
||||||
|
`parked webview unexpectedly captured size=${analysis.width}x${analysis.height} bright=${analysis.brightRatio.toFixed(4)}`,
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
pass(`parked webview has no copyable viewport frame error=${message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = [];
|
||||||
|
for (let index = 1; index <= REPEAT_COUNT; index += 1) {
|
||||||
|
results.push(
|
||||||
|
await captureWithPrep({
|
||||||
|
win,
|
||||||
|
contents: guest,
|
||||||
|
mode: "viewport",
|
||||||
|
repeatIndex: index,
|
||||||
|
guestMetrics,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (let index = 1; index <= REPEAT_COUNT; index += 1) {
|
||||||
|
results.push(
|
||||||
|
await captureWithPrep({
|
||||||
|
win,
|
||||||
|
contents: guest,
|
||||||
|
mode: "full-page",
|
||||||
|
repeatIndex: index,
|
||||||
|
guestMetrics,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await fsp.writeFile(
|
||||||
|
path.join(OUT_DIR, "results.json"),
|
||||||
|
`${JSON.stringify({ generatedAt: new Date().toISOString(), guestMetrics, results }, null, 2)}\n`,
|
||||||
|
);
|
||||||
|
pass(`capture harness complete output=${OUT_DIR}`);
|
||||||
|
|
||||||
|
if (!win.isDestroyed()) {
|
||||||
|
win.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
app
|
||||||
|
.whenReady()
|
||||||
|
.then(main)
|
||||||
|
.then(() => app.quit())
|
||||||
|
.catch(async (error) => {
|
||||||
|
console.error(error);
|
||||||
|
try {
|
||||||
|
await fsp.writeFile(
|
||||||
|
path.join(OUT_DIR, "fatal-error.txt"),
|
||||||
|
`${error && error.stack ? error.stack : String(error)}\n`,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Ignore reporting failures during shutdown.
|
||||||
|
}
|
||||||
|
app.exit(1);
|
||||||
|
});
|
||||||
7
packages/desktop/capture-harness/run.sh
Executable file
7
packages/desktop/capture-harness/run.sh
Executable file
@@ -0,0 +1,7 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||||
|
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../../.." && pwd)
|
||||||
|
|
||||||
|
exec "$REPO_ROOT/node_modules/.bin/electron" "$SCRIPT_DIR/main.js"
|
||||||
@@ -17,6 +17,7 @@
|
|||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "npm --prefix ../.. run build:server:clean && npm run build:main && electron-builder --config electron-builder.yml",
|
"build": "npm --prefix ../.. run build:server:clean && npm run build:main && electron-builder --config electron-builder.yml",
|
||||||
"build:main": "tsc -p tsconfig.json",
|
"build:main": "tsc -p tsconfig.json",
|
||||||
|
"capture-harness": "./capture-harness/run.sh",
|
||||||
"dev": "./scripts/dev.sh",
|
"dev": "./scripts/dev.sh",
|
||||||
"dev:win": "powershell ./scripts/dev.ps1",
|
"dev:win": "powershell ./scripts/dev.ps1",
|
||||||
"test": "vitest run",
|
"test": "vitest run",
|
||||||
|
|||||||
239
packages/desktop/src/features/browser-automation/ipc.test.ts
Normal file
239
packages/desktop/src/features/browser-automation/ipc.test.ts
Normal file
@@ -0,0 +1,239 @@
|
|||||||
|
import type { Rectangle } from "electron";
|
||||||
|
import { describe, expect, test } from "vitest";
|
||||||
|
import type { TabImage } from "./service.js";
|
||||||
|
import { adaptWebContents } from "./ipc.js";
|
||||||
|
|
||||||
|
class FakeImage implements TabImage {
|
||||||
|
public toPNG(): Uint8Array {
|
||||||
|
return new Uint8Array([137, 80, 78, 71]);
|
||||||
|
}
|
||||||
|
|
||||||
|
public getSize(): { width: number; height: number } {
|
||||||
|
return { width: 640, height: 480 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeDebugger {
|
||||||
|
public isAttached(): boolean {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public attach(): void {}
|
||||||
|
|
||||||
|
public async sendCommand(): Promise<unknown> {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeHostWebContents {
|
||||||
|
public readonly sentMessages: Array<{ channel: string; payload: unknown }> = [];
|
||||||
|
public destroyed = false;
|
||||||
|
|
||||||
|
public constructor(public readonly id: number) {}
|
||||||
|
|
||||||
|
public isDestroyed(): boolean {
|
||||||
|
return this.destroyed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public send(channel: string, payload: unknown): void {
|
||||||
|
this.sentMessages.push({ channel, payload });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type IpcListener = (event: unknown, payload: unknown) => void;
|
||||||
|
|
||||||
|
class FakeIpcBridge {
|
||||||
|
private readonly listeners = new Map<string, IpcListener[]>();
|
||||||
|
|
||||||
|
public on(channel: string, listener: IpcListener): void {
|
||||||
|
const listeners = this.listeners.get(channel) ?? [];
|
||||||
|
listeners.push(listener);
|
||||||
|
this.listeners.set(channel, listeners);
|
||||||
|
}
|
||||||
|
|
||||||
|
public removeListener(channel: string, listener: IpcListener): void {
|
||||||
|
const listeners = this.listeners.get(channel) ?? [];
|
||||||
|
this.listeners.set(
|
||||||
|
channel,
|
||||||
|
listeners.filter((candidate) => candidate !== listener),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public emit(channel: string, payload: unknown): void {
|
||||||
|
for (const listener of this.listeners.get(channel) ?? []) {
|
||||||
|
listener({}, payload);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public listenerCount(channel: string): number {
|
||||||
|
return this.listeners.get(channel)?.length ?? 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeWebContents {
|
||||||
|
public readonly debugger = new FakeDebugger();
|
||||||
|
public readonly consoleMessages: unknown[] = [];
|
||||||
|
public readonly destroyedListeners: Array<() => void> = [];
|
||||||
|
public destroyed = false;
|
||||||
|
|
||||||
|
public constructor(
|
||||||
|
public readonly id: number,
|
||||||
|
public hostWebContents: FakeHostWebContents | null,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
public getURL(): string {
|
||||||
|
return "https://example.com";
|
||||||
|
}
|
||||||
|
|
||||||
|
public getTitle(): string {
|
||||||
|
return "Example";
|
||||||
|
}
|
||||||
|
|
||||||
|
public canGoBack(): boolean {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public canGoForward(): boolean {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public isLoading(): boolean {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public isDestroyed(): boolean {
|
||||||
|
return this.destroyed;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async executeJavaScript(): Promise<unknown> {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async loadURL(): Promise<void> {}
|
||||||
|
|
||||||
|
public goBack(): void {}
|
||||||
|
|
||||||
|
public goForward(): void {}
|
||||||
|
|
||||||
|
public reload(): void {}
|
||||||
|
|
||||||
|
public async capturePage(
|
||||||
|
_rect?: Rectangle,
|
||||||
|
_options?: { stayHidden?: boolean },
|
||||||
|
): Promise<TabImage> {
|
||||||
|
return new FakeImage();
|
||||||
|
}
|
||||||
|
|
||||||
|
public invalidate(): void {}
|
||||||
|
|
||||||
|
public getBackgroundThrottling(): boolean {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public setBackgroundThrottling(): void {}
|
||||||
|
|
||||||
|
public on(
|
||||||
|
event: "console-message",
|
||||||
|
listener: (
|
||||||
|
event: unknown,
|
||||||
|
level: unknown,
|
||||||
|
message: unknown,
|
||||||
|
line: unknown,
|
||||||
|
sourceId: unknown,
|
||||||
|
) => void,
|
||||||
|
): void {
|
||||||
|
this.consoleMessages.push({ event, listener });
|
||||||
|
}
|
||||||
|
|
||||||
|
public once(event: "destroyed", listener: () => void): void {
|
||||||
|
expect(event).toBe("destroyed");
|
||||||
|
this.destroyedListeners.push(listener);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("browser automation IPC adapter", () => {
|
||||||
|
test("prepareForPixelCapture asks the embedder renderer and resolves the ack token", async () => {
|
||||||
|
const host = new FakeHostWebContents(10);
|
||||||
|
const contents = new FakeWebContents(20, host);
|
||||||
|
const ipc = new FakeIpcBridge();
|
||||||
|
const tab = adaptWebContents(contents, "browser-a", {
|
||||||
|
ipc,
|
||||||
|
createRequestId: () => "prepare-1",
|
||||||
|
});
|
||||||
|
|
||||||
|
const preparation = tab.prepareForPixelCapture();
|
||||||
|
|
||||||
|
expect(host.sentMessages).toEqual([
|
||||||
|
{
|
||||||
|
channel: "paseo:browser:capture-prepare",
|
||||||
|
payload: { requestId: "prepare-1", browserId: "browser-a" },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
ipc.emit("paseo:browser:capture-prepared", {
|
||||||
|
requestId: "other",
|
||||||
|
ok: true,
|
||||||
|
token: "wrong-token",
|
||||||
|
});
|
||||||
|
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(1);
|
||||||
|
|
||||||
|
ipc.emit("paseo:browser:capture-prepared", {
|
||||||
|
requestId: "prepare-1",
|
||||||
|
ok: true,
|
||||||
|
token: "token-a",
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(preparation).resolves.toEqual({ token: "token-a" });
|
||||||
|
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("restorePixelCapture sends the capture token back to the embedder renderer", async () => {
|
||||||
|
const host = new FakeHostWebContents(10);
|
||||||
|
const contents = new FakeWebContents(20, host);
|
||||||
|
const ipc = new FakeIpcBridge();
|
||||||
|
const tab = adaptWebContents(contents, "browser-a", {
|
||||||
|
ipc,
|
||||||
|
createRequestId: () => "restore-1",
|
||||||
|
});
|
||||||
|
|
||||||
|
const restored = tab.restorePixelCapture({ token: "token-a" });
|
||||||
|
|
||||||
|
expect(host.sentMessages).toEqual([
|
||||||
|
{
|
||||||
|
channel: "paseo:browser:capture-restore",
|
||||||
|
payload: { requestId: "restore-1", browserId: "browser-a", token: "token-a" },
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
ipc.emit("paseo:browser:capture-restored", { requestId: "restore-1", ok: true });
|
||||||
|
|
||||||
|
await expect(restored).resolves.toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("prepareForPixelCapture rejects when the renderer reports preparation failure", async () => {
|
||||||
|
const host = new FakeHostWebContents(10);
|
||||||
|
const contents = new FakeWebContents(20, host);
|
||||||
|
const ipc = new FakeIpcBridge();
|
||||||
|
const tab = adaptWebContents(contents, "browser-a", {
|
||||||
|
ipc,
|
||||||
|
createRequestId: () => "prepare-1",
|
||||||
|
});
|
||||||
|
|
||||||
|
const preparation = tab.prepareForPixelCapture();
|
||||||
|
ipc.emit("paseo:browser:capture-prepared", {
|
||||||
|
requestId: "prepare-1",
|
||||||
|
ok: false,
|
||||||
|
message: "renderer could not prep",
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(preparation).rejects.toThrow("renderer could not prep");
|
||||||
|
expect(ipc.listenerCount("paseo:browser:capture-prepared")).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("prepareForPixelCapture rejects when the guest has no embedder renderer", async () => {
|
||||||
|
const contents = new FakeWebContents(20, null);
|
||||||
|
const tab = adaptWebContents(contents, "browser-a");
|
||||||
|
|
||||||
|
await expect(tab.prepareForPixelCapture()).rejects.toThrow(
|
||||||
|
"Browser host renderer is not available.",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
import type { WebContents } from "electron";
|
import type { Rectangle } from "electron";
|
||||||
import { ipcMain } from "electron";
|
import { ipcMain } from "electron";
|
||||||
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
import { BrowserAutomationExecuteRequestSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||||
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
import type { BrowserAutomationConsoleLogEntry } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||||
import type { TabContents, BrowserRegistry } from "./service.js";
|
import type { TabContents, BrowserRegistry, TabImage } from "./service.js";
|
||||||
import { executeAutomationCommand } from "./service.js";
|
import { executeAutomationCommand } from "./service.js";
|
||||||
import {
|
import {
|
||||||
listRegisteredPaseoBrowserIds,
|
listRegisteredPaseoBrowserIds,
|
||||||
@@ -13,14 +13,86 @@ import {
|
|||||||
} from "../browser-webviews/index.js";
|
} from "../browser-webviews/index.js";
|
||||||
|
|
||||||
const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
|
const MAX_CONSOLE_MESSAGES_PER_TAB = 200;
|
||||||
|
const PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS = 5_000;
|
||||||
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
|
const consoleMessagesByContentsId = new Map<number, BrowserAutomationConsoleLogEntry[]>();
|
||||||
const observedContentsIds = new Set<number>();
|
const observedContentsIds = new Set<number>();
|
||||||
|
let nextPixelCaptureBridgeRequest = 0;
|
||||||
|
|
||||||
interface IpcHandlerRegistry {
|
interface IpcHandlerRegistry {
|
||||||
handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
|
handle(channel: string, listener: (event: unknown, ...args: unknown[]) => unknown): void;
|
||||||
}
|
}
|
||||||
|
|
||||||
function adaptWebContents(contents: WebContents): TabContents {
|
type IpcListener = (event: unknown, payload: unknown) => void;
|
||||||
|
|
||||||
|
interface IpcCaptureBridge {
|
||||||
|
on(channel: string, listener: IpcListener): void;
|
||||||
|
removeListener(channel: string, listener: IpcListener): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface HostWebContents {
|
||||||
|
readonly id: number;
|
||||||
|
isDestroyed(): boolean;
|
||||||
|
send(channel: string, payload: unknown): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface WebContentsDebugger {
|
||||||
|
isAttached(): boolean;
|
||||||
|
attach(protocolVersion?: string): void;
|
||||||
|
sendCommand(command: string, params?: Record<string, unknown>): Promise<unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ConsoleMessageEmitter {
|
||||||
|
on(
|
||||||
|
event: "console-message",
|
||||||
|
listener: (
|
||||||
|
event: unknown,
|
||||||
|
level: unknown,
|
||||||
|
message: unknown,
|
||||||
|
line: unknown,
|
||||||
|
sourceId: unknown,
|
||||||
|
) => void,
|
||||||
|
): void;
|
||||||
|
once(event: "destroyed", listener: () => void): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BrowserAutomationWebContents extends ConsoleMessageEmitter {
|
||||||
|
readonly id: number;
|
||||||
|
readonly hostWebContents: HostWebContents | null;
|
||||||
|
readonly debugger: WebContentsDebugger;
|
||||||
|
getURL(): string;
|
||||||
|
getTitle(): string;
|
||||||
|
canGoBack(): boolean;
|
||||||
|
canGoForward(): boolean;
|
||||||
|
isLoading(): boolean;
|
||||||
|
isDestroyed(): boolean;
|
||||||
|
executeJavaScript(code: string): Promise<unknown>;
|
||||||
|
loadURL(url: string): Promise<void>;
|
||||||
|
goBack(): void;
|
||||||
|
goForward(): void;
|
||||||
|
reload(): void;
|
||||||
|
capturePage(rect?: Rectangle, options?: { stayHidden?: boolean }): Promise<TabImage>;
|
||||||
|
invalidate(): void;
|
||||||
|
getBackgroundThrottling(): boolean;
|
||||||
|
setBackgroundThrottling(allowed: boolean): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
type PixelCaptureBridgeKind = "prepare" | "restore";
|
||||||
|
|
||||||
|
interface PixelCaptureBridgeSuccess {
|
||||||
|
token?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface PixelCaptureBridgeOptions {
|
||||||
|
ipc?: IpcCaptureBridge;
|
||||||
|
createRequestId?: () => string;
|
||||||
|
timeoutMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function adaptWebContents(
|
||||||
|
contents: BrowserAutomationWebContents,
|
||||||
|
browserId: string,
|
||||||
|
options?: PixelCaptureBridgeOptions,
|
||||||
|
): TabContents {
|
||||||
observeConsoleMessages(contents);
|
observeConsoleMessages(contents);
|
||||||
return {
|
return {
|
||||||
id: contents.id,
|
id: contents.id,
|
||||||
@@ -35,7 +107,19 @@ function adaptWebContents(contents: WebContents): TabContents {
|
|||||||
goBack: () => contents.goBack(),
|
goBack: () => contents.goBack(),
|
||||||
goForward: () => contents.goForward(),
|
goForward: () => contents.goForward(),
|
||||||
reload: () => contents.reload(),
|
reload: () => contents.reload(),
|
||||||
capturePage: (options) => contents.capturePage(undefined, options),
|
capturePage: (captureOptions) => contents.capturePage(undefined, captureOptions),
|
||||||
|
prepareForPixelCapture: async () => {
|
||||||
|
const result = await requestPixelCaptureBridge(contents, browserId, "prepare", options);
|
||||||
|
if (!result.token) {
|
||||||
|
throw new Error("Browser pixel capture preparation did not return a token.");
|
||||||
|
}
|
||||||
|
return { token: result.token };
|
||||||
|
},
|
||||||
|
restorePixelCapture: async (preparation) => {
|
||||||
|
await requestPixelCaptureBridge(contents, browserId, "restore", options, {
|
||||||
|
token: preparation.token,
|
||||||
|
});
|
||||||
|
},
|
||||||
invalidate: () => contents.invalidate(),
|
invalidate: () => contents.invalidate(),
|
||||||
isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(),
|
isBackgroundThrottlingAllowed: () => contents.getBackgroundThrottling(),
|
||||||
setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed),
|
setBackgroundThrottling: (allowed) => contents.setBackgroundThrottling(allowed),
|
||||||
@@ -49,7 +133,101 @@ function adaptWebContents(contents: WebContents): TabContents {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function observeConsoleMessages(contents: WebContents): void {
|
function requestPixelCaptureBridge(
|
||||||
|
contents: BrowserAutomationWebContents,
|
||||||
|
browserId: string,
|
||||||
|
kind: PixelCaptureBridgeKind,
|
||||||
|
options: PixelCaptureBridgeOptions | undefined,
|
||||||
|
extraPayload?: { token: string },
|
||||||
|
): Promise<PixelCaptureBridgeSuccess> {
|
||||||
|
const host = contents.hostWebContents;
|
||||||
|
if (!host || host.isDestroyed()) {
|
||||||
|
return Promise.reject(new Error("Browser host renderer is not available."));
|
||||||
|
}
|
||||||
|
|
||||||
|
const ipc = options?.ipc ?? ipcMain;
|
||||||
|
const requestId =
|
||||||
|
options?.createRequestId?.() ?? `browser-pixel-capture-${++nextPixelCaptureBridgeRequest}`;
|
||||||
|
const timeoutMs = options?.timeoutMs ?? PIXEL_CAPTURE_BRIDGE_TIMEOUT_MS;
|
||||||
|
const requestChannel =
|
||||||
|
kind === "prepare" ? "paseo:browser:capture-prepare" : "paseo:browser:capture-restore";
|
||||||
|
const responseChannel =
|
||||||
|
kind === "prepare" ? "paseo:browser:capture-prepared" : "paseo:browser:capture-restored";
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
const cleanup = () => {
|
||||||
|
if (timeoutId) {
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
}
|
||||||
|
ipc.removeListener(responseChannel, listener);
|
||||||
|
};
|
||||||
|
const listener: IpcListener = (_event, payload) => {
|
||||||
|
const response = readPixelCaptureBridgeResponse(payload, requestId);
|
||||||
|
if (!response) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
cleanup();
|
||||||
|
if (response.ok) {
|
||||||
|
resolve(response);
|
||||||
|
} else {
|
||||||
|
reject(new Error(response.message));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
ipc.on(responseChannel, listener);
|
||||||
|
timeoutId = setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
reject(new Error(`Browser pixel capture ${kind} timed out.`));
|
||||||
|
}, timeoutMs);
|
||||||
|
|
||||||
|
try {
|
||||||
|
host.send(requestChannel, {
|
||||||
|
requestId,
|
||||||
|
browserId,
|
||||||
|
...(extraPayload ? { token: extraPayload.token } : {}),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
cleanup();
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function readPixelCaptureBridgeResponse(
|
||||||
|
payload: unknown,
|
||||||
|
requestId: string,
|
||||||
|
): ({ ok: true } & PixelCaptureBridgeSuccess) | { ok: false; message: string } | null {
|
||||||
|
if (!isRecord(payload)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const record = payload;
|
||||||
|
if (record.requestId !== requestId) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (record.ok === true) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
...(typeof record.token === "string" && record.token.length > 0
|
||||||
|
? { token: record.token }
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (record.ok === false) {
|
||||||
|
const message =
|
||||||
|
typeof record.message === "string" && record.message.length > 0
|
||||||
|
? record.message
|
||||||
|
: "Browser pixel capture bridge failed.";
|
||||||
|
return { ok: false, message };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function observeConsoleMessages(contents: BrowserAutomationWebContents): void {
|
||||||
if (observedContentsIds.has(contents.id)) {
|
if (observedContentsIds.has(contents.id)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -89,7 +267,7 @@ function createRegistry(): BrowserRegistry {
|
|||||||
listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace,
|
listRegisteredBrowserIdsForWorkspace: listRegisteredPaseoBrowserIdsForWorkspace,
|
||||||
getTabContents(browserId: string): TabContents | null {
|
getTabContents(browserId: string): TabContents | null {
|
||||||
const contents = getPaseoBrowserWebContents(browserId);
|
const contents = getPaseoBrowserWebContents(browserId);
|
||||||
return contents ? adaptWebContents(contents) : null;
|
return contents ? adaptWebContents(contents, browserId) : null;
|
||||||
},
|
},
|
||||||
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
|
getBrowserWorkspaceId: getPaseoBrowserWorkspaceId,
|
||||||
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,
|
getWorkspaceActiveBrowserId: getWorkspaceActivePaseoBrowserId,
|
||||||
|
|||||||
@@ -7,7 +7,12 @@ import type {
|
|||||||
BrowserAutomationExecuteRequest,
|
BrowserAutomationExecuteRequest,
|
||||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||||
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
|
import { BrowserSnapshotEngine } from "./snapshot-engine.js";
|
||||||
import type { BrowserRegistry, TabContents, TabImage } from "./service.js";
|
import type {
|
||||||
|
BrowserRegistry,
|
||||||
|
TabContents,
|
||||||
|
TabImage,
|
||||||
|
TabPixelCapturePreparation,
|
||||||
|
} from "./service.js";
|
||||||
import { executeAutomationCommand } from "./service.js";
|
import { executeAutomationCommand } from "./service.js";
|
||||||
|
|
||||||
const BROWSER_A = "11111111-1111-4111-8111-111111111111";
|
const BROWSER_A = "11111111-1111-4111-8111-111111111111";
|
||||||
@@ -36,6 +41,9 @@ class FakeTab implements TabContents {
|
|||||||
public readonly actions: string[] = [];
|
public readonly actions: string[] = [];
|
||||||
public readonly capturedViewports: Array<{ stayHidden?: boolean }> = [];
|
public readonly capturedViewports: Array<{ stayHidden?: boolean }> = [];
|
||||||
public readonly debugCommands: Array<{ command: string; params?: Record<string, unknown> }> = [];
|
public readonly debugCommands: Array<{ command: string; params?: Record<string, unknown> }> = [];
|
||||||
|
public readonly restoredPixelCaptureTokens: string[] = [];
|
||||||
|
private readonly captureStartWaiters: Array<() => void> = [];
|
||||||
|
private readonly deferredCaptures: Array<(image: TabImage) => void> = [];
|
||||||
|
|
||||||
public destroyed = false;
|
public destroyed = false;
|
||||||
public bodyText = "";
|
public bodyText = "";
|
||||||
@@ -44,6 +52,9 @@ class FakeTab implements TabContents {
|
|||||||
public networkEntries: unknown[] = [];
|
public networkEntries: unknown[] = [];
|
||||||
public consoleMessages: BrowserAutomationConsoleLogEntry[] = [];
|
public consoleMessages: BrowserAutomationConsoleLogEntry[] = [];
|
||||||
public captureNeverPaints = false;
|
public captureNeverPaints = false;
|
||||||
|
public captureThrows = false;
|
||||||
|
public deferCaptures = false;
|
||||||
|
public prepareNeverAcks = false;
|
||||||
public layoutMetrics = {
|
public layoutMetrics = {
|
||||||
cssLayoutViewport: { clientWidth: 390, clientHeight: 844 },
|
cssLayoutViewport: { clientWidth: 390, clientHeight: 844 },
|
||||||
cssContentSize: { width: 390, height: 1200 },
|
cssContentSize: { width: 390, height: 1200 },
|
||||||
@@ -52,6 +63,7 @@ class FakeTab implements TabContents {
|
|||||||
public documentNodeId = 1;
|
public documentNodeId = 1;
|
||||||
public queriedNodeId = 2;
|
public queriedNodeId = 2;
|
||||||
public backgroundThrottlingAllowed = true;
|
public backgroundThrottlingAllowed = true;
|
||||||
|
private nextPixelCapturePreparationId = 0;
|
||||||
|
|
||||||
public constructor(
|
public constructor(
|
||||||
public readonly id: number,
|
public readonly id: number,
|
||||||
@@ -116,12 +128,35 @@ class FakeTab implements TabContents {
|
|||||||
public async capturePage(options?: { stayHidden?: boolean }): Promise<TabImage> {
|
public async capturePage(options?: { stayHidden?: boolean }): Promise<TabImage> {
|
||||||
this.capturedViewports.push(options ?? {});
|
this.capturedViewports.push(options ?? {});
|
||||||
this.actions.push("capture");
|
this.actions.push("capture");
|
||||||
|
this.resolveCaptureStartWaiters();
|
||||||
|
if (this.captureThrows) {
|
||||||
|
throw new Error("capture failed");
|
||||||
|
}
|
||||||
if (this.captureNeverPaints) {
|
if (this.captureNeverPaints) {
|
||||||
return new Promise<never>(() => {});
|
return new Promise<never>(() => {});
|
||||||
}
|
}
|
||||||
|
if (this.deferCaptures) {
|
||||||
|
return new Promise<TabImage>((resolve) => {
|
||||||
|
this.deferredCaptures.push(resolve);
|
||||||
|
});
|
||||||
|
}
|
||||||
return new FakeImage();
|
return new FakeImage();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async prepareForPixelCapture(): Promise<TabPixelCapturePreparation> {
|
||||||
|
this.actions.push("prepare");
|
||||||
|
if (this.prepareNeverAcks) {
|
||||||
|
return new Promise<never>(() => {});
|
||||||
|
}
|
||||||
|
const token = `capture-${++this.nextPixelCapturePreparationId}`;
|
||||||
|
return { token };
|
||||||
|
}
|
||||||
|
|
||||||
|
public async restorePixelCapture(preparation: TabPixelCapturePreparation): Promise<void> {
|
||||||
|
this.actions.push(`restore:${preparation.token}`);
|
||||||
|
this.restoredPixelCaptureTokens.push(preparation.token);
|
||||||
|
}
|
||||||
|
|
||||||
public invalidate(): void {
|
public invalidate(): void {
|
||||||
this.actions.push("invalidate");
|
this.actions.push("invalidate");
|
||||||
}
|
}
|
||||||
@@ -143,6 +178,7 @@ class FakeTab implements TabContents {
|
|||||||
command: string,
|
command: string,
|
||||||
params?: Record<string, unknown>,
|
params?: Record<string, unknown>,
|
||||||
): Promise<unknown> {
|
): Promise<unknown> {
|
||||||
|
this.actions.push(`debug:${command}`);
|
||||||
this.debugCommands.push({ command, ...(params ? { params } : {}) });
|
this.debugCommands.push({ command, ...(params ? { params } : {}) });
|
||||||
if (command === "Page.getLayoutMetrics") {
|
if (command === "Page.getLayoutMetrics") {
|
||||||
return this.layoutMetrics;
|
return this.layoutMetrics;
|
||||||
@@ -158,6 +194,33 @@ class FakeTab implements TabContents {
|
|||||||
}
|
}
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public waitForCaptureStart(count: number): Promise<void> {
|
||||||
|
if (this.capturedViewports.length >= count) {
|
||||||
|
return Promise.resolve();
|
||||||
|
}
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
this.captureStartWaiters.push(() => {
|
||||||
|
if (this.capturedViewports.length >= count) {
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public finishNextCapture(): void {
|
||||||
|
const resolve = this.deferredCaptures.shift();
|
||||||
|
if (!resolve) {
|
||||||
|
throw new Error("No deferred capture is waiting");
|
||||||
|
}
|
||||||
|
resolve(new FakeImage());
|
||||||
|
}
|
||||||
|
|
||||||
|
private resolveCaptureStartWaiters(): void {
|
||||||
|
for (const waiter of this.captureStartWaiters.splice(0)) {
|
||||||
|
waiter();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class FakeRegistry implements BrowserRegistry {
|
class FakeRegistry implements BrowserRegistry {
|
||||||
@@ -852,9 +915,11 @@ describe("executeAutomationCommand", () => {
|
|||||||
});
|
});
|
||||||
expect(browser.tab.capturedViewports).toEqual([{ stayHidden: false }]);
|
expect(browser.tab.capturedViewports).toEqual([{ stayHidden: false }]);
|
||||||
expect(browser.tab.actions).toEqual([
|
expect(browser.tab.actions).toEqual([
|
||||||
|
"prepare",
|
||||||
"background:false",
|
"background:false",
|
||||||
"invalidate",
|
"invalidate",
|
||||||
"capture",
|
"capture",
|
||||||
|
"restore:capture-1",
|
||||||
"background:true",
|
"background:true",
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
@@ -882,9 +947,11 @@ describe("executeAutomationCommand", () => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
expect(browser.tab.actions).toEqual([
|
expect(browser.tab.actions).toEqual([
|
||||||
|
"prepare",
|
||||||
"background:false",
|
"background:false",
|
||||||
"invalidate",
|
"invalidate",
|
||||||
"capture",
|
"capture",
|
||||||
|
"restore:capture-1",
|
||||||
"background:true",
|
"background:true",
|
||||||
]);
|
]);
|
||||||
} finally {
|
} finally {
|
||||||
@@ -892,6 +959,111 @@ describe("executeAutomationCommand", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("screenshot restores capture preparation when viewport capture fails", async () => {
|
||||||
|
const browser = new BrowserAutomationHarness();
|
||||||
|
browser.tab.captureThrows = true;
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
browser.execute({
|
||||||
|
command: "screenshot",
|
||||||
|
args: { browserId: BROWSER_A },
|
||||||
|
}),
|
||||||
|
).rejects.toThrow("capture failed");
|
||||||
|
|
||||||
|
expect(browser.tab.actions).toEqual([
|
||||||
|
"prepare",
|
||||||
|
"background:false",
|
||||||
|
"invalidate",
|
||||||
|
"capture",
|
||||||
|
"restore:capture-1",
|
||||||
|
"background:true",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("screenshot returns no-frame when capture preparation does not ack", async () => {
|
||||||
|
vi.useFakeTimers();
|
||||||
|
try {
|
||||||
|
const browser = new BrowserAutomationHarness();
|
||||||
|
browser.tab.prepareNeverAcks = true;
|
||||||
|
|
||||||
|
const resultPromise = browser.execute({
|
||||||
|
command: "screenshot",
|
||||||
|
args: { browserId: BROWSER_A },
|
||||||
|
});
|
||||||
|
await vi.advanceTimersByTimeAsync(5_000);
|
||||||
|
|
||||||
|
await expect(resultPromise).resolves.toEqual({
|
||||||
|
requestId: "req-screenshot",
|
||||||
|
ok: false,
|
||||||
|
error: {
|
||||||
|
code: "screenshot_no_frame",
|
||||||
|
message:
|
||||||
|
"The browser tab has no painted frame. Focus the tab in the app, then try again.",
|
||||||
|
retryable: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(browser.tab.actions).toEqual(["prepare", "background:true"]);
|
||||||
|
} finally {
|
||||||
|
vi.useRealTimers();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("overlapping screenshots serialize capture preparation and restore", async () => {
|
||||||
|
const browser = new BrowserAutomationHarness();
|
||||||
|
browser.tab.deferCaptures = true;
|
||||||
|
|
||||||
|
const first = browser.execute({
|
||||||
|
command: "screenshot",
|
||||||
|
args: { browserId: BROWSER_A },
|
||||||
|
});
|
||||||
|
await browser.tab.waitForCaptureStart(1);
|
||||||
|
|
||||||
|
const second = browser.execute(
|
||||||
|
{
|
||||||
|
command: "screenshot",
|
||||||
|
args: { browserId: BROWSER_A },
|
||||||
|
},
|
||||||
|
{ requestId: "req-screenshot-2" },
|
||||||
|
);
|
||||||
|
await Promise.resolve();
|
||||||
|
|
||||||
|
expect(browser.tab.actions).toEqual(["prepare", "background:false", "invalidate", "capture"]);
|
||||||
|
|
||||||
|
browser.tab.finishNextCapture();
|
||||||
|
await browser.tab.waitForCaptureStart(2);
|
||||||
|
|
||||||
|
expect(browser.tab.actions).toEqual([
|
||||||
|
"prepare",
|
||||||
|
"background:false",
|
||||||
|
"invalidate",
|
||||||
|
"capture",
|
||||||
|
"restore:capture-1",
|
||||||
|
"background:true",
|
||||||
|
"prepare",
|
||||||
|
"background:false",
|
||||||
|
"invalidate",
|
||||||
|
"capture",
|
||||||
|
]);
|
||||||
|
|
||||||
|
browser.tab.finishNextCapture();
|
||||||
|
await expect(first).resolves.toMatchObject({ requestId: "req-screenshot", ok: true });
|
||||||
|
await expect(second).resolves.toMatchObject({ requestId: "req-screenshot-2", ok: true });
|
||||||
|
expect(browser.tab.actions).toEqual([
|
||||||
|
"prepare",
|
||||||
|
"background:false",
|
||||||
|
"invalidate",
|
||||||
|
"capture",
|
||||||
|
"restore:capture-1",
|
||||||
|
"background:true",
|
||||||
|
"prepare",
|
||||||
|
"background:false",
|
||||||
|
"invalidate",
|
||||||
|
"capture",
|
||||||
|
"restore:capture-2",
|
||||||
|
"background:true",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
test("screenshot with fullPage captures the page content area through CDP", async () => {
|
test("screenshot with fullPage captures the page content area through CDP", async () => {
|
||||||
const browser = new BrowserAutomationHarness();
|
const browser = new BrowserAutomationHarness();
|
||||||
|
|
||||||
@@ -923,6 +1095,44 @@ describe("executeAutomationCommand", () => {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
|
expect(browser.tab.actions).toEqual([
|
||||||
|
"prepare",
|
||||||
|
"background:false",
|
||||||
|
"invalidate",
|
||||||
|
"debug:Page.getLayoutMetrics",
|
||||||
|
"debug:Page.captureScreenshot",
|
||||||
|
"restore:capture-1",
|
||||||
|
"background:true",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("screenshot with fullPage restores capture preparation when CDP returns no image", async () => {
|
||||||
|
const browser = new BrowserAutomationHarness();
|
||||||
|
browser.tab.fullPageScreenshotData = "";
|
||||||
|
|
||||||
|
const result = await browser.execute({
|
||||||
|
command: "screenshot",
|
||||||
|
args: { browserId: BROWSER_A, fullPage: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toEqual({
|
||||||
|
requestId: "req-screenshot",
|
||||||
|
ok: false,
|
||||||
|
error: {
|
||||||
|
code: "browser_unsupported",
|
||||||
|
message: "browser_screenshot fullPage returned no data",
|
||||||
|
retryable: false,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(browser.tab.actions).toEqual([
|
||||||
|
"prepare",
|
||||||
|
"background:false",
|
||||||
|
"invalidate",
|
||||||
|
"debug:Page.getLayoutMetrics",
|
||||||
|
"debug:Page.captureScreenshot",
|
||||||
|
"restore:capture-1",
|
||||||
|
"background:true",
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("upload resolves workspace files before setting them on the file input", async () => {
|
test("upload resolves workspace files before setting them on the file input", async () => {
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ export interface TabContents {
|
|||||||
goForward(): void;
|
goForward(): void;
|
||||||
reload(): void;
|
reload(): void;
|
||||||
capturePage(options?: TabCapturePageOptions): Promise<TabImage>;
|
capturePage(options?: TabCapturePageOptions): Promise<TabImage>;
|
||||||
|
prepareForPixelCapture(): Promise<TabPixelCapturePreparation>;
|
||||||
|
restorePixelCapture(preparation: TabPixelCapturePreparation): Promise<void>;
|
||||||
invalidate(): void;
|
invalidate(): void;
|
||||||
isBackgroundThrottlingAllowed(): boolean;
|
isBackgroundThrottlingAllowed(): boolean;
|
||||||
setBackgroundThrottling(allowed: boolean): void;
|
setBackgroundThrottling(allowed: boolean): void;
|
||||||
@@ -31,6 +33,10 @@ export interface TabContents {
|
|||||||
sendDebugCommand?(command: string, params?: Record<string, unknown>): Promise<unknown>;
|
sendDebugCommand?(command: string, params?: Record<string, unknown>): Promise<unknown>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface TabPixelCapturePreparation {
|
||||||
|
token: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface TabImage {
|
export interface TabImage {
|
||||||
toPNG(): Uint8Array;
|
toPNG(): Uint8Array;
|
||||||
getSize(): { width: number; height: number };
|
getSize(): { width: number; height: number };
|
||||||
@@ -124,19 +130,56 @@ async function runSerializedPixelCapture<T>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function capturePaintedViewport(contents: TabContents): Promise<TabImage> {
|
async function prepareForPixelCapture(contents: TabContents): Promise<TabPixelCapturePreparation> {
|
||||||
|
try {
|
||||||
|
return await withPixelCaptureTimeout(contents.prepareForPixelCapture());
|
||||||
|
} catch {
|
||||||
|
throw new ScreenshotNoFrameError();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function restorePixelCapture(
|
||||||
|
contents: TabContents,
|
||||||
|
preparation: TabPixelCapturePreparation,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await withPixelCaptureTimeout(contents.restorePixelCapture(preparation));
|
||||||
|
} catch {
|
||||||
|
throw new ScreenshotNoFrameError();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runPreparedPixelCapture<T>(
|
||||||
|
contents: TabContents,
|
||||||
|
capture: () => Promise<T>,
|
||||||
|
): Promise<T> {
|
||||||
return runSerializedPixelCapture(contents, async () => {
|
return runSerializedPixelCapture(contents, async () => {
|
||||||
const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed();
|
const previousBackgroundThrottling = contents.isBackgroundThrottlingAllowed();
|
||||||
contents.setBackgroundThrottling(false);
|
let preparation: TabPixelCapturePreparation | null = null;
|
||||||
try {
|
try {
|
||||||
|
// Offscreen-parked webview guests have no compositor surface for
|
||||||
|
// capturePage/CDP to copy. The renderer must briefly make the host
|
||||||
|
// paintable before capture; see ~/.paseo/plans/browser-capture-harness-results.md.
|
||||||
|
preparation = await prepareForPixelCapture(contents);
|
||||||
|
contents.setBackgroundThrottling(false);
|
||||||
contents.invalidate();
|
contents.invalidate();
|
||||||
return await withPixelCaptureTimeout(contents.capturePage({ stayHidden: false }));
|
return await withPixelCaptureTimeout(capture());
|
||||||
} finally {
|
} finally {
|
||||||
contents.setBackgroundThrottling(previousBackgroundThrottling);
|
try {
|
||||||
|
if (preparation) {
|
||||||
|
await restorePixelCapture(contents, preparation);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
contents.setBackgroundThrottling(previousBackgroundThrottling);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function capturePaintedViewport(contents: TabContents): Promise<TabImage> {
|
||||||
|
return runPreparedPixelCapture(contents, () => contents.capturePage({ stayHidden: false }));
|
||||||
|
}
|
||||||
|
|
||||||
function tabInfoFromContents(
|
function tabInfoFromContents(
|
||||||
browserId: string,
|
browserId: string,
|
||||||
contents: TabContents,
|
contents: TabContents,
|
||||||
@@ -860,18 +903,21 @@ async function executeFullPageScreenshot(
|
|||||||
if (!target.contents.sendDebugCommand) {
|
if (!target.contents.sendDebugCommand) {
|
||||||
return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP");
|
return fail(requestId, "browser_unsupported", "browser_screenshot fullPage requires CDP");
|
||||||
}
|
}
|
||||||
const metrics = await getCdpLayoutMetrics(target.contents);
|
const sendDebugCommand = target.contents.sendDebugCommand.bind(target.contents);
|
||||||
const width = metrics.contentWidth;
|
|
||||||
const height = metrics.contentHeight;
|
|
||||||
let screenshot: CdpCaptureScreenshotResult;
|
let screenshot: CdpCaptureScreenshotResult;
|
||||||
|
let width = 0;
|
||||||
|
let height = 0;
|
||||||
try {
|
try {
|
||||||
screenshot = (await withPixelCaptureTimeout(
|
screenshot = await runPreparedPixelCapture(target.contents, async () => {
|
||||||
target.contents.sendDebugCommand("Page.captureScreenshot", {
|
const metrics = await getCdpLayoutMetrics(target.contents);
|
||||||
|
width = metrics.contentWidth;
|
||||||
|
height = metrics.contentHeight;
|
||||||
|
return (await sendDebugCommand("Page.captureScreenshot", {
|
||||||
format: "png",
|
format: "png",
|
||||||
captureBeyondViewport: true,
|
captureBeyondViewport: true,
|
||||||
clip: { x: 0, y: 0, width, height, scale: 1 },
|
clip: { x: 0, y: 0, width, height, scale: 1 },
|
||||||
}),
|
})) as CdpCaptureScreenshotResult;
|
||||||
)) as CdpCaptureScreenshotResult;
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (isScreenshotNoFrameError(error)) {
|
if (isScreenshotNoFrameError(error)) {
|
||||||
return screenshotNoFrameFailure(requestId);
|
return screenshotNoFrameFailure(requestId);
|
||||||
|
|||||||
@@ -1,6 +1,81 @@
|
|||||||
import { contextBridge, ipcRenderer, webUtils } from "electron";
|
import { contextBridge, ipcRenderer, webUtils } from "electron";
|
||||||
|
|
||||||
type EventHandler = (payload: unknown) => void;
|
type EventHandler = (payload: unknown) => void;
|
||||||
|
type BrowserPixelCapturePrepareHandler = (input: {
|
||||||
|
browserId: string;
|
||||||
|
}) => Promise<{ token: string }>;
|
||||||
|
type BrowserPixelCaptureRestoreHandler = (input: { token: string }) => Promise<void>;
|
||||||
|
|
||||||
|
let prepareForPixelCaptureHandler: BrowserPixelCapturePrepareHandler | null = null;
|
||||||
|
let restorePixelCaptureHandler: BrowserPixelCaptureRestoreHandler | null = null;
|
||||||
|
|
||||||
|
function readStringField(payload: unknown, key: string): string | null {
|
||||||
|
if (!isRecord(payload)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const value = payload[key];
|
||||||
|
return typeof value === "string" && value.length > 0 ? value : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorMessage(error: unknown): string {
|
||||||
|
return error instanceof Error ? error.message : String(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
ipcRenderer.on("paseo:browser:capture-prepare", async (_event, payload: unknown) => {
|
||||||
|
const requestId = readStringField(payload, "requestId");
|
||||||
|
const browserId = readStringField(payload, "browserId");
|
||||||
|
if (!requestId || !browserId || !prepareForPixelCaptureHandler) {
|
||||||
|
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||||
|
requestId: requestId ?? "unknown",
|
||||||
|
ok: false,
|
||||||
|
message: "Browser pixel capture preparation is unavailable.",
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const preparation = await prepareForPixelCaptureHandler({ browserId });
|
||||||
|
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||||
|
requestId,
|
||||||
|
ok: true,
|
||||||
|
token: preparation.token,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
ipcRenderer.send("paseo:browser:capture-prepared", {
|
||||||
|
requestId,
|
||||||
|
ok: false,
|
||||||
|
message: errorMessage(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
ipcRenderer.on("paseo:browser:capture-restore", async (_event, payload: unknown) => {
|
||||||
|
const requestId = readStringField(payload, "requestId");
|
||||||
|
const token = readStringField(payload, "token");
|
||||||
|
if (!requestId || !token || !restorePixelCaptureHandler) {
|
||||||
|
ipcRenderer.send("paseo:browser:capture-restored", {
|
||||||
|
requestId: requestId ?? "unknown",
|
||||||
|
ok: false,
|
||||||
|
message: "Browser pixel capture restore is unavailable.",
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await restorePixelCaptureHandler({ token });
|
||||||
|
ipcRenderer.send("paseo:browser:capture-restored", { requestId, ok: true });
|
||||||
|
} catch (error) {
|
||||||
|
ipcRenderer.send("paseo:browser:capture-restored", {
|
||||||
|
requestId,
|
||||||
|
ok: false,
|
||||||
|
message: errorMessage(error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
contextBridge.exposeInMainWorld("paseoDesktop", {
|
contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||||
platform: process.platform,
|
platform: process.platform,
|
||||||
@@ -90,5 +165,21 @@ contextBridge.exposeInMainWorld("paseoDesktop", {
|
|||||||
) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect),
|
) => ipcRenderer.invoke("paseo:browser:capture-element", browserId, rect),
|
||||||
copyElement: (payload: { text?: string; imageDataUrl?: string }) =>
|
copyElement: (payload: { text?: string; imageDataUrl?: string }) =>
|
||||||
ipcRenderer.invoke("paseo:browser:copy-element", payload),
|
ipcRenderer.invoke("paseo:browser:copy-element", payload),
|
||||||
|
onPrepareForPixelCapture: (handler: BrowserPixelCapturePrepareHandler): (() => void) => {
|
||||||
|
prepareForPixelCaptureHandler = handler;
|
||||||
|
return () => {
|
||||||
|
if (prepareForPixelCaptureHandler === handler) {
|
||||||
|
prepareForPixelCaptureHandler = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
},
|
||||||
|
onRestorePixelCapture: (handler: BrowserPixelCaptureRestoreHandler): (() => void) => {
|
||||||
|
restorePixelCaptureHandler = handler;
|
||||||
|
return () => {
|
||||||
|
if (restorePixelCaptureHandler === handler) {
|
||||||
|
restorePixelCaptureHandler = null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user