Files
paseo/packages/app/src/desktop/host.ts
T
huiliaoningandClaude Opus 4.8 b8b66816ca feat(browser): inspect, annotate, and grab page elements for the agent (#1708)
* feat(browser): inspect, annotate, and grab page elements for the agent

Build a design-review flow on top of the in-app Electron browser so users
can send page elements to their coding agent with context.

- Annotate: pick an element, write a comment, choose an intent
  (fix/change/question/approve); the element context + intent + comment go
  to the agent as text, and a cropped screenshot rides along as an image.
- Grab: pick an element to copy its info + screenshot straight to the
  system clipboard (no comment), with a toast on success/failure.
- Hover inspector: in select mode each element shows a floating label with
  tag, id/class, React component name, and pixel size.
- Page markers: annotated elements on the current page get numbered badges
  that track scroll/resize.
- Device sizes: a viewport-size menu (responsive + 13 common device presets)
  renders the page centered in a fixed-size frame.
- Toolbar buttons now expose hover tooltips and no longer get squeezed out
  on narrow panes; the element selector is available to all desktop users
  (previously dev-only).

Screenshots and clipboard writes go through new Electron main-process IPCs
(capturePage + clipboard) so they work regardless of webview focus. Element
screenshots are referenced by id in the workspace attachment store and are
protected from the draft-store attachment GC. All new strings are
translated across the six supported locales.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* fix(browser): write element clipboard exactly once

Resolve the image before writing so a combined text+image grab no longer
does a redundant text-only writeText() first (which flashed an intermediate
clipboard state). Addresses greptile review feedback.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* i18n: add browser annotate/grab/devices strings for ja and pt-BR

main added Japanese and Brazilian Portuguese locales after this branch was
created; add the browser annotation, grab, and device-size keys to keep all
locales in sync with en. resources.test.ts parity passes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-30 22:58:16 +02:00

200 lines
5.8 KiB
TypeScript

import { Platform } from "react-native";
import { getElectronHost } from "@/desktop/electron/host";
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
type BrowserAutomationExecuteRequest = Extract<
SessionOutboundMessage,
{ type: "browser.automation.execute.request" }
>;
type BrowserAutomationExecuteResponse = Extract<
SessionInboundMessage,
{ type: "browser.automation.execute.response" }
>;
export type DesktopNotificationPermission = "granted" | "denied" | "default";
export interface DesktopDialogAskOptions {
title?: string;
okLabel?: string;
cancelLabel?: string;
kind?: "info" | "warning" | "error";
}
export interface DesktopDialogOpenOptions {
title?: string;
defaultPath?: string;
directory?: boolean;
multiple?: boolean;
filters?: Array<{
name: string;
extensions: string[];
}>;
}
export interface DesktopDialogAskWithCheckboxOptions extends DesktopDialogAskOptions {
checkboxLabel: string;
checkboxChecked?: boolean;
}
export interface DesktopDialogAskWithCheckboxResult {
confirmed: boolean;
dontAskAgain: boolean;
}
export interface DesktopDialogBridge {
ask?: (message: string, options?: DesktopDialogAskOptions) => Promise<boolean>;
askWithCheckbox?: (
message: string,
options: DesktopDialogAskWithCheckboxOptions,
) => Promise<DesktopDialogAskWithCheckboxResult>;
open?: (options?: DesktopDialogOpenOptions) => Promise<string | string[] | null>;
}
export interface DesktopNotificationBridge {
isSupported?: () => Promise<boolean>;
sendNotification?: (
payload: string | { title: string; body?: string; data?: Record<string, unknown> },
) => Promise<boolean>;
}
export interface DesktopOpenerBridge {
openUrl?: (url: string) => Promise<void>;
}
export interface DesktopEditorTargetDescriptor {
id: string;
label: string;
kind: "editor" | "file-manager";
}
export interface DesktopEditorOpenTargetInput {
editorId: string;
path: string;
cwd?: string;
mode?: "open" | "reveal";
}
export interface DesktopEditorBridge {
listTargets?: () => Promise<DesktopEditorTargetDescriptor[]>;
openTarget?: (input: DesktopEditorOpenTargetInput) => Promise<void>;
}
export interface DesktopWebUtilsBridge {
getPathForFile?: (file: File) => string;
}
export interface DesktopMenuBridge {
showContextMenu?: (input?: { kind?: "terminal"; hasSelection?: boolean }) => Promise<void>;
}
export interface DesktopWindowControlsOverlayUpdate {
height?: number;
backgroundColor?: string;
foregroundColor?: string;
}
export interface DesktopWindowBridge {
label?: string;
toggleMaximize?: () => Promise<void>;
isFullscreen?: () => Promise<boolean>;
updateWindowControls?: (update: DesktopWindowControlsOverlayUpdate) => Promise<void>;
onResized?: <TEvent = unknown>(
handler: (event: TEvent) => void,
) => Promise<() => void> | (() => void);
setBadgeCount?: (count?: number) => Promise<void>;
onDragDropEvent?: <TEvent = unknown>(
handler: (event: TEvent) => void,
) => Promise<() => void> | (() => void);
}
export interface DesktopWindowModuleBridge {
openNew?: (options?: { pendingOpenProjectPath?: string | null }) => Promise<void>;
getCurrentWindow?: () => DesktopWindowBridge;
}
export interface DesktopEventsBridge {
on?: (event: string, handler: (payload: unknown) => void) => Promise<() => void> | (() => void);
}
export interface DesktopBrowserShortcutEvent {
browserId?: string;
action: "focus-url";
}
export interface DesktopBrowserNewTabRequestEvent {
sourceBrowserId: string;
url: string;
}
export interface DesktopBrowserBridge {
registerWorkspaceBrowser?: (input: { browserId: string; workspaceId: string }) => Promise<void>;
setWorkspaceActiveBrowser?: (input: {
workspaceId: string;
browserId: string | null;
}) => Promise<void>;
setAgentActiveBrowser?: (input: { agentId: string; browserId: string | null }) => Promise<void>;
openDevTools?: (browserId: string) => Promise<unknown>;
clearPartition?: (browserId: string) => Promise<void>;
executeAutomationCommand?: (
request: BrowserAutomationExecuteRequest,
) => Promise<BrowserAutomationExecuteResponse["payload"]>;
/** Capture a PNG screenshot of the guest viewport cropped to `rect`. */
captureElement?: (
browserId: string,
rect: { x: number; y: number; width: number; height: number },
) => Promise<string | null>;
/** Copy element text and/or an image to the system clipboard from main. */
copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
}
export interface DesktopInvokeBridge {
invoke?: (command: string, args?: Record<string, unknown>) => Promise<unknown>;
}
export interface DesktopHostBridge {
platform?: string;
invoke?: DesktopInvokeBridge["invoke"];
getPendingOpenProject?: () => Promise<string | null>;
events?: DesktopEventsBridge;
window?: DesktopWindowModuleBridge;
dialog?: DesktopDialogBridge;
notification?: DesktopNotificationBridge;
opener?: DesktopOpenerBridge;
editor?: DesktopEditorBridge;
webUtils?: DesktopWebUtilsBridge;
menu?: DesktopMenuBridge;
browser?: DesktopBrowserBridge;
}
declare global {
interface Window {
paseoDesktop?: DesktopHostBridge;
}
}
export function getDesktopHost(): DesktopHostBridge | null {
if (Platform.OS !== "web") {
return null;
}
return getElectronHost();
}
export function isElectronRuntime(): boolean {
return getDesktopHost() !== null;
}
export function isElectronRuntimeMac(): boolean {
if (!isElectronRuntime()) {
return false;
}
if (typeof navigator === "undefined") {
return false;
}
const hostPlatform = getDesktopHost()?.platform?.toLowerCase();
if (hostPlatform === "darwin" || hostPlatform === "mac" || hostPlatform === "macos") {
return true;
}
const ua = navigator.userAgent;
return ua.includes("Mac OS") || ua.includes("Macintosh");
}