mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-03 23:23:21 +00:00
* feat(browser): inspect, annotate, and grab page elements for the agent Build a design-review flow on top of the in-app Electron browser so users can send page elements to their coding agent with context. - Annotate: pick an element, write a comment, choose an intent (fix/change/question/approve); the element context + intent + comment go to the agent as text, and a cropped screenshot rides along as an image. - Grab: pick an element to copy its info + screenshot straight to the system clipboard (no comment), with a toast on success/failure. - Hover inspector: in select mode each element shows a floating label with tag, id/class, React component name, and pixel size. - Page markers: annotated elements on the current page get numbered badges that track scroll/resize. - Device sizes: a viewport-size menu (responsive + 13 common device presets) renders the page centered in a fixed-size frame. - Toolbar buttons now expose hover tooltips and no longer get squeezed out on narrow panes; the element selector is available to all desktop users (previously dev-only). Screenshots and clipboard writes go through new Electron main-process IPCs (capturePage + clipboard) so they work regardless of webview focus. Element screenshots are referenced by id in the workspace attachment store and are protected from the draft-store attachment GC. All new strings are translated across the six supported locales. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(browser): write element clipboard exactly once Resolve the image before writing so a combined text+image grab no longer does a redundant text-only writeText() first (which flashed an intermediate clipboard state). Addresses greptile review feedback. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * i18n: add browser annotate/grab/devices strings for ja and pt-BR main added Japanese and Brazilian Portuguese locales after this branch was created; add the browser annotation, grab, and device-size keys to keep all locales in sync with en. resources.test.ts parity passes. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
200 lines
5.8 KiB
TypeScript
200 lines
5.8 KiB
TypeScript
import { Platform } from "react-native";
|
|
import { getElectronHost } from "@/desktop/electron/host";
|
|
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
|
|
|
type BrowserAutomationExecuteRequest = Extract<
|
|
SessionOutboundMessage,
|
|
{ type: "browser.automation.execute.request" }
|
|
>;
|
|
type BrowserAutomationExecuteResponse = Extract<
|
|
SessionInboundMessage,
|
|
{ type: "browser.automation.execute.response" }
|
|
>;
|
|
|
|
export type DesktopNotificationPermission = "granted" | "denied" | "default";
|
|
|
|
export interface DesktopDialogAskOptions {
|
|
title?: string;
|
|
okLabel?: string;
|
|
cancelLabel?: string;
|
|
kind?: "info" | "warning" | "error";
|
|
}
|
|
|
|
export interface DesktopDialogOpenOptions {
|
|
title?: string;
|
|
defaultPath?: string;
|
|
directory?: boolean;
|
|
multiple?: boolean;
|
|
filters?: Array<{
|
|
name: string;
|
|
extensions: string[];
|
|
}>;
|
|
}
|
|
|
|
export interface DesktopDialogAskWithCheckboxOptions extends DesktopDialogAskOptions {
|
|
checkboxLabel: string;
|
|
checkboxChecked?: boolean;
|
|
}
|
|
|
|
export interface DesktopDialogAskWithCheckboxResult {
|
|
confirmed: boolean;
|
|
dontAskAgain: boolean;
|
|
}
|
|
|
|
export interface DesktopDialogBridge {
|
|
ask?: (message: string, options?: DesktopDialogAskOptions) => Promise<boolean>;
|
|
askWithCheckbox?: (
|
|
message: string,
|
|
options: DesktopDialogAskWithCheckboxOptions,
|
|
) => Promise<DesktopDialogAskWithCheckboxResult>;
|
|
open?: (options?: DesktopDialogOpenOptions) => Promise<string | string[] | null>;
|
|
}
|
|
|
|
export interface DesktopNotificationBridge {
|
|
isSupported?: () => Promise<boolean>;
|
|
sendNotification?: (
|
|
payload: string | { title: string; body?: string; data?: Record<string, unknown> },
|
|
) => Promise<boolean>;
|
|
}
|
|
|
|
export interface DesktopOpenerBridge {
|
|
openUrl?: (url: string) => Promise<void>;
|
|
}
|
|
|
|
export interface DesktopEditorTargetDescriptor {
|
|
id: string;
|
|
label: string;
|
|
kind: "editor" | "file-manager";
|
|
}
|
|
|
|
export interface DesktopEditorOpenTargetInput {
|
|
editorId: string;
|
|
path: string;
|
|
cwd?: string;
|
|
mode?: "open" | "reveal";
|
|
}
|
|
|
|
export interface DesktopEditorBridge {
|
|
listTargets?: () => Promise<DesktopEditorTargetDescriptor[]>;
|
|
openTarget?: (input: DesktopEditorOpenTargetInput) => Promise<void>;
|
|
}
|
|
|
|
export interface DesktopWebUtilsBridge {
|
|
getPathForFile?: (file: File) => string;
|
|
}
|
|
|
|
export interface DesktopMenuBridge {
|
|
showContextMenu?: (input?: { kind?: "terminal"; hasSelection?: boolean }) => Promise<void>;
|
|
}
|
|
|
|
export interface DesktopWindowControlsOverlayUpdate {
|
|
height?: number;
|
|
backgroundColor?: string;
|
|
foregroundColor?: string;
|
|
}
|
|
|
|
export interface DesktopWindowBridge {
|
|
label?: string;
|
|
toggleMaximize?: () => Promise<void>;
|
|
isFullscreen?: () => Promise<boolean>;
|
|
updateWindowControls?: (update: DesktopWindowControlsOverlayUpdate) => Promise<void>;
|
|
onResized?: <TEvent = unknown>(
|
|
handler: (event: TEvent) => void,
|
|
) => Promise<() => void> | (() => void);
|
|
setBadgeCount?: (count?: number) => Promise<void>;
|
|
onDragDropEvent?: <TEvent = unknown>(
|
|
handler: (event: TEvent) => void,
|
|
) => Promise<() => void> | (() => void);
|
|
}
|
|
|
|
export interface DesktopWindowModuleBridge {
|
|
openNew?: (options?: { pendingOpenProjectPath?: string | null }) => Promise<void>;
|
|
getCurrentWindow?: () => DesktopWindowBridge;
|
|
}
|
|
|
|
export interface DesktopEventsBridge {
|
|
on?: (event: string, handler: (payload: unknown) => void) => Promise<() => void> | (() => void);
|
|
}
|
|
|
|
export interface DesktopBrowserShortcutEvent {
|
|
browserId?: string;
|
|
action: "focus-url";
|
|
}
|
|
|
|
export interface DesktopBrowserNewTabRequestEvent {
|
|
sourceBrowserId: string;
|
|
url: string;
|
|
}
|
|
|
|
export interface DesktopBrowserBridge {
|
|
registerWorkspaceBrowser?: (input: { browserId: string; workspaceId: string }) => Promise<void>;
|
|
setWorkspaceActiveBrowser?: (input: {
|
|
workspaceId: string;
|
|
browserId: string | null;
|
|
}) => Promise<void>;
|
|
setAgentActiveBrowser?: (input: { agentId: string; browserId: string | null }) => Promise<void>;
|
|
openDevTools?: (browserId: string) => Promise<unknown>;
|
|
clearPartition?: (browserId: string) => Promise<void>;
|
|
executeAutomationCommand?: (
|
|
request: BrowserAutomationExecuteRequest,
|
|
) => Promise<BrowserAutomationExecuteResponse["payload"]>;
|
|
/** Capture a PNG screenshot of the guest viewport cropped to `rect`. */
|
|
captureElement?: (
|
|
browserId: string,
|
|
rect: { x: number; y: number; width: number; height: number },
|
|
) => Promise<string | null>;
|
|
/** Copy element text and/or an image to the system clipboard from main. */
|
|
copyElement?: (payload: { text?: string; imageDataUrl?: string }) => Promise<boolean>;
|
|
}
|
|
|
|
export interface DesktopInvokeBridge {
|
|
invoke?: (command: string, args?: Record<string, unknown>) => Promise<unknown>;
|
|
}
|
|
|
|
export interface DesktopHostBridge {
|
|
platform?: string;
|
|
invoke?: DesktopInvokeBridge["invoke"];
|
|
getPendingOpenProject?: () => Promise<string | null>;
|
|
events?: DesktopEventsBridge;
|
|
window?: DesktopWindowModuleBridge;
|
|
dialog?: DesktopDialogBridge;
|
|
notification?: DesktopNotificationBridge;
|
|
opener?: DesktopOpenerBridge;
|
|
editor?: DesktopEditorBridge;
|
|
webUtils?: DesktopWebUtilsBridge;
|
|
menu?: DesktopMenuBridge;
|
|
browser?: DesktopBrowserBridge;
|
|
}
|
|
|
|
declare global {
|
|
interface Window {
|
|
paseoDesktop?: DesktopHostBridge;
|
|
}
|
|
}
|
|
|
|
export function getDesktopHost(): DesktopHostBridge | null {
|
|
if (Platform.OS !== "web") {
|
|
return null;
|
|
}
|
|
return getElectronHost();
|
|
}
|
|
|
|
export function isElectronRuntime(): boolean {
|
|
return getDesktopHost() !== null;
|
|
}
|
|
|
|
export function isElectronRuntimeMac(): boolean {
|
|
if (!isElectronRuntime()) {
|
|
return false;
|
|
}
|
|
if (typeof navigator === "undefined") {
|
|
return false;
|
|
}
|
|
const hostPlatform = getDesktopHost()?.platform?.toLowerCase();
|
|
if (hostPlatform === "darwin" || hostPlatform === "mac" || hostPlatform === "macos") {
|
|
return true;
|
|
}
|
|
const ua = navigator.userAgent;
|
|
return ua.includes("Mac OS") || ua.includes("Macintosh");
|
|
}
|