diff --git a/packages/app/src/command-center/root-registration.tsx b/packages/app/src/command-center/root-registration.tsx index 9978aed6e1..adf3190784 100644 --- a/packages/app/src/command-center/root-registration.tsx +++ b/packages/app/src/command-center/root-registration.tsx @@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next"; import { CalendarClock, CircleDashed, + Copy, Folder, FolderPlus, History, @@ -17,6 +18,8 @@ import { } from "lucide-react-native"; import { withUnistyles } from "react-native-unistyles"; import { useIsCompactFormFactor } from "@/constants/layout"; +import { useToast } from "@/contexts/toast-context"; +import { getDesktopHost } from "@/desktop/host"; import { useKeyboardShortcutOverrides } from "@/hooks/use-keyboard-shortcut-overrides"; import { useOpenAddProject } from "@/hooks/use-open-add-project"; import { useImportSession } from "@/hooks/use-import-session"; @@ -27,6 +30,7 @@ import { useKeyboardShortcutsStore } from "@/stores/keyboard-shortcuts-store"; import { usePanelStore } from "@/stores/panel-store"; import { useSidebarViewStore } from "@/stores/sidebar-view-store"; import { clearCommandCenterFocusRestoreElement } from "@/utils/command-center-focus-restore"; +import { copyToClipboard } from "@/utils/copy-to-clipboard"; import { buildOpenProjectRoute, buildSchedulesRoute, @@ -35,6 +39,7 @@ import { } from "@/utils/host-routes"; import { getShortcutOs } from "@/utils/shortcut-platform"; import type { CommandCenterContribution, CommandCenterIconProps } from "./contributions"; +import { getCommandCenterIcon } from "./icon"; import { useCommandCenterActions } from "./provider"; import { buildGroupingContribution } from "./root-contributions"; @@ -63,6 +68,7 @@ const ThemedCircleDashed = withUnistyles(CircleDashed, (theme) => ({ const ThemedPanelLeft = withUnistyles(PanelLeft, (theme) => ({ color: theme.colors.foregroundMuted, })); +const BrowserHostCopyIcon = getCommandCenterIcon(Copy); function PlusIcon({ size }: CommandCenterIconProps) { return ; @@ -111,6 +117,8 @@ function PanelLeftIcon({ size }: CommandCenterIconProps) { export function CommandCenterRootActions() { const keyboardActionDispatcher = useKeyboardActionDispatcher(); const { t } = useTranslation(); + const toast = useToast(); + const browserHostLabel = getDesktopHost()?.browserHostLabel; const { overrides } = useKeyboardShortcutOverrides(); const shortcutsAvailable = useHardwareKeyboardPresent(); const openAddProject = useOpenAddProject(); @@ -296,6 +304,32 @@ export function CommandCenterRootActions() { }, ]; + if (browserHostLabel) { + availableActions.push({ + id: "copy-browser-host-name", + group: "actions", + groupRank: 0, + rank: 9, + keywords: ["browser", "host", "window", browserHostLabel], + visibility: "query", + run: async () => { + try { + await copyToClipboard(browserHostLabel); + toast.copied(); + } catch { + toast.error(t("common.errors.unableToCopy")); + } + }, + presentation: { + kind: "action", + title: t("shell.commandCenter.copyBrowserHostName"), + subtitle: browserHostLabel, + sectionTitle: t("shell.commandCenter.actions"), + icon: BrowserHostCopyIcon, + }, + }); + } + if (shortcutsAvailable) { availableActions.push({ id: "keyboard-shortcuts", @@ -332,6 +366,7 @@ export function CommandCenterRootActions() { return availableActions; }, [ + browserHostLabel, groupMode, homeRoute, keyboardActionDispatcher, @@ -346,6 +381,7 @@ export function CommandCenterRootActions() { shortcutPlatform, shortcutsAvailable, t, + toast, toggleAgentList, ]); diff --git a/packages/app/src/desktop/browser/automation/handler.ts b/packages/app/src/desktop/browser/automation/handler.ts index 08bb9927e5..e566d89767 100644 --- a/packages/app/src/desktop/browser/automation/handler.ts +++ b/packages/app/src/desktop/browser/automation/handler.ts @@ -1,4 +1,5 @@ import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas"; +import type { DaemonClient } from "@getpaseo/client/internal/daemon-client"; import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages"; import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host"; import { @@ -26,6 +27,7 @@ type BrowserAutomationExecuteResponse = Extract< type BrowserAutomationResponsePayload = BrowserAutomationExecuteResponse["payload"]; type BrowserAutomationFailurePayload = Extract; type BrowserAutomationErrorCode = BrowserAutomationFailurePayload["error"]["code"]; +type BrowserHostRegistration = Parameters[0]; interface BrowserAutomationClient { on( @@ -69,14 +71,24 @@ export function mountBrowserAutomationHandler( }; } +export function getDesktopBrowserHostRegistration(): BrowserHostRegistration | null { + const desktop = getDesktopHost(); + if (typeof desktop?.browser?.executeAutomationCommand !== "function") return null; + + return { + hostKind: "desktop app", + label: desktop.browserHostLabel, + instanceId: desktop.browserHostInstanceId, + supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], + }; +} + export function mountBrowserAutomationDaemonClientHandler( - client: import("@getpaseo/client/internal/daemon-client").DaemonClient, + client: DaemonClient, + registration: BrowserHostRegistration, options?: { serverId?: string }, ): () => void { - const observation = client.registerBrowserHost({ - hostKind: "desktop app", - supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], - }); + const observation = client.registerBrowserHost(registration); const unmount = mountBrowserAutomationHandler({ client: { on: (_type, handler) => diff --git a/packages/app/src/desktop/host.ts b/packages/app/src/desktop/host.ts index 2541da9f79..71a8489469 100644 --- a/packages/app/src/desktop/host.ts +++ b/packages/app/src/desktop/host.ts @@ -175,6 +175,8 @@ export interface DesktopInvokeBridge { export interface DesktopHostBridge { platform?: string; windowChromeMode?: string; + browserHostLabel?: string; + browserHostInstanceId?: string; invoke?: DesktopInvokeBridge["invoke"]; getPendingOpenProject?: () => Promise; agentNavigation?: DesktopAgentNavigationBridge; diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index fdaf7ae68e..e885b13256 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -115,6 +115,7 @@ export const ar: TranslationResources = { home: "بيت", groupByProject: "تجميع حسب المشروع", groupByStatus: "تجميع حسب الحالة", + copyBrowserHostName: "نسخ اسم مضيف المتصفح", modelGroupLabel: "النموذج", modelSearchKeywords: "تبديل النموذج تغيير النموذج تعيين النموذج اختيار النموذج", thinkingGroupLabel: "التفكير", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index bfb5f526d6..df535636a6 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -113,6 +113,7 @@ export const en = { home: "Home", groupByProject: "Group by project", groupByStatus: "Group by status", + copyBrowserHostName: "Copy browser host name", modelGroupLabel: "Model", modelSearchKeywords: "switch model change model set model select model", thinkingGroupLabel: "Thinking", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index db26ae87b0..afa54968e6 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -116,6 +116,7 @@ export const es: TranslationResources = { home: "Hogar", groupByProject: "Agrupar por proyecto", groupByStatus: "Agrupar por estado", + copyBrowserHostName: "Copiar el nombre del host del navegador", modelGroupLabel: "Modelo", modelSearchKeywords: "cambiar modelo modificar modelo establecer modelo seleccionar modelo", thinkingGroupLabel: "Razonamiento", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index b92da11e0e..f4d498f10d 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -116,6 +116,7 @@ export const fr: TranslationResources = { home: "Accueil", groupByProject: "Grouper par projet", groupByStatus: "Grouper par statut", + copyBrowserHostName: "Copier le nom de l’hôte du navigateur", modelGroupLabel: "Modèle", modelSearchKeywords: "changer de modèle modifier le modèle définir le modèle sélectionner le modèle model", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index 61bc4f74e1..c8860bde1e 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -116,6 +116,7 @@ export const ja: TranslationResources = { home: "ホーム", groupByProject: "プロジェクトでグループ化", groupByStatus: "ステータスでグループ化", + copyBrowserHostName: "ブラウザホスト名をコピー", modelGroupLabel: "モデル", modelSearchKeywords: "モデルを切り替え モデルを変更 モデルを設定 モデルを選択", thinkingGroupLabel: "思考", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index 96fb40cc78..40608117a0 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -115,6 +115,7 @@ export const ko: TranslationResources = { home: "홈", groupByProject: "프로젝트별 그룹화", groupByStatus: "상태별 그룹화", + copyBrowserHostName: "브라우저 호스트 이름 복사", modelGroupLabel: "모델", modelSearchKeywords: "모델 전환 모델 변경 모델 설정 모델 선택", thinkingGroupLabel: "추론", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index e310be4547..dbb2ce47f3 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -116,6 +116,7 @@ export const ptBR: TranslationResources = { home: "Início", groupByProject: "Agrupar por projeto", groupByStatus: "Agrupar por status", + copyBrowserHostName: "Copiar o nome do host do navegador", modelGroupLabel: "Modelo", modelSearchKeywords: "trocar modelo mudar modelo definir modelo selecionar modelo", thinkingGroupLabel: "Raciocínio", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index 3c645e117f..548d55f2bc 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -116,6 +116,7 @@ export const ru: TranslationResources = { home: "Главная", groupByProject: "Группировать по проекту", groupByStatus: "Группировать по статусу", + copyBrowserHostName: "Скопировать имя хоста браузера", modelGroupLabel: "Модель", modelSearchKeywords: "Сменить, изменить, выбрать или установить модель", thinkingGroupLabel: "Мышление", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index ff62199322..8ce7eb5e36 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -115,6 +115,7 @@ export const zhCN: TranslationResources = { home: "首页", groupByProject: "按项目分组", groupByStatus: "按状态分组", + copyBrowserHostName: "复制浏览器宿主名称", modelGroupLabel: "模型", modelSearchKeywords: "切换模型 更改模型 设置模型 选择模型", thinkingGroupLabel: "思考", diff --git a/packages/app/src/runtime/host-runtime.ts b/packages/app/src/runtime/host-runtime.ts index 4cdee90385..8e19c0bbc4 100644 --- a/packages/app/src/runtime/host-runtime.ts +++ b/packages/app/src/runtime/host-runtime.ts @@ -55,10 +55,8 @@ import { buildDesktopDaemonTransportUrl, createDesktopDaemonTransportFactory, } from "@/desktop/daemon/desktop-daemon-transport"; -import { getDesktopHost } from "@/desktop/host"; import { readDesktopManagedLocalCredential } from "@/desktop/daemon/local-credential"; import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities"; -import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import { useSessionStore, toDaemonServerInfo, @@ -74,7 +72,10 @@ import { invalidateServerDataQueriesAfterReconnect, mountServerDataPushRouter, } from "@/data/push-router"; -import { mountBrowserAutomationDaemonClientHandler } from "@/desktop/browser/automation/handler"; +import { + getDesktopBrowserHostRegistration, + mountBrowserAutomationDaemonClientHandler, +} from "@/desktop/browser/automation/handler"; import { schedulesQueryBaseKey } from "@/schedules/aggregated-schedules"; import { dispatchComposerAgentMessage, sendQueuedComposerMessageNow } from "@/composer/actions"; import { createMessageSubmissionWriter } from "@/composer/submission/writer"; @@ -566,19 +567,10 @@ function probeIntervalForConnection( function createDefaultDeps( resolveOrigin: (serverId: string) => DaemonClient | null = () => null, ): HostRuntimeControllerDeps { - const browserHostAvailable = - typeof getDesktopHost()?.browser?.executeAutomationCommand === "function"; - const browserAutomationCapabilities = browserHostAvailable - ? { - [CLIENT_CAPS.browserHost]: { - supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], - hostKind: "desktop app", - }, - } - : undefined; - const appCapabilities = { - ...browserAutomationCapabilities, - }; + const browserHostRegistration = getDesktopBrowserHostRegistration(); + const appCapabilities = browserHostRegistration + ? { [CLIENT_CAPS.browserHost]: browserHostRegistration } + : {}; return { createClient: ({ host, connection, clientId, runtimeGeneration }) => { @@ -702,12 +694,14 @@ function createDefaultDeps( queryClient, serverId: host.serverId, }); - if (!browserAutomationCapabilities) { + if (!browserHostRegistration) { return unmountServerData; } - const unmountBrowserAutomation = mountBrowserAutomationDaemonClientHandler(client, { - serverId: host.serverId, - }); + const unmountBrowserAutomation = mountBrowserAutomationDaemonClientHandler( + client, + browserHostRegistration, + { serverId: host.serverId }, + ); return () => { unmountBrowserAutomation(); unmountServerData(); diff --git a/packages/desktop/e2e/browser-tabs.e2e.mjs b/packages/desktop/e2e/browser-tabs.e2e.mjs index 1c6223e1ca..7234555dcb 100644 --- a/packages/desktop/e2e/browser-tabs.e2e.mjs +++ b/packages/desktop/e2e/browser-tabs.e2e.mjs @@ -338,6 +338,86 @@ async function callBrowserTool(client, name, args = {}) { return mcpPayload(await client.callTool({ name, args }), name); } +async function connectBrowserToolClient(daemonPort, callerAgentId) { + const transport = new StreamableHTTPClientTransport( + new URL( + `http://127.0.0.1:${daemonPort}/mcp/agents?callerAgentId=${encodeURIComponent(callerAgentId)}`, + ), + ); + return experimental_createMCPClient({ transport }); +} + +async function openBrowserWorkspace(page, workspaceKey) { + await page.getByTestId(`sidebar-workspace-row-${workspaceKey}`).click({ timeout: timeoutMs }); + await expect(page.getByTestId(`workspace-deck-entry-${workspaceKey}`)).toBeVisible(); +} + +async function openSecondDesktopWindow(page) { + const nextWindow = page.context().waitForEvent("page", { timeout: timeoutMs }); + await page.evaluate(() => window.paseoDesktop.window.openNew()); + return nextWindow; +} + +async function listBrowserHosts(client) { + const reply = await client.callTool({ name: "browser_list_hosts", args: {} }); + assert(Array.isArray(reply.structuredContent?.browserHosts), JSON.stringify(reply)); + return reply.structuredContent.browserHosts; +} + +async function expectBrowserHostNameInCommandCenter(page, name) { + await page.getByTestId("sidebar-search").click(); + const search = page.getByTestId("command-center-input"); + await search.fill("Copy browser host name"); + const action = page.getByRole("button", { name: /Copy browser host name/ }); + await expect(action).toContainText(name); + await action.click(); + await expect(search).toBeHidden(); + await expect(page.getByText("Copied", { exact: true })).toBeVisible(); +} + +async function verifyBrowserHostRouting({ page, client, targetUrl, workspaceKey }) { + await openBrowserWorkspace(page, workspaceKey); + await expect.poll(async () => (await listBrowserHosts(client)).length).toBe(1); + const [firstHost] = await listBrowserHosts(client); + await expectBrowserHostNameInCommandCenter(page, firstHost.label); + const secondWindow = await openSecondDesktopWindow(page); + + try { + await openBrowserWorkspace(secondWindow, workspaceKey); + await expect.poll(async () => (await listBrowserHosts(client)).length).toBe(2); + const secondHost = (await listBrowserHosts(client)).find( + (host) => host.browserHostId !== firstHost.browserHostId, + ); + assert(secondHost?.label && secondHost.label !== firstHost.label, "Windows share a host label"); + await expectBrowserHostNameInCommandCenter(secondWindow, secondHost.label); + + const ambiguous = await client.callTool({ name: "browser_new_tab", args: { url: targetUrl } }); + expect(ambiguous.structuredContent).toMatchObject({ + ok: false, + error: { code: "browser_host_ambiguous", browserHosts: [firstHost, secondHost] }, + }); + expect((await callBrowserTool(client, "browser_list_tabs")).tabs).toEqual([]); + + const { browserId, browserHost } = await callBrowserTool(client, "browser_new_tab", { + url: targetUrl, + browserHostId: firstHost.browserHostId, + }); + expect(browserHost).toEqual(firstHost); + // Only the tab's test ID carries the exact ID returned by the agent tool. + await expect(page.getByTestId(`workspace-tab-browser_${browserId}`)).toBeVisible(); + await expect(secondWindow.getByTestId(`workspace-tab-browser_${browserId}`)).toHaveCount(0); + expect((await callBrowserTool(client, "browser_list_tabs")).tabs).toMatchObject([ + { browserId, browserHost: firstHost, isActive: false }, + ]); + await callBrowserTool(client, "browser_wait", { browserId, text: "Bridge target" }); + await callBrowserTool(client, "browser_close_tab", { browserId }); + } finally { + await secondWindow.close(); + } + + await expect.poll(() => listBrowserHosts(client)).toEqual([firstHost]); +} + async function waitForGuestSelector(client, browserId) { const deadline = Date.now() + 5_000; while (Date.now() < deadline) { @@ -1413,12 +1493,7 @@ async function main() { workspaceId: workspaceIds.at(-1), agentId: await createCallerAgent(daemonPort, workspaceIds.at(-1)), }; - const transport = new StreamableHTTPClientTransport( - new URL( - `http://127.0.0.1:${daemonPort}/mcp/agents?callerAgentId=${encodeURIComponent(callerAgentId)}`, - ), - ); - client = await experimental_createMCPClient({ transport }); + client = await connectBrowserToolClient(daemonPort, callerAgentId); const report = await runRegression({ page, client, @@ -1430,11 +1505,23 @@ async function main() { artifactDir, }); const pluginLinks = await checkPluginLinks(); + const routingClient = await connectBrowserToolClient(daemonPort, typingAgent.agentId); + try { + await verifyBrowserHostRouting({ + page, + client: routingClient, + targetUrl: target.url, + workspaceKey: `${status.serverId}:${typingAgent.workspaceId}`, + }); + } finally { + await routingClient.close(); + } writeJson(path.join(artifactDir, "result.json"), { ...report, settingsMemory, pluginLinks, adaptiveTheme, + browserHostRouting: "passed", }); console.log( `Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click, local-page selectors passed.`, diff --git a/packages/desktop/src/main.ts b/packages/desktop/src/main.ts index 196f9a47f4..c59d3f27af 100644 --- a/packages/desktop/src/main.ts +++ b/packages/desktop/src/main.ts @@ -10,6 +10,8 @@ log.initialize({ spyRendererConsole: true }); import { inheritLoginShellEnv } from "./login-shell-env.js"; import path from "node:path"; +import { randomBytes, randomUUID } from "node:crypto"; +import { hostname } from "node:os"; import { pathToFileURL } from "node:url"; import { existsSync } from "node:fs"; import { execFileSync } from "node:child_process"; @@ -668,6 +670,14 @@ function getWorkAreasPrimaryFirst(): Electron.Rectangle[] { return [primary, ...others].map((display) => display.workArea); } +const browserHostComputerName = + hostname() + .replace(/\.local$/i, "") + .trim() + .slice(0, 120) || "Desktop"; +const browserHostSessionCode = randomBytes(3).toString("hex").toUpperCase(); +let browserHostWindowNumber = 0; + async function createWindow( options: { initialRoute?: string | null; @@ -693,6 +703,7 @@ async function createWindow( : null; const title = devWorktreeName ? `${APP_NAME} (${devWorktreeName})` : APP_NAME; + const browserHostLabel = `${browserHostComputerName} · Window ${++browserHostWindowNumber} (${browserHostSessionCode})`; const mainWindow = new BrowserWindow({ title, ...resolveWindowBounds(restoredWindowState), @@ -704,7 +715,11 @@ async function createWindow( }), webPreferences: { preload: getPreloadPath(), - additionalArguments: [windowChromeModeArgument(DESKTOP_WINDOW_CHROME_MODE)], + additionalArguments: [ + windowChromeModeArgument(DESKTOP_WINDOW_CHROME_MODE), + `--paseo-browser-host-label=${browserHostLabel}`, + `--paseo-browser-host-instance-id=${randomUUID()}`, + ], contextIsolation: true, nodeIntegration: false, webviewTag: true, diff --git a/packages/desktop/src/preload.ts b/packages/desktop/src/preload.ts index 0942134521..bebb452cec 100644 --- a/packages/desktop/src/preload.ts +++ b/packages/desktop/src/preload.ts @@ -12,6 +12,10 @@ const PASEO_BROWSER_PROFILE_PARTITION = "persist:paseo-browser"; type EventHandler = (payload: unknown) => void; +function readArgValue(prefix: string): string | undefined { + return process.argv.find((argument) => argument.startsWith(prefix))?.slice(prefix.length); +} + function readWindowChromeMode(): DesktopWindowChromeMode { const prefix = "--paseo-window-chrome-mode="; const value = process.argv.find((argument) => argument.startsWith(prefix))?.slice(prefix.length); @@ -32,6 +36,8 @@ interface AttachedBrowserRegistration { contextBridge.exposeInMainWorld("paseoDesktop", { platform: process.platform, windowChromeMode: readWindowChromeMode(), + browserHostLabel: readArgValue("--paseo-browser-host-label="), + browserHostInstanceId: readArgValue("--paseo-browser-host-instance-id="), invoke: (command: string, args?: Record) => ipcRenderer.invoke("paseo:invoke", command, args), getPendingOpenProject: () => diff --git a/packages/protocol/src/browser-automation/capabilities.ts b/packages/protocol/src/browser-automation/capabilities.ts index 415c6b442a..767c52650a 100644 --- a/packages/protocol/src/browser-automation/capabilities.ts +++ b/packages/protocol/src/browser-automation/capabilities.ts @@ -31,6 +31,8 @@ export const BrowserAutomationHostCapabilitySchema = z return supportedCommands; }), hostKind: z.string().min(1).default("browser host"), + label: z.string().min(1).max(160).optional(), + instanceId: z.uuid().optional(), }) .passthrough(); diff --git a/packages/protocol/src/messages.browser-automation.test.ts b/packages/protocol/src/messages.browser-automation.test.ts index 3e5805b699..5cf7d33f29 100644 --- a/packages/protocol/src/messages.browser-automation.test.ts +++ b/packages/protocol/src/messages.browser-automation.test.ts @@ -13,7 +13,7 @@ import { describe("browser automation protocol integration", () => { const browserId = "11111111-1111-4111-8111-111111111111"; - test("browser host capability parses supported commands in hello", () => { + test("browser host capability parses supported commands and a label in hello", () => { expect( WSHelloMessageSchema.parse({ type: "hello", @@ -24,6 +24,7 @@ describe("browser automation protocol integration", () => { [CLIENT_CAPS.browserHost]: { supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], hostKind: "desktop app", + label: "MacBook Pro · Window 2", }, }, }).capabilities, @@ -31,6 +32,7 @@ describe("browser automation protocol integration", () => { [CLIENT_CAPS.browserHost]: { supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES], hostKind: "desktop app", + label: "MacBook Pro · Window 2", }, }); }); @@ -133,6 +135,23 @@ describe("browser automation protocol integration", () => { ).toBeUndefined(); }); + test("browser host registration accepts window identity and remains valid without it", () => { + const request = { + type: "browser.host.register.request", + requestId: "register-browser", + hostKind: "desktop app", + supportedCommands: ["new_tab"], + }; + const label = "MacBook Pro · Window 2"; + const instanceId = "33333333-3333-4333-8333-333333333333"; + expect(SessionInboundMessageSchema.parse(request)).toEqual(request); + expect(SessionInboundMessageSchema.parse({ ...request, label, instanceId })).toEqual({ + ...request, + label, + instanceId, + }); + }); + test("daemon to browser host execute request is an outbound session message", () => { const parsed = SessionOutboundMessageSchema.parse({ type: "browser.automation.execute.request", diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index e62cf8357e..be4041976e 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -3493,6 +3493,8 @@ export const BrowserHostRegisterRequestSchema = z.object({ type: z.literal("browser.host.register.request"), requestId: z.string(), hostKind: z.string().min(1), + label: z.string().min(1).max(160).optional(), + instanceId: z.uuid().optional(), supportedCommands: z.array(z.enum(BROWSER_AUTOMATION_COMMAND_NAMES)).min(1), }); export const BrowserHostRegisterResponseSchema = z.object({ diff --git a/packages/protocol/src/paseo-tool-presentation/catalog.ts b/packages/protocol/src/paseo-tool-presentation/catalog.ts index 57562bc15d..d60b245b05 100644 --- a/packages/protocol/src/paseo-tool-presentation/catalog.ts +++ b/packages/protocol/src/paseo-tool-presentation/catalog.ts @@ -390,6 +390,7 @@ export const PASEO_TOOL_CATALOG = { }, // Browser + browser_list_hosts: browser("List browser hosts", "Listed browser hosts"), browser_list_tabs: browser("List browser tabs", "Listed browser tabs"), browser_new_tab: browser("Open browser tab", "Opened browser tab", field("url")), browser_close_tab: browser("Close browser tab", "Closed browser tab", field("browserId")), diff --git a/packages/server/src/server/browser-tools/broker.test.ts b/packages/server/src/server/browser-tools/broker.test.ts index 17c49f0c03..21cdecb618 100644 --- a/packages/server/src/server/browser-tools/broker.test.ts +++ b/packages/server/src/server/browser-tools/broker.test.ts @@ -14,6 +14,8 @@ const SECOND_BROWSER_ID = "22222222-2222-4222-8222-222222222222"; class FakeBrowserHostClient implements BrowserHostClient { public readonly receivedRequests: BrowserAutomationExecuteRequest[] = []; public readonly hostKind: string; + public readonly label?: string; + public readonly instanceId?: string; public readonly supportedCommands: readonly BrowserAutomationCommandName[]; public constructor( @@ -21,9 +23,13 @@ class FakeBrowserHostClient implements BrowserHostClient { options: { hostKind?: string; supportedCommands?: readonly BrowserAutomationCommandName[]; + label?: string; + instanceId?: string; } = {}, ) { this.hostKind = options.hostKind ?? "desktop app"; + this.label = options.label; + this.instanceId = options.instanceId; this.supportedCommands = options.supportedCommands ?? [...BROWSER_AUTOMATION_COMMAND_NAMES]; } @@ -170,6 +176,7 @@ describe("BrowserToolsBroker", () => { title: "Example", isActive: false, isLoading: false, + browserHost: broker.listBrowserHosts()[0], }, ], }, @@ -229,20 +236,31 @@ describe("BrowserToolsBroker", () => { }); }); - test("new tabs target the most recently registered host and tab commands stay with that host", async () => { + test("new tabs require a choice with multiple hosts and tab commands stay with the chosen host", async () => { const broker = createBroker(); - const firstHost = new FakeBrowserHostClient("host-1"); + const firstHost = new FakeBrowserHostClient("host-1", { label: "MacBook Pro · Window 1" }); const recentHost = new FakeBrowserHostClient("host-2"); broker.registerClient(firstHost); broker.registerClient(recentHost); - const newTabPromise = broker.execute({ + const input = { command: { command: "new_tab", args: { url: "https://example.com" } }, workspaceId: "workspace-1", + } as const; + const [selectedHost, otherHost] = broker.listBrowserHosts(); + expect(selectedHost.label).toBe("MacBook Pro · Window 1"); + expect(otherHost.label).toBe("desktop app · Connection 2"); + await expect(broker.execute(input)).resolves.toMatchObject({ + ok: false, + error: { code: "browser_host_ambiguous", browserHosts: [selectedHost, otherHost] }, }); - expect(firstHost.receivedRequests).toEqual([]); - expect(recentHost.receivedRequests).toEqual([ + expect(recentHost.receivedRequests).toEqual([]); + + const newTabPromise = broker.execute({ ...input, browserHostId: selectedHost.browserHostId }); + + expect(recentHost.receivedRequests).toEqual([]); + expect(firstHost.receivedRequests).toEqual([ { type: "browser.automation.execute.request", requestId: "req-1", @@ -251,7 +269,7 @@ describe("BrowserToolsBroker", () => { }, ]); - recentHost.resolveLatestWith(broker, { + firstHost.resolveLatestWith(broker, { requestId: "req-1", ok: true, result: { @@ -264,7 +282,7 @@ describe("BrowserToolsBroker", () => { await expect(newTabPromise).resolves.toMatchObject({ ok: true, - result: { command: "new_tab", browserId: BROWSER_ID }, + result: { command: "new_tab", browserId: BROWSER_ID, browserHost: selectedHost }, }); const snapshotPromise = broker.execute({ @@ -272,15 +290,15 @@ describe("BrowserToolsBroker", () => { workspaceId: "workspace-1", }); - expect(firstHost.receivedRequests).toEqual([]); - expect(recentHost.receivedRequests.at(-1)).toEqual({ + expect(recentHost.receivedRequests).toEqual([]); + expect(firstHost.receivedRequests.at(-1)).toEqual({ type: "browser.automation.execute.request", requestId: "req-1", workspaceId: "workspace-1", command: { command: "snapshot", args: { browserId: BROWSER_ID } }, }); - recentHost.resolveLatestWith(broker, { + firstHost.resolveLatestWith(broker, { requestId: "req-1", ok: true, result: { @@ -302,6 +320,102 @@ describe("BrowserToolsBroker", () => { }); }); + test("a disconnected selection stays invalid and the sole eligible host is automatic", async () => { + const broker = createBroker(); + const formerHost = new FakeBrowserHostClient("former", { label: "Mac Mini · Window 1" }); + const remainingHost = new FakeBrowserHostClient("remaining"); + const inventoryHost = new FakeBrowserHostClient("inventory", { + supportedCommands: ["list_tabs"], + }); + const unregister = broker.registerClient(formerHost); + broker.registerClient(remainingHost); + broker.registerClient(inventoryHost); + const [previous, remaining, inventoryOnly] = broker.listBrowserHosts(); + expect(inventoryOnly.canOpenTabs).toBe(false); + unregister(); + expect(broker.listBrowserHosts()).toEqual([remaining, inventoryOnly]); + + const input = { + command: { command: "new_tab", args: {} }, + workspaceId: "workspace-1", + } as const; + await expect( + broker.execute({ ...input, browserHostId: previous.browserHostId }), + ).resolves.toMatchObject({ ok: false, error: { code: "browser_no_host" } }); + await expect( + broker.execute({ ...input, browserHostId: inventoryOnly.browserHostId }), + ).resolves.toMatchObject({ ok: false, error: { code: "browser_unsupported" } }); + expect(remainingHost.receivedRequests).toEqual([]); + expect(inventoryHost.receivedRequests).toEqual([]); + + const creation = broker.execute(input); + remainingHost.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "new_tab", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + }, + }); + await expect(creation).resolves.toMatchObject({ + ok: true, + result: { browserHost: remaining }, + }); + expect(inventoryHost.receivedRequests).toEqual([]); + + broker.registerClient(new FakeBrowserHostClient("reconnected", { label: previous.label })); + const reconnected = broker.listBrowserHosts().find(({ label }) => label === previous.label); + expect(reconnected).toBeDefined(); + expect(reconnected?.browserHostId).not.toBe(previous.browserHostId); + await expect( + broker.execute({ ...input, browserHostId: previous.browserHostId }), + ).resolves.toMatchObject({ ok: false, error: { code: "browser_no_host" } }); + expect(formerHost.receivedRequests).toEqual([]); + }); + + test("a reconnect replaces the old connection for the same browser host", async () => { + const broker = createBroker(); + const identity = { + instanceId: "33333333-3333-4333-8333-333333333333", + label: "Desktop · Window 1", + }; + const oldConnection = new FakeBrowserHostClient("old", identity); + const releaseOld = broker.registerClient(oldConnection); + const [target] = broker.listBrowserHosts(); + const pending = broker.execute({ command: { command: "new_tab", args: {} } }); + + const newConnection = new FakeBrowserHostClient("new", identity); + broker.registerClient(newConnection); + expect(broker.listBrowserHosts()).toEqual([target]); + await expect(pending).resolves.toMatchObject({ + ok: false, + error: { code: "browser_no_host" }, + }); + releaseOld(); + + const creation = broker.execute({ + command: { command: "new_tab", args: {} }, + browserHostId: target.browserHostId, + }); + newConnection.resolveLatestWith(broker, { + requestId: "req-1", + ok: true, + result: { + command: "new_tab", + browserId: BROWSER_ID, + workspaceId: "workspace-1", + url: "https://example.com", + }, + }); + await expect(creation).resolves.toMatchObject({ + ok: true, + result: { browserHost: target }, + }); + expect(oldConnection.receivedRequests).toHaveLength(1); + }); + test("list tabs aggregates all hosts and seeds browser id affinity", async () => { const broker = createBroker(); const firstHost = new FakeBrowserHostClient("host-1"); @@ -375,6 +489,7 @@ describe("BrowserToolsBroker", () => { title: "One", isActive: false, isLoading: false, + browserHost: broker.listBrowserHosts()[0], }, { browserId: SECOND_BROWSER_ID, @@ -383,6 +498,7 @@ describe("BrowserToolsBroker", () => { title: "Two", isActive: false, isLoading: false, + browserHost: broker.listBrowserHosts()[1], }, ], }, @@ -479,7 +595,11 @@ describe("BrowserToolsBroker", () => { broker.registerClient(other); broker.registerClient(owner); - const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } }); + const [, selectedHost] = broker.listBrowserHosts(); + const newTabPromise = broker.execute({ + command: { command: "new_tab", args: {} }, + browserHostId: selectedHost.browserHostId, + }); owner.resolveLatestWith(broker, { requestId: "req-1", ok: true, @@ -521,7 +641,11 @@ describe("BrowserToolsBroker", () => { broker.registerClient(other); broker.registerClient(owner); - const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } }); + const [, selectedHost] = broker.listBrowserHosts(); + const newTabPromise = broker.execute({ + command: { command: "new_tab", args: {} }, + browserHostId: selectedHost.browserHostId, + }); owner.resolveLatestWith(broker, { requestId: "req-1", ok: true, diff --git a/packages/server/src/server/browser-tools/broker.ts b/packages/server/src/server/browser-tools/broker.ts index cd66b0d7b0..dd628a6848 100644 --- a/packages/server/src/server/browser-tools/broker.ts +++ b/packages/server/src/server/browser-tools/broker.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHmac, randomBytes, randomUUID } from "node:crypto"; import { BrowserAutomationExecuteRequestSchema, BrowserAutomationExecuteResponseSchema, @@ -7,11 +7,17 @@ import { type BrowserAutomationExecuteRequest, type BrowserAutomationExecuteResponse, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; -import { browserToolsFailure, type BrowserToolsResponsePayload } from "./errors.js"; +import { + browserToolsFailure, + type BrowserHostTarget, + type BrowserToolsResponsePayload, +} from "./errors.js"; export interface BrowserHostClient { id: string; hostKind: string; + label?: string; + instanceId?: string; supportedCommands: readonly BrowserAutomationCommandName[]; sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void | Promise; } @@ -23,10 +29,11 @@ export interface BrowserToolsExecuteInput { workspaceId?: string; requestId?: string; timeoutMs?: number; + browserHostId?: string; } interface PendingBrowserToolsRequest { - clientId: string; + host: RegisteredBrowserHost; rememberAffinity: boolean; timeout: ReturnType; resolve: (payload: BrowserToolsResponsePayload) => void; @@ -34,10 +41,15 @@ interface PendingBrowserToolsRequest { interface RegisteredBrowserHost { client: BrowserHostClient; + target: BrowserHostTarget; registeredAt: number; supportedCommands: ReadonlySet; } +type HostSelection = + | { ok: true; value: RegisteredBrowserHost } + | { ok: false; payload: BrowserToolsResponsePayload }; + export interface BrowserToolsBrokerOptions { defaultTimeoutMs?: number; createRequestId?: () => string; @@ -51,7 +63,7 @@ export class BrowserToolsBroker { private readonly clients = new Map(); private readonly pending = new Map(); private readonly browserHostByBrowserId = new Map(); - private readonly strandedBrowserHostByBrowserId = new Map(); + private readonly hostIdKey = randomBytes(32); private registrationSequence = 0; public constructor(options: BrowserToolsBrokerOptions) { @@ -60,35 +72,36 @@ export class BrowserToolsBroker { } public registerClient(client: BrowserHostClient): () => void { - this.unregisterClient(client.id); + // The discovery ID must not allow another connection to claim this window. + const browserHostId = createHmac("sha256", this.hostIdKey) + .update(client.instanceId ? `instance:${client.instanceId}` : `connection:${client.id}`) + .digest("hex"); + this.unregisterHost(browserHostId); const registeredAt = ++this.registrationSequence; - this.clients.set(client.id, { + const label = client.label?.trim() || `${client.hostKind} · Connection ${registeredAt}`; + const supportedCommands = new Set(client.supportedCommands); + this.clients.set(browserHostId, { client, + target: Object.freeze({ + browserHostId, + label, + canOpenTabs: supportedCommands.has("new_tab"), + }), registeredAt, - supportedCommands: new Set(client.supportedCommands), + supportedCommands, }); - return () => this.unregisterClient(client.id, registeredAt); + return () => this.unregisterHost(browserHostId, registeredAt); } - public unregisterClient(clientId: string, registeredAt?: number): void { - const current = this.clients.get(clientId); + private unregisterHost(browserHostId: string, registeredAt?: number): void { + const current = this.clients.get(browserHostId); if (!current || (registeredAt !== undefined && current.registeredAt !== registeredAt)) { return; } - this.clients.delete(clientId); - - for (const [browserId, ownerClientId] of this.browserHostByBrowserId) { - if (ownerClientId !== clientId) { - continue; - } - this.browserHostByBrowserId.delete(browserId); - this.strandedBrowserHostByBrowserId.set(browserId, clientId); - } + this.clients.delete(browserHostId); for (const [requestId, pending] of this.pending) { - if (pending.clientId !== clientId) { - continue; - } + if (pending.host !== current) continue; this.pending.delete(requestId); clearTimeout(pending.timeout); pending.resolve( @@ -110,6 +123,10 @@ export class BrowserToolsBroker { return this.clients.size; } + public listBrowserHosts(): BrowserHostTarget[] { + return Array.from(this.clients.values(), ({ target }) => target); + } + public async execute(input: BrowserToolsExecuteInput): Promise { const requestId = input.requestId ?? this.createRequestId(); @@ -137,7 +154,11 @@ export class BrowserToolsBroker { }); } - const host = this.selectHostForCommand(request.data.command, requestId); + const command = request.data.command; + const host = + command.command === "new_tab" + ? this.selectHostForNewTab(requestId, input.browserHostId) + : this.selectHostForTab(command.args.browserId, requestId); if (!host.ok) { return host.payload; } @@ -170,7 +191,7 @@ export class BrowserToolsBroker { } const pending = this.pending.get(requestId); - if (!pending || (clientIds && !clientIds.includes(pending.clientId))) { + if (!pending || (clientIds && !clientIds.includes(pending.host.client.id))) { return false; } @@ -187,16 +208,16 @@ export class BrowserToolsBroker { } const pending = this.pending.get(parsed.data.payload.requestId); - if (!pending || (clientIds && !clientIds.includes(pending.clientId))) { + if (!pending || (clientIds && !clientIds.includes(pending.host.client.id))) { return false; } this.pending.delete(parsed.data.payload.requestId); clearTimeout(pending.timeout); - if (pending.rememberAffinity) { - this.rememberBrowserHostForPayload(pending.clientId, parsed.data.payload); + if (pending.rememberAffinity && parsed.data.payload.ok) { + this.rememberBrowserHostForPayload(pending.host.target.browserHostId, parsed.data.payload); } - pending.resolve(parsed.data.payload); + pending.resolve(attachBrowserHost(parsed.data.payload, pending.host.target)); return true; } @@ -249,7 +270,9 @@ export class BrowserToolsBroker { } for (const { host, payload } of hostResponses) { - this.rememberBrowserHostForPayload(host.client.id, payload); + if (payload.ok && this.clients.get(host.target.browserHostId) === host) { + this.rememberBrowserHostForPayload(host.target.browserHostId, payload); + } } return { @@ -264,63 +287,60 @@ export class BrowserToolsBroker { }; } - private selectHostForCommand( - command: BrowserAutomationCommand, - requestId: string, - ): - | { ok: true; value: RegisteredBrowserHost } - | { ok: false; payload: BrowserToolsResponsePayload } { - if (command.command === "new_tab") { - const host = this.selectMostRecentlyRegisteredHost(); - return host - ? { ok: true, value: host } - : { ok: false, payload: this.noBrowserHostFailure(requestId) }; - } - - const browserId = getBrowserIdForCommand(command); - if (!browserId) { - const host = this.selectMostRecentlyRegisteredHost(); - return host - ? { ok: true, value: host } - : { ok: false, payload: this.noBrowserHostFailure(requestId) }; - } - - const ownerClientId = this.browserHostByBrowserId.get(browserId); - if (ownerClientId) { - const host = this.clients.get(ownerClientId); - if (host) { - return { ok: true, value: host }; - } + private selectHostForNewTab(requestId: string, browserHostId: string | undefined): HostSelection { + if (browserHostId !== undefined) { + const host = this.clients.get(browserHostId); + if (host) return { ok: true, value: host }; return { ok: false, - payload: this.strandedBrowserTabFailure({ requestId, browserId }), + payload: browserToolsFailure({ + requestId, + code: "browser_no_host", + message: + "The selected browser host is not connected. Call browser_list_hosts to choose a connected host.", + }), }; } - const strandedOwnerClientId = this.strandedBrowserHostByBrowserId.get(browserId); - if (strandedOwnerClientId) { - const reconnectedHost = this.clients.get(strandedOwnerClientId); - if (reconnectedHost) { - this.strandedBrowserHostByBrowserId.delete(browserId); - this.browserHostByBrowserId.set(browserId, strandedOwnerClientId); - return { ok: true, value: reconnectedHost }; - } + if (this.clients.size === 0) + return { ok: false, payload: this.noBrowserHostFailure(requestId) }; + const eligible = Array.from(this.clients.values()).filter(({ target }) => target.canOpenTabs); + if (eligible.length === 1) return { ok: true, value: eligible[0] }; + if (eligible.length === 0) { return { ok: false, - payload: this.strandedBrowserTabFailure({ requestId, browserId }), + payload: browserToolsFailure({ + requestId, + code: "browser_unsupported", + message: "None of the connected browser hosts support opening tabs.", + }), }; } + return { + ok: false, + payload: browserToolsFailure({ + requestId, + code: "browser_host_ambiguous", + message: "Choose a browser host and retry with its browserHostId.", + browserHosts: eligible.map(({ target }) => target), + }), + }; + } - if (this.clients.size === 1) { - const host = this.selectMostRecentlyRegisteredHost(); - if (host) { - return { ok: true, value: host }; - } + private selectHostForTab(browserId: string, requestId: string): HostSelection { + const browserHostId = this.browserHostByBrowserId.get(browserId); + if (browserHostId) { + const host = this.clients.get(browserHostId); + return host + ? { ok: true, value: host } + : { ok: false, payload: this.strandedBrowserTabFailure({ requestId, browserId }) }; } - if (this.clients.size === 0) { + const firstHost = this.clients.values().next().value; + if (!firstHost) { return { ok: false, payload: this.noBrowserHostFailure(requestId) }; } + if (this.clients.size === 1) return { ok: true, value: firstHost }; return { ok: false, @@ -332,14 +352,6 @@ export class BrowserToolsBroker { }; } - private selectMostRecentlyRegisteredHost(): RegisteredBrowserHost | null { - let selected: RegisteredBrowserHost | null = null; - for (const host of this.clients.values()) { - selected = host; - } - return selected; - } - private unsupportedCommandFailure(params: { host: RegisteredBrowserHost; commandName: BrowserAutomationCommandName; @@ -377,30 +389,23 @@ export class BrowserToolsBroker { } private rememberBrowserHostForPayload( - clientId: string, - payload: BrowserToolsResponsePayload, + browserHostId: string, + payload: Extract, ): void { - if (!payload.ok) { - return; - } - if (payload.result.command === "list_tabs") { for (const tab of payload.result.tabs) { - this.browserHostByBrowserId.set(tab.browserId, clientId); - this.strandedBrowserHostByBrowserId.delete(tab.browserId); + this.browserHostByBrowserId.set(tab.browserId, browserHostId); } return; } if (payload.result.command === "close_tab") { this.browserHostByBrowserId.delete(payload.result.browserId); - this.strandedBrowserHostByBrowserId.delete(payload.result.browserId); return; } if ("browserId" in payload.result) { - this.browserHostByBrowserId.set(payload.result.browserId, clientId); - this.strandedBrowserHostByBrowserId.delete(payload.result.browserId); + this.browserHostByBrowserId.set(payload.result.browserId, browserHostId); } } @@ -429,7 +434,7 @@ export class BrowserToolsBroker { }, timeoutMs); this.pending.set(request.requestId, { - clientId: client.id, + host, rememberAffinity: params.rememberAffinity ?? true, timeout, resolve, @@ -458,11 +463,20 @@ export class BrowserToolsBroker { } } -function getBrowserIdForCommand(command: BrowserAutomationCommand): string | null { - if (command.command === "list_tabs" || command.command === "new_tab") { - return null; +function attachBrowserHost( + payload: BrowserAutomationExecuteResponse["payload"], + target: BrowserHostTarget, +): BrowserToolsResponsePayload { + if (!payload.ok) return payload; + const result = payload.result; + if (result.command === "new_tab") { + return { ...payload, result: { ...result, browserHost: target } }; } - return command.args.browserId; + if (result.command === "list_tabs") { + const tabs = result.tabs.map((tab) => ({ ...tab, browserHost: target })); + return { ...payload, result: { ...result, tabs } }; + } + return { ...payload, result }; } function describeBrowserHost(host: RegisteredBrowserHost): string { diff --git a/packages/server/src/server/browser-tools/errors.ts b/packages/server/src/server/browser-tools/errors.ts index fe49edef6f..7a117834bd 100644 --- a/packages/server/src/server/browser-tools/errors.ts +++ b/packages/server/src/server/browser-tools/errors.ts @@ -3,14 +3,40 @@ import type { BrowserAutomationExecuteResponse, } from "@getpaseo/protocol/browser-automation/rpc-schemas"; -export type BrowserToolsResponsePayload = BrowserAutomationExecuteResponse["payload"]; -export type BrowserToolsErrorPayload = Extract["error"]; +export interface BrowserHostTarget { + readonly browserHostId: string; + readonly label: string; + readonly canOpenTabs: boolean; +} + +type BrowserHostResponse = BrowserAutomationExecuteResponse["payload"]; +type BrowserHostSuccess = Extract; +type BrowserHostResult = BrowserHostSuccess["result"]; +type NewTabResult = Extract; +type ListTabsResult = Extract; +type BrowserHostFailure = Extract; + +export type BrowserToolsErrorPayload = Omit & { + code: BrowserAutomationErrorCode | "browser_host_ambiguous"; + browserHosts?: BrowserHostTarget[]; +}; + +export type BrowserToolsResponsePayload = + | (Omit & { + result: + | Exclude + | (NewTabResult & { browserHost: BrowserHostTarget }) + | (Omit & { + tabs: (ListTabsResult["tabs"][number] & { browserHost: BrowserHostTarget })[]; + }); + }) + | (Omit & { error: BrowserToolsErrorPayload }); export class BrowserToolsRequestError extends Error { public readonly code: BrowserAutomationErrorCode; public readonly retryable: boolean; - public constructor(error: BrowserToolsErrorPayload) { + public constructor(error: BrowserHostFailure["error"]) { super(error.message); this.name = "BrowserToolsRequestError"; this.code = error.code; @@ -20,9 +46,10 @@ export class BrowserToolsRequestError extends Error { export function browserToolsFailure(params: { requestId: string; - code: BrowserAutomationErrorCode; + code: BrowserToolsErrorPayload["code"]; message: string; retryable?: boolean; + browserHosts?: BrowserHostTarget[]; }): BrowserToolsResponsePayload { return { requestId: params.requestId, @@ -31,6 +58,7 @@ export function browserToolsFailure(params: { code: params.code, message: params.message, retryable: params.retryable ?? false, + ...(params.browserHosts ? { browserHosts: params.browserHosts } : {}), }, }; } diff --git a/packages/server/src/server/browser-tools/tools.test.ts b/packages/server/src/server/browser-tools/tools.test.ts index e4bd3650e0..197717ccd8 100644 --- a/packages/server/src/server/browser-tools/tools.test.ts +++ b/packages/server/src/server/browser-tools/tools.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "vitest"; import { z } from "zod"; -import type { BrowserToolsBroker, BrowserToolsExecuteInput } from "./broker.js"; +import type { BrowserToolsExecuteInput } from "./broker.js"; import type { BrowserToolsResponsePayload } from "./errors.js"; import { registerBrowserTools, type RegisterBrowserToolsOptions } from "./tools.js"; import type { @@ -10,6 +10,11 @@ import type { } from "../agent/tools/types.js"; const BROWSER_ID = "11111111-1111-4111-8111-111111111111"; +const BROWSER_HOST = { + browserHostId: "22222222-2222-4222-8222-222222222222", + label: "MacBook Pro · Window 1", + canOpenTabs: true, +}; const BROWSER_ID_MESSAGE = "browserId must be a real id returned by browser_new_tab or browser_list_tabs"; const WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url"; @@ -27,6 +32,10 @@ class FakeBrowserBroker { public constructor(private response: BrowserToolsResponsePayload = listTabsPayload()) {} + public listBrowserHosts() { + return [BROWSER_HOST]; + } + public setResponse(response: BrowserToolsResponsePayload): void { this.response = response; } @@ -53,7 +62,7 @@ class BrowserToolHarness { registerTool: (name, config, handler) => { this.tools.set(name, { config, handler }); }, - broker: this.broker as Pick, + broker: this.broker, ...(this.callerAgentId ? { callerAgentId: this.callerAgentId } : {}), resolveCallerAgent: () => this.callerAgent, }); @@ -100,6 +109,7 @@ function listTabsPayload(): Extract { tabs: [ { browserId: BROWSER_ID, + browserHost: BROWSER_HOST, url: "https://example.com", title: "Example", isActive: true, @@ -117,6 +127,7 @@ function newTabPayload(): Extract { result: { command: "new_tab", browserId: BROWSER_ID, + browserHost: BROWSER_HOST, workspaceId: "wks_workspace_a", url: "https://example.com", }, @@ -565,6 +576,7 @@ describe("registerBrowserTools", () => { const harness = new BrowserToolHarness(); expect(harness.toolNames()).toEqual([ + "browser_list_hosts", "browser_list_tabs", "browser_new_tab", "browser_snapshot", @@ -606,7 +618,7 @@ describe("registerBrowserTools", () => { expect(response.content).toEqual([ { type: "text", - text: `Found 1 Paseo browser tab. Use these browserId values for tab-scoped browser tools.\n- browserId=${BROWSER_ID} active title="Example" url=https://example.com`, + text: `Found 1 Paseo browser tab. Use these browserId values for tab-scoped browser tools.\n- browserId=${BROWSER_ID} active title="Example" url=https://example.com host="${BROWSER_HOST.label}"`, }, ]); }); @@ -628,11 +640,48 @@ describe("registerBrowserTools", () => { expect(response.content).toEqual([ { type: "text", - text: `Created browser tab browserId=${BROWSER_ID} url=https://example.com. Use this browserId for tab-scoped browser tools.`, + text: `Created browser tab browserId=${BROWSER_ID} url=https://example.com on ${BROWSER_HOST.label}. Use this browserId for tab-scoped browser tools.`, }, ]); }); + test("lists labeled browser hosts and passes the selected ID for new tabs", async () => { + const harness = new BrowserToolHarness(); + const [host] = harness.broker.listBrowserHosts(); + const hosts = await harness.execute("browser_list_hosts", {}); + expect(hosts.structuredContent).toEqual({ browserHosts: [host] }); + expect(hosts.content).toEqual([ + { type: "text", text: `Connected browser hosts\n- ${host.label} (${host.browserHostId})` }, + ]); + expect(harness.broker.calls).toEqual([]); + + harness.broker.setResponse({ + requestId: "ambiguous", + ok: false, + error: { + code: "browser_host_ambiguous", + message: "Choose a browser host.", + retryable: false, + browserHosts: [host], + }, + }); + const ambiguous = await harness.execute("browser_new_tab", {}); + expect(ambiguous.structuredContent).toMatchObject({ + ok: false, + error: { code: "browser_host_ambiguous", browserHosts: [host] }, + }); + expect(ambiguous.content).toEqual([ + { type: "text", text: `Choose a browser host.\n- ${host.label} (${host.browserHostId})` }, + ]); + + harness.broker.setResponse(newTabPayload()); + await harness.execute("browser_new_tab", { browserHostId: host.browserHostId }); + expect(harness.broker.calls.at(-1)).toMatchObject({ + browserHostId: host.browserHostId, + command: { command: "new_tab", args: {} }, + }); + }); + test.each([ { name: "navigate accepts localhost without a scheme as http", diff --git a/packages/server/src/server/browser-tools/tools.ts b/packages/server/src/server/browser-tools/tools.ts index ddb3c81327..c386d1b249 100644 --- a/packages/server/src/server/browser-tools/tools.ts +++ b/packages/server/src/server/browser-tools/tools.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas"; import type { BrowserToolsBroker } from "./broker.js"; -import type { BrowserToolsResponsePayload } from "./errors.js"; +import type { BrowserHostTarget, BrowserToolsResponsePayload } from "./errors.js"; import type { PaseoToolConfig, PaseoToolExecutionContext, @@ -24,7 +24,7 @@ export interface RegisterBrowserToolsOptions { context: PaseoToolExecutionContext, ) => Promise, ) => void; - broker: Pick; + broker: Pick; callerAgentId?: string; resolveCallerAgent: () => CallerAgentContext | null; } @@ -64,6 +64,24 @@ const BrowserWaitInputSchema = z }); export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { + options.registerTool( + "browser_list_hosts", + { + title: "List browser hosts", + description: + "List connected Paseo browser hosts, including hosts with no tabs. Use a browserHostId from this list to choose where browser_new_tab opens.", + inputSchema: {}, + }, + async () => { + const browserHosts = options.broker.listBrowserHosts(); + const text = + browserHosts.length > 0 + ? ["Connected browser hosts", ...browserHosts.map(formatBrowserHost)].join("\n") + : "No browser hosts are connected."; + return { content: [{ type: "text", text }], structuredContent: { browserHosts } }; + }, + ); + options.registerTool( "browser_list_tabs", { @@ -96,12 +114,20 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void { title: "Create browser tab", description: - "Create a new Paseo browser tab in this agent's workspace on the most recently connected browser automation host, opened in the background without switching the user's view. Pass an http(s) URL or a scheme-less host URL, which is treated as http; the returned browserId is used by tab-scoped tools.", + "Open a background browser tab in this agent's workspace. Choose a browserHostId when multiple browser hosts can open tabs. Use the returned browserId for tab commands.", inputSchema: { - url: BrowserHttpUrlInputSchema.optional(), + url: BrowserHttpUrlInputSchema.optional().describe( + "An HTTP or HTTPS URL. A URL without a scheme uses HTTP.", + ), + browserHostId: z + .string() + .min(1) + .max(128) + .optional() + .describe("Destination ID from browser_list_hosts."), }, }, - async ({ url }) => { + async ({ url, browserHostId }) => { const context = resolveBrowserToolContext(options); const missingWorkspace = requireWorkspaceContext(context); if (missingWorkspace) { @@ -111,6 +137,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void agentId: context.agentId, cwd: context.cwd, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), + ...(browserHostId ? { browserHostId } : {}), command: { command: "new_tab", args: url ? { url } : {}, @@ -875,7 +902,7 @@ function summarizeBrowserSuccess( } const tabLines = payload.result.tabs.map((tab) => { const active = tab.isActive ? " active" : ""; - return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url}`; + return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url} host=${JSON.stringify(tab.browserHost.label)}`; }); return withDialogs( [ @@ -887,7 +914,7 @@ function summarizeBrowserSuccess( if (payload.result.command === "new_tab") { return withDialogs( - `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`, + `Created browser tab browserId=${payload.result.browserId} url=${payload.result.url} on ${payload.result.browserHost.label}. Use this browserId for tab-scoped browser tools.`, ); } @@ -1041,7 +1068,14 @@ function summarizeBrowserError( return "Browser tools are disabled. Enable browser tools on the host, then try again."; case "browser_stale_ref": return "That browser element reference is stale. Take a new browser snapshot and try again."; + case "browser_host_ambiguous": + return [error.message, ...(error.browserHosts ?? []).map(formatBrowserHost)].join("\n"); default: return error.message; } } + +function formatBrowserHost(host: BrowserHostTarget): string { + const capability = host.canOpenTabs ? "" : " (cannot open tabs)"; + return `- ${host.label} (${host.browserHostId})${capability}`; +} diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index d4a9141124..18080617e7 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -1407,6 +1407,8 @@ export class Session { type: "browser.host.register.request" as const, requestId: uuidv4(), hostKind: host.data.hostKind, + label: host.data.label, + instanceId: host.data.instanceId, supportedCommands: host.data.supportedCommands, }; void this.delivery @@ -2571,6 +2573,8 @@ export class Session { unregister = this.browserToolsBroker.registerClient({ id: owner.id, hostKind: request.hostKind, + label: request.label, + instanceId: request.instanceId, supportedCommands: request.supportedCommands, sendBrowserAutomationRequest: (message) => owner.emit(message), }); diff --git a/public-docs/browser-tools.md b/public-docs/browser-tools.md index f1d586d3c0..351927bf1f 100644 --- a/public-docs/browser-tools.md +++ b/public-docs/browser-tools.md @@ -20,12 +20,15 @@ Arguments marked `?` are optional. ## Tabs -| Tool | Arguments | Purpose | -| ------------------- | -------------------------- | ------------------------------------------------------------------------- | -| `browser_list_tabs` | — | List open tabs in the agent's workspace across connected hosts. | -| `browser_new_tab` | `url?` | Open a tab in the background and return its `browserId`. | -| `browser_close_tab` | `browserId` | Close a tab and clean up its webview. | -| `browser_resize` | `browserId, width, height` | Resize the tab's viewport — check a layout at phone or tablet dimensions. | +| Tool | Arguments | Purpose | +| -------------------- | -------------------------- | -------------------------------------------------------------------------- | +| `browser_list_hosts` | — | List connected browser hosts by name and ID, including hosts with no tabs. | +| `browser_list_tabs` | — | List workspace tabs and their browser hosts. | +| `browser_new_tab` | `url?, browserHostId?` | Open a background tab and return its `browserId` and browser host. | +| `browser_close_tab` | `browserId` | Close a tab and clean up its webview. | +| `browser_resize` | `browserId, width, height` | Resize the tab's viewport — check a layout at phone or tablet dimensions. | + +With one host that can open tabs, omit `browserHostId`. With several, pass an ID from `browser_list_hosts`. In Paseo Desktop, search for **Copy browser host name** in the Command Center. The result shows the same name the agent sees. Select it to copy the name. If the selected host is unavailable, list hosts again. ## Reading the page @@ -69,10 +72,11 @@ Arguments marked `?` are optional. Tools return structured errors rather than failing silently. The ones agents see most: -| Code | Meaning | -| ------------------- | ------------------------------------------------------------------------------- | -| `browser_disabled` | Browser tools are turned off on this host. | -| `browser_no_host` | No browser host (desktop app) is connected. Retryable. | -| `browser_stale_ref` | The ref no longer matches the page — take a new snapshot. | -| `browser_timeout` | The element never became actionable, or the wait condition never held. | -| `browser_denied` | The action isn't allowed — e.g. a non-`http(s)` URL, or no history to navigate. | +| Code | Meaning | +| ------------------------ | ------------------------------------------------------------------------------- | +| `browser_disabled` | Browser tools are turned off on this host. | +| `browser_no_host` | There is no connected browser host, or the selected host disconnected. | +| `browser_host_ambiguous` | Several hosts can open tabs. Choose one from the error and retry with its ID. | +| `browser_stale_ref` | The ref no longer matches the page — take a new snapshot. | +| `browser_timeout` | The element never became actionable, or the wait condition never held. | +| `browser_denied` | The action isn't allowed — e.g. a non-`http(s)` URL, or no history to navigate. | diff --git a/public-docs/browser-tools.zh-CN.md b/public-docs/browser-tools.zh-CN.md index 9b72216445..dc68a509dc 100644 --- a/public-docs/browser-tools.zh-CN.md +++ b/public-docs/browser-tools.zh-CN.md @@ -18,12 +18,15 @@ nav: 工具参考 ## 标签页 {#tabs} -| 工具 | 参数 | 用途 | -| ------------------- | -------------------------- | ------------------------------------------------- | -| `browser_list_tabs` | — | 列出 Agent 工作区在所有已连接宿主上打开的标签页。 | -| `browser_new_tab` | `url?` | 在后台打开一个标签页并返回它的 `browserId`。 | -| `browser_close_tab` | `browserId` | 关闭标签页并清理它的 webview。 | -| `browser_resize` | `browserId, width, height` | 调整标签页视口大小,用手机或平板的尺寸检查布局。 | +| 工具 | 参数 | 用途 | +| -------------------- | -------------------------- | -------------------------------------------------------------- | +| `browser_list_hosts` | — | 按名称和 ID 列出已连接的浏览器宿主,包括尚未打开标签页的宿主。 | +| `browser_list_tabs` | — | 列出工作区中的标签页及其浏览器宿主。 | +| `browser_new_tab` | `url?, browserHostId?` | 在后台打开标签页,并返回它的 `browserId` 和浏览器宿主。 | +| `browser_close_tab` | `browserId` | 关闭标签页并清理它的 webview。 | +| `browser_resize` | `browserId, width, height` | 调整标签页视口大小,用手机或平板的尺寸检查布局。 | + +如果只有一个宿主支持打开标签页,可以省略 `browserHostId`。如果有多个,请传入 `browser_list_hosts` 返回的 ID。在 Paseo Desktop 的命令中心搜索 **复制浏览器宿主名称**。结果会显示 Agent 看到的名称,选择该结果即可复制名称。如果选定的宿主不可用,请重新列出宿主。 ## 读取页面 {#reading-the-page} @@ -67,10 +70,11 @@ nav: 工具参考 工具会返回结构化的错误,而不是静默失败。Agent 最常遇到的错误: -| 错误码 | 含义 | -| ------------------- | --------------------------------------------------------------- | -| `browser_disabled` | 该主机上的浏览器工具已关闭。 | -| `browser_no_host` | 没有已连接的浏览器宿主(桌面应用)。可以重试。 | -| `browser_stale_ref` | 引用与页面不再匹配,请重新获取快照。 | -| `browser_timeout` | 元素始终无法操作,或等待条件始终未满足。 | -| `browser_denied` | 不允许该操作,例如非 `http(s)` 的 URL,或没有可导航的历史记录。 | +| 错误码 | 含义 | +| ------------------------ | ------------------------------------------------------------------------------ | +| `browser_disabled` | 该主机上的浏览器工具已关闭。 | +| `browser_no_host` | 没有已连接的浏览器宿主,或选定的宿主已断开连接。 | +| `browser_host_ambiguous` | 多个宿主支持打开标签页。请从错误返回的列表中选择一个宿主,并使用它的 ID 重试。 | +| `browser_stale_ref` | 引用与页面不再匹配,请重新获取快照。 | +| `browser_timeout` | 元素始终无法操作,或等待条件始终未满足。 | +| `browser_denied` | 不允许该操作,例如非 `http(s)` 的 URL,或没有可导航的历史记录。 |