mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-19 23:05:21 +00:00
Let agents choose which desktop opens a browser tab (#10189)
* fix(browser): let agents choose which desktop opens a tab Registration order sent new tabs to the wrong desktop window. Let the agent discover and select the intended window by name, and require a choice when several windows can open tabs. * test(desktop): run browser host selection in a fresh app session Closing the second window can leave the existing browser journey in a different renderer state. Give both journeys their own windows and daemon so the routing test does not change the old test setup. * fix(browser): keep window names visible and stable across reconnects Frameless desktop windows need an in-app place to see and copy their name. Give each window one browser host identity so a reconnect replaces a lingering connection without creating a duplicate choice. * test(desktop): cover host selection in the browser journey Open the second window after the existing browser checks finish so their setup remains isolated within one CI run.
This commit is contained in:
@@ -5,6 +5,7 @@ import { useTranslation } from "react-i18next";
|
||||
import {
|
||||
CalendarClock,
|
||||
CircleDashed,
|
||||
Copy,
|
||||
Folder,
|
||||
FolderPlus,
|
||||
History,
|
||||
@@ -17,6 +18,8 @@ import {
|
||||
} from "lucide-react-native";
|
||||
import { withUnistyles } from "react-native-unistyles";
|
||||
import { useIsCompactFormFactor } from "@/constants/layout";
|
||||
import { useToast } from "@/contexts/toast-context";
|
||||
import { getDesktopHost } from "@/desktop/host";
|
||||
import { useKeyboardShortcutOverrides } from "@/hooks/use-keyboard-shortcut-overrides";
|
||||
import { useOpenAddProject } from "@/hooks/use-open-add-project";
|
||||
import { useImportSession } from "@/hooks/use-import-session";
|
||||
@@ -27,6 +30,7 @@ import { useKeyboardShortcutsStore } from "@/stores/keyboard-shortcuts-store";
|
||||
import { usePanelStore } from "@/stores/panel-store";
|
||||
import { useSidebarViewStore } from "@/stores/sidebar-view-store";
|
||||
import { clearCommandCenterFocusRestoreElement } from "@/utils/command-center-focus-restore";
|
||||
import { copyToClipboard } from "@/utils/copy-to-clipboard";
|
||||
import {
|
||||
buildOpenProjectRoute,
|
||||
buildSchedulesRoute,
|
||||
@@ -35,6 +39,7 @@ import {
|
||||
} from "@/utils/host-routes";
|
||||
import { getShortcutOs } from "@/utils/shortcut-platform";
|
||||
import type { CommandCenterContribution, CommandCenterIconProps } from "./contributions";
|
||||
import { getCommandCenterIcon } from "./icon";
|
||||
import { useCommandCenterActions } from "./provider";
|
||||
import { buildGroupingContribution } from "./root-contributions";
|
||||
|
||||
@@ -63,6 +68,7 @@ const ThemedCircleDashed = withUnistyles(CircleDashed, (theme) => ({
|
||||
const ThemedPanelLeft = withUnistyles(PanelLeft, (theme) => ({
|
||||
color: theme.colors.foregroundMuted,
|
||||
}));
|
||||
const BrowserHostCopyIcon = getCommandCenterIcon(Copy);
|
||||
|
||||
function PlusIcon({ size }: CommandCenterIconProps) {
|
||||
return <ThemedPlus size={size} strokeWidth={2.4} />;
|
||||
@@ -111,6 +117,8 @@ function PanelLeftIcon({ size }: CommandCenterIconProps) {
|
||||
export function CommandCenterRootActions() {
|
||||
const keyboardActionDispatcher = useKeyboardActionDispatcher();
|
||||
const { t } = useTranslation();
|
||||
const toast = useToast();
|
||||
const browserHostLabel = getDesktopHost()?.browserHostLabel;
|
||||
const { overrides } = useKeyboardShortcutOverrides();
|
||||
const shortcutsAvailable = useHardwareKeyboardPresent();
|
||||
const openAddProject = useOpenAddProject();
|
||||
@@ -296,6 +304,32 @@ export function CommandCenterRootActions() {
|
||||
},
|
||||
];
|
||||
|
||||
if (browserHostLabel) {
|
||||
availableActions.push({
|
||||
id: "copy-browser-host-name",
|
||||
group: "actions",
|
||||
groupRank: 0,
|
||||
rank: 9,
|
||||
keywords: ["browser", "host", "window", browserHostLabel],
|
||||
visibility: "query",
|
||||
run: async () => {
|
||||
try {
|
||||
await copyToClipboard(browserHostLabel);
|
||||
toast.copied();
|
||||
} catch {
|
||||
toast.error(t("common.errors.unableToCopy"));
|
||||
}
|
||||
},
|
||||
presentation: {
|
||||
kind: "action",
|
||||
title: t("shell.commandCenter.copyBrowserHostName"),
|
||||
subtitle: browserHostLabel,
|
||||
sectionTitle: t("shell.commandCenter.actions"),
|
||||
icon: BrowserHostCopyIcon,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
if (shortcutsAvailable) {
|
||||
availableActions.push({
|
||||
id: "keyboard-shortcuts",
|
||||
@@ -332,6 +366,7 @@ export function CommandCenterRootActions() {
|
||||
|
||||
return availableActions;
|
||||
}, [
|
||||
browserHostLabel,
|
||||
groupMode,
|
||||
homeRoute,
|
||||
keyboardActionDispatcher,
|
||||
@@ -346,6 +381,7 @@ export function CommandCenterRootActions() {
|
||||
shortcutPlatform,
|
||||
shortcutsAvailable,
|
||||
t,
|
||||
toast,
|
||||
toggleAgentList,
|
||||
]);
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { DaemonClient } from "@getpaseo/client/internal/daemon-client";
|
||||
import type { SessionInboundMessage, SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import { getDesktopHost, type DesktopHostBridge } from "@/desktop/host";
|
||||
import {
|
||||
@@ -26,6 +27,7 @@ type BrowserAutomationExecuteResponse = Extract<
|
||||
type BrowserAutomationResponsePayload = BrowserAutomationExecuteResponse["payload"];
|
||||
type BrowserAutomationFailurePayload = Extract<BrowserAutomationResponsePayload, { ok: false }>;
|
||||
type BrowserAutomationErrorCode = BrowserAutomationFailurePayload["error"]["code"];
|
||||
type BrowserHostRegistration = Parameters<DaemonClient["registerBrowserHost"]>[0];
|
||||
|
||||
interface BrowserAutomationClient {
|
||||
on(
|
||||
@@ -69,14 +71,24 @@ export function mountBrowserAutomationHandler(
|
||||
};
|
||||
}
|
||||
|
||||
export function getDesktopBrowserHostRegistration(): BrowserHostRegistration | null {
|
||||
const desktop = getDesktopHost();
|
||||
if (typeof desktop?.browser?.executeAutomationCommand !== "function") return null;
|
||||
|
||||
return {
|
||||
hostKind: "desktop app",
|
||||
label: desktop.browserHostLabel,
|
||||
instanceId: desktop.browserHostInstanceId,
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
};
|
||||
}
|
||||
|
||||
export function mountBrowserAutomationDaemonClientHandler(
|
||||
client: import("@getpaseo/client/internal/daemon-client").DaemonClient,
|
||||
client: DaemonClient,
|
||||
registration: BrowserHostRegistration,
|
||||
options?: { serverId?: string },
|
||||
): () => void {
|
||||
const observation = client.registerBrowserHost({
|
||||
hostKind: "desktop app",
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
});
|
||||
const observation = client.registerBrowserHost(registration);
|
||||
const unmount = mountBrowserAutomationHandler({
|
||||
client: {
|
||||
on: (_type, handler) =>
|
||||
|
||||
@@ -175,6 +175,8 @@ export interface DesktopInvokeBridge {
|
||||
export interface DesktopHostBridge {
|
||||
platform?: string;
|
||||
windowChromeMode?: string;
|
||||
browserHostLabel?: string;
|
||||
browserHostInstanceId?: string;
|
||||
invoke?: DesktopInvokeBridge["invoke"];
|
||||
getPendingOpenProject?: () => Promise<string | null>;
|
||||
agentNavigation?: DesktopAgentNavigationBridge;
|
||||
|
||||
@@ -115,6 +115,7 @@ export const ar: TranslationResources = {
|
||||
home: "بيت",
|
||||
groupByProject: "تجميع حسب المشروع",
|
||||
groupByStatus: "تجميع حسب الحالة",
|
||||
copyBrowserHostName: "نسخ اسم مضيف المتصفح",
|
||||
modelGroupLabel: "النموذج",
|
||||
modelSearchKeywords: "تبديل النموذج تغيير النموذج تعيين النموذج اختيار النموذج",
|
||||
thinkingGroupLabel: "التفكير",
|
||||
|
||||
@@ -113,6 +113,7 @@ export const en = {
|
||||
home: "Home",
|
||||
groupByProject: "Group by project",
|
||||
groupByStatus: "Group by status",
|
||||
copyBrowserHostName: "Copy browser host name",
|
||||
modelGroupLabel: "Model",
|
||||
modelSearchKeywords: "switch model change model set model select model",
|
||||
thinkingGroupLabel: "Thinking",
|
||||
|
||||
@@ -116,6 +116,7 @@ export const es: TranslationResources = {
|
||||
home: "Hogar",
|
||||
groupByProject: "Agrupar por proyecto",
|
||||
groupByStatus: "Agrupar por estado",
|
||||
copyBrowserHostName: "Copiar el nombre del host del navegador",
|
||||
modelGroupLabel: "Modelo",
|
||||
modelSearchKeywords: "cambiar modelo modificar modelo establecer modelo seleccionar modelo",
|
||||
thinkingGroupLabel: "Razonamiento",
|
||||
|
||||
@@ -116,6 +116,7 @@ export const fr: TranslationResources = {
|
||||
home: "Accueil",
|
||||
groupByProject: "Grouper par projet",
|
||||
groupByStatus: "Grouper par statut",
|
||||
copyBrowserHostName: "Copier le nom de l’hôte du navigateur",
|
||||
modelGroupLabel: "Modèle",
|
||||
modelSearchKeywords:
|
||||
"changer de modèle modifier le modèle définir le modèle sélectionner le modèle model",
|
||||
|
||||
@@ -116,6 +116,7 @@ export const ja: TranslationResources = {
|
||||
home: "ホーム",
|
||||
groupByProject: "プロジェクトでグループ化",
|
||||
groupByStatus: "ステータスでグループ化",
|
||||
copyBrowserHostName: "ブラウザホスト名をコピー",
|
||||
modelGroupLabel: "モデル",
|
||||
modelSearchKeywords: "モデルを切り替え モデルを変更 モデルを設定 モデルを選択",
|
||||
thinkingGroupLabel: "思考",
|
||||
|
||||
@@ -115,6 +115,7 @@ export const ko: TranslationResources = {
|
||||
home: "홈",
|
||||
groupByProject: "프로젝트별 그룹화",
|
||||
groupByStatus: "상태별 그룹화",
|
||||
copyBrowserHostName: "브라우저 호스트 이름 복사",
|
||||
modelGroupLabel: "모델",
|
||||
modelSearchKeywords: "모델 전환 모델 변경 모델 설정 모델 선택",
|
||||
thinkingGroupLabel: "추론",
|
||||
|
||||
@@ -116,6 +116,7 @@ export const ptBR: TranslationResources = {
|
||||
home: "Início",
|
||||
groupByProject: "Agrupar por projeto",
|
||||
groupByStatus: "Agrupar por status",
|
||||
copyBrowserHostName: "Copiar o nome do host do navegador",
|
||||
modelGroupLabel: "Modelo",
|
||||
modelSearchKeywords: "trocar modelo mudar modelo definir modelo selecionar modelo",
|
||||
thinkingGroupLabel: "Raciocínio",
|
||||
|
||||
@@ -116,6 +116,7 @@ export const ru: TranslationResources = {
|
||||
home: "Главная",
|
||||
groupByProject: "Группировать по проекту",
|
||||
groupByStatus: "Группировать по статусу",
|
||||
copyBrowserHostName: "Скопировать имя хоста браузера",
|
||||
modelGroupLabel: "Модель",
|
||||
modelSearchKeywords: "Сменить, изменить, выбрать или установить модель",
|
||||
thinkingGroupLabel: "Мышление",
|
||||
|
||||
@@ -115,6 +115,7 @@ export const zhCN: TranslationResources = {
|
||||
home: "首页",
|
||||
groupByProject: "按项目分组",
|
||||
groupByStatus: "按状态分组",
|
||||
copyBrowserHostName: "复制浏览器宿主名称",
|
||||
modelGroupLabel: "模型",
|
||||
modelSearchKeywords: "切换模型 更改模型 设置模型 选择模型",
|
||||
thinkingGroupLabel: "思考",
|
||||
|
||||
@@ -55,10 +55,8 @@ import {
|
||||
buildDesktopDaemonTransportUrl,
|
||||
createDesktopDaemonTransportFactory,
|
||||
} from "@/desktop/daemon/desktop-daemon-transport";
|
||||
import { getDesktopHost } from "@/desktop/host";
|
||||
import { readDesktopManagedLocalCredential } from "@/desktop/daemon/local-credential";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import {
|
||||
useSessionStore,
|
||||
toDaemonServerInfo,
|
||||
@@ -74,7 +72,10 @@ import {
|
||||
invalidateServerDataQueriesAfterReconnect,
|
||||
mountServerDataPushRouter,
|
||||
} from "@/data/push-router";
|
||||
import { mountBrowserAutomationDaemonClientHandler } from "@/desktop/browser/automation/handler";
|
||||
import {
|
||||
getDesktopBrowserHostRegistration,
|
||||
mountBrowserAutomationDaemonClientHandler,
|
||||
} from "@/desktop/browser/automation/handler";
|
||||
import { schedulesQueryBaseKey } from "@/schedules/aggregated-schedules";
|
||||
import { dispatchComposerAgentMessage, sendQueuedComposerMessageNow } from "@/composer/actions";
|
||||
import { createMessageSubmissionWriter } from "@/composer/submission/writer";
|
||||
@@ -566,19 +567,10 @@ function probeIntervalForConnection(
|
||||
function createDefaultDeps(
|
||||
resolveOrigin: (serverId: string) => DaemonClient | null = () => null,
|
||||
): HostRuntimeControllerDeps {
|
||||
const browserHostAvailable =
|
||||
typeof getDesktopHost()?.browser?.executeAutomationCommand === "function";
|
||||
const browserAutomationCapabilities = browserHostAvailable
|
||||
? {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
const appCapabilities = {
|
||||
...browserAutomationCapabilities,
|
||||
};
|
||||
const browserHostRegistration = getDesktopBrowserHostRegistration();
|
||||
const appCapabilities = browserHostRegistration
|
||||
? { [CLIENT_CAPS.browserHost]: browserHostRegistration }
|
||||
: {};
|
||||
|
||||
return {
|
||||
createClient: ({ host, connection, clientId, runtimeGeneration }) => {
|
||||
@@ -702,12 +694,14 @@ function createDefaultDeps(
|
||||
queryClient,
|
||||
serverId: host.serverId,
|
||||
});
|
||||
if (!browserAutomationCapabilities) {
|
||||
if (!browserHostRegistration) {
|
||||
return unmountServerData;
|
||||
}
|
||||
const unmountBrowserAutomation = mountBrowserAutomationDaemonClientHandler(client, {
|
||||
serverId: host.serverId,
|
||||
});
|
||||
const unmountBrowserAutomation = mountBrowserAutomationDaemonClientHandler(
|
||||
client,
|
||||
browserHostRegistration,
|
||||
{ serverId: host.serverId },
|
||||
);
|
||||
return () => {
|
||||
unmountBrowserAutomation();
|
||||
unmountServerData();
|
||||
|
||||
@@ -338,6 +338,86 @@ async function callBrowserTool(client, name, args = {}) {
|
||||
return mcpPayload(await client.callTool({ name, args }), name);
|
||||
}
|
||||
|
||||
async function connectBrowserToolClient(daemonPort, callerAgentId) {
|
||||
const transport = new StreamableHTTPClientTransport(
|
||||
new URL(
|
||||
`http://127.0.0.1:${daemonPort}/mcp/agents?callerAgentId=${encodeURIComponent(callerAgentId)}`,
|
||||
),
|
||||
);
|
||||
return experimental_createMCPClient({ transport });
|
||||
}
|
||||
|
||||
async function openBrowserWorkspace(page, workspaceKey) {
|
||||
await page.getByTestId(`sidebar-workspace-row-${workspaceKey}`).click({ timeout: timeoutMs });
|
||||
await expect(page.getByTestId(`workspace-deck-entry-${workspaceKey}`)).toBeVisible();
|
||||
}
|
||||
|
||||
async function openSecondDesktopWindow(page) {
|
||||
const nextWindow = page.context().waitForEvent("page", { timeout: timeoutMs });
|
||||
await page.evaluate(() => window.paseoDesktop.window.openNew());
|
||||
return nextWindow;
|
||||
}
|
||||
|
||||
async function listBrowserHosts(client) {
|
||||
const reply = await client.callTool({ name: "browser_list_hosts", args: {} });
|
||||
assert(Array.isArray(reply.structuredContent?.browserHosts), JSON.stringify(reply));
|
||||
return reply.structuredContent.browserHosts;
|
||||
}
|
||||
|
||||
async function expectBrowserHostNameInCommandCenter(page, name) {
|
||||
await page.getByTestId("sidebar-search").click();
|
||||
const search = page.getByTestId("command-center-input");
|
||||
await search.fill("Copy browser host name");
|
||||
const action = page.getByRole("button", { name: /Copy browser host name/ });
|
||||
await expect(action).toContainText(name);
|
||||
await action.click();
|
||||
await expect(search).toBeHidden();
|
||||
await expect(page.getByText("Copied", { exact: true })).toBeVisible();
|
||||
}
|
||||
|
||||
async function verifyBrowserHostRouting({ page, client, targetUrl, workspaceKey }) {
|
||||
await openBrowserWorkspace(page, workspaceKey);
|
||||
await expect.poll(async () => (await listBrowserHosts(client)).length).toBe(1);
|
||||
const [firstHost] = await listBrowserHosts(client);
|
||||
await expectBrowserHostNameInCommandCenter(page, firstHost.label);
|
||||
const secondWindow = await openSecondDesktopWindow(page);
|
||||
|
||||
try {
|
||||
await openBrowserWorkspace(secondWindow, workspaceKey);
|
||||
await expect.poll(async () => (await listBrowserHosts(client)).length).toBe(2);
|
||||
const secondHost = (await listBrowserHosts(client)).find(
|
||||
(host) => host.browserHostId !== firstHost.browserHostId,
|
||||
);
|
||||
assert(secondHost?.label && secondHost.label !== firstHost.label, "Windows share a host label");
|
||||
await expectBrowserHostNameInCommandCenter(secondWindow, secondHost.label);
|
||||
|
||||
const ambiguous = await client.callTool({ name: "browser_new_tab", args: { url: targetUrl } });
|
||||
expect(ambiguous.structuredContent).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "browser_host_ambiguous", browserHosts: [firstHost, secondHost] },
|
||||
});
|
||||
expect((await callBrowserTool(client, "browser_list_tabs")).tabs).toEqual([]);
|
||||
|
||||
const { browserId, browserHost } = await callBrowserTool(client, "browser_new_tab", {
|
||||
url: targetUrl,
|
||||
browserHostId: firstHost.browserHostId,
|
||||
});
|
||||
expect(browserHost).toEqual(firstHost);
|
||||
// Only the tab's test ID carries the exact ID returned by the agent tool.
|
||||
await expect(page.getByTestId(`workspace-tab-browser_${browserId}`)).toBeVisible();
|
||||
await expect(secondWindow.getByTestId(`workspace-tab-browser_${browserId}`)).toHaveCount(0);
|
||||
expect((await callBrowserTool(client, "browser_list_tabs")).tabs).toMatchObject([
|
||||
{ browserId, browserHost: firstHost, isActive: false },
|
||||
]);
|
||||
await callBrowserTool(client, "browser_wait", { browserId, text: "Bridge target" });
|
||||
await callBrowserTool(client, "browser_close_tab", { browserId });
|
||||
} finally {
|
||||
await secondWindow.close();
|
||||
}
|
||||
|
||||
await expect.poll(() => listBrowserHosts(client)).toEqual([firstHost]);
|
||||
}
|
||||
|
||||
async function waitForGuestSelector(client, browserId) {
|
||||
const deadline = Date.now() + 5_000;
|
||||
while (Date.now() < deadline) {
|
||||
@@ -1413,12 +1493,7 @@ async function main() {
|
||||
workspaceId: workspaceIds.at(-1),
|
||||
agentId: await createCallerAgent(daemonPort, workspaceIds.at(-1)),
|
||||
};
|
||||
const transport = new StreamableHTTPClientTransport(
|
||||
new URL(
|
||||
`http://127.0.0.1:${daemonPort}/mcp/agents?callerAgentId=${encodeURIComponent(callerAgentId)}`,
|
||||
),
|
||||
);
|
||||
client = await experimental_createMCPClient({ transport });
|
||||
client = await connectBrowserToolClient(daemonPort, callerAgentId);
|
||||
const report = await runRegression({
|
||||
page,
|
||||
client,
|
||||
@@ -1430,11 +1505,23 @@ async function main() {
|
||||
artifactDir,
|
||||
});
|
||||
const pluginLinks = await checkPluginLinks();
|
||||
const routingClient = await connectBrowserToolClient(daemonPort, typingAgent.agentId);
|
||||
try {
|
||||
await verifyBrowserHostRouting({
|
||||
page,
|
||||
client: routingClient,
|
||||
targetUrl: target.url,
|
||||
workspaceKey: `${status.serverId}:${typingAgent.workspaceId}`,
|
||||
});
|
||||
} finally {
|
||||
await routingClient.close();
|
||||
}
|
||||
writeJson(path.join(artifactDir, "result.json"), {
|
||||
...report,
|
||||
settingsMemory,
|
||||
pluginLinks,
|
||||
adaptiveTheme,
|
||||
browserHostRouting: "passed",
|
||||
});
|
||||
console.log(
|
||||
`Browser desktop browser E2E passed: WebContents ${report.originalWebContentsId} remained ${report.finalWebContentsId}; viewport, inactive capture, focus continuity, list, snapshot, click, local-page selectors passed.`,
|
||||
|
||||
@@ -10,6 +10,8 @@ log.initialize({ spyRendererConsole: true });
|
||||
import { inheritLoginShellEnv } from "./login-shell-env.js";
|
||||
|
||||
import path from "node:path";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { hostname } from "node:os";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { existsSync } from "node:fs";
|
||||
import { execFileSync } from "node:child_process";
|
||||
@@ -668,6 +670,14 @@ function getWorkAreasPrimaryFirst(): Electron.Rectangle[] {
|
||||
return [primary, ...others].map((display) => display.workArea);
|
||||
}
|
||||
|
||||
const browserHostComputerName =
|
||||
hostname()
|
||||
.replace(/\.local$/i, "")
|
||||
.trim()
|
||||
.slice(0, 120) || "Desktop";
|
||||
const browserHostSessionCode = randomBytes(3).toString("hex").toUpperCase();
|
||||
let browserHostWindowNumber = 0;
|
||||
|
||||
async function createWindow(
|
||||
options: {
|
||||
initialRoute?: string | null;
|
||||
@@ -693,6 +703,7 @@ async function createWindow(
|
||||
: null;
|
||||
|
||||
const title = devWorktreeName ? `${APP_NAME} (${devWorktreeName})` : APP_NAME;
|
||||
const browserHostLabel = `${browserHostComputerName} · Window ${++browserHostWindowNumber} (${browserHostSessionCode})`;
|
||||
const mainWindow = new BrowserWindow({
|
||||
title,
|
||||
...resolveWindowBounds(restoredWindowState),
|
||||
@@ -704,7 +715,11 @@ async function createWindow(
|
||||
}),
|
||||
webPreferences: {
|
||||
preload: getPreloadPath(),
|
||||
additionalArguments: [windowChromeModeArgument(DESKTOP_WINDOW_CHROME_MODE)],
|
||||
additionalArguments: [
|
||||
windowChromeModeArgument(DESKTOP_WINDOW_CHROME_MODE),
|
||||
`--paseo-browser-host-label=${browserHostLabel}`,
|
||||
`--paseo-browser-host-instance-id=${randomUUID()}`,
|
||||
],
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
webviewTag: true,
|
||||
|
||||
@@ -12,6 +12,10 @@ const PASEO_BROWSER_PROFILE_PARTITION = "persist:paseo-browser";
|
||||
|
||||
type EventHandler = (payload: unknown) => void;
|
||||
|
||||
function readArgValue(prefix: string): string | undefined {
|
||||
return process.argv.find((argument) => argument.startsWith(prefix))?.slice(prefix.length);
|
||||
}
|
||||
|
||||
function readWindowChromeMode(): DesktopWindowChromeMode {
|
||||
const prefix = "--paseo-window-chrome-mode=";
|
||||
const value = process.argv.find((argument) => argument.startsWith(prefix))?.slice(prefix.length);
|
||||
@@ -32,6 +36,8 @@ interface AttachedBrowserRegistration {
|
||||
contextBridge.exposeInMainWorld("paseoDesktop", {
|
||||
platform: process.platform,
|
||||
windowChromeMode: readWindowChromeMode(),
|
||||
browserHostLabel: readArgValue("--paseo-browser-host-label="),
|
||||
browserHostInstanceId: readArgValue("--paseo-browser-host-instance-id="),
|
||||
invoke: (command: string, args?: Record<string, unknown>) =>
|
||||
ipcRenderer.invoke("paseo:invoke", command, args),
|
||||
getPendingOpenProject: () =>
|
||||
|
||||
@@ -31,6 +31,8 @@ export const BrowserAutomationHostCapabilitySchema = z
|
||||
return supportedCommands;
|
||||
}),
|
||||
hostKind: z.string().min(1).default("browser host"),
|
||||
label: z.string().min(1).max(160).optional(),
|
||||
instanceId: z.uuid().optional(),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
describe("browser automation protocol integration", () => {
|
||||
const browserId = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
test("browser host capability parses supported commands in hello", () => {
|
||||
test("browser host capability parses supported commands and a label in hello", () => {
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
@@ -24,6 +24,7 @@ describe("browser automation protocol integration", () => {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
hostKind: "desktop app",
|
||||
label: "MacBook Pro · Window 2",
|
||||
},
|
||||
},
|
||||
}).capabilities,
|
||||
@@ -31,6 +32,7 @@ describe("browser automation protocol integration", () => {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
hostKind: "desktop app",
|
||||
label: "MacBook Pro · Window 2",
|
||||
},
|
||||
});
|
||||
});
|
||||
@@ -133,6 +135,23 @@ describe("browser automation protocol integration", () => {
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
test("browser host registration accepts window identity and remains valid without it", () => {
|
||||
const request = {
|
||||
type: "browser.host.register.request",
|
||||
requestId: "register-browser",
|
||||
hostKind: "desktop app",
|
||||
supportedCommands: ["new_tab"],
|
||||
};
|
||||
const label = "MacBook Pro · Window 2";
|
||||
const instanceId = "33333333-3333-4333-8333-333333333333";
|
||||
expect(SessionInboundMessageSchema.parse(request)).toEqual(request);
|
||||
expect(SessionInboundMessageSchema.parse({ ...request, label, instanceId })).toEqual({
|
||||
...request,
|
||||
label,
|
||||
instanceId,
|
||||
});
|
||||
});
|
||||
|
||||
test("daemon to browser host execute request is an outbound session message", () => {
|
||||
const parsed = SessionOutboundMessageSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
|
||||
@@ -3493,6 +3493,8 @@ export const BrowserHostRegisterRequestSchema = z.object({
|
||||
type: z.literal("browser.host.register.request"),
|
||||
requestId: z.string(),
|
||||
hostKind: z.string().min(1),
|
||||
label: z.string().min(1).max(160).optional(),
|
||||
instanceId: z.uuid().optional(),
|
||||
supportedCommands: z.array(z.enum(BROWSER_AUTOMATION_COMMAND_NAMES)).min(1),
|
||||
});
|
||||
export const BrowserHostRegisterResponseSchema = z.object({
|
||||
|
||||
@@ -390,6 +390,7 @@ export const PASEO_TOOL_CATALOG = {
|
||||
},
|
||||
|
||||
// Browser
|
||||
browser_list_hosts: browser("List browser hosts", "Listed browser hosts"),
|
||||
browser_list_tabs: browser("List browser tabs", "Listed browser tabs"),
|
||||
browser_new_tab: browser("Open browser tab", "Opened browser tab", field("url")),
|
||||
browser_close_tab: browser("Close browser tab", "Closed browser tab", field("browserId")),
|
||||
|
||||
@@ -14,6 +14,8 @@ const SECOND_BROWSER_ID = "22222222-2222-4222-8222-222222222222";
|
||||
class FakeBrowserHostClient implements BrowserHostClient {
|
||||
public readonly receivedRequests: BrowserAutomationExecuteRequest[] = [];
|
||||
public readonly hostKind: string;
|
||||
public readonly label?: string;
|
||||
public readonly instanceId?: string;
|
||||
public readonly supportedCommands: readonly BrowserAutomationCommandName[];
|
||||
|
||||
public constructor(
|
||||
@@ -21,9 +23,13 @@ class FakeBrowserHostClient implements BrowserHostClient {
|
||||
options: {
|
||||
hostKind?: string;
|
||||
supportedCommands?: readonly BrowserAutomationCommandName[];
|
||||
label?: string;
|
||||
instanceId?: string;
|
||||
} = {},
|
||||
) {
|
||||
this.hostKind = options.hostKind ?? "desktop app";
|
||||
this.label = options.label;
|
||||
this.instanceId = options.instanceId;
|
||||
this.supportedCommands = options.supportedCommands ?? [...BROWSER_AUTOMATION_COMMAND_NAMES];
|
||||
}
|
||||
|
||||
@@ -170,6 +176,7 @@ describe("BrowserToolsBroker", () => {
|
||||
title: "Example",
|
||||
isActive: false,
|
||||
isLoading: false,
|
||||
browserHost: broker.listBrowserHosts()[0],
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -229,20 +236,31 @@ describe("BrowserToolsBroker", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("new tabs target the most recently registered host and tab commands stay with that host", async () => {
|
||||
test("new tabs require a choice with multiple hosts and tab commands stay with the chosen host", async () => {
|
||||
const broker = createBroker();
|
||||
const firstHost = new FakeBrowserHostClient("host-1");
|
||||
const firstHost = new FakeBrowserHostClient("host-1", { label: "MacBook Pro · Window 1" });
|
||||
const recentHost = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(firstHost);
|
||||
broker.registerClient(recentHost);
|
||||
|
||||
const newTabPromise = broker.execute({
|
||||
const input = {
|
||||
command: { command: "new_tab", args: { url: "https://example.com" } },
|
||||
workspaceId: "workspace-1",
|
||||
} as const;
|
||||
const [selectedHost, otherHost] = broker.listBrowserHosts();
|
||||
expect(selectedHost.label).toBe("MacBook Pro · Window 1");
|
||||
expect(otherHost.label).toBe("desktop app · Connection 2");
|
||||
await expect(broker.execute(input)).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "browser_host_ambiguous", browserHosts: [selectedHost, otherHost] },
|
||||
});
|
||||
|
||||
expect(firstHost.receivedRequests).toEqual([]);
|
||||
expect(recentHost.receivedRequests).toEqual([
|
||||
expect(recentHost.receivedRequests).toEqual([]);
|
||||
|
||||
const newTabPromise = broker.execute({ ...input, browserHostId: selectedHost.browserHostId });
|
||||
|
||||
expect(recentHost.receivedRequests).toEqual([]);
|
||||
expect(firstHost.receivedRequests).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
@@ -251,7 +269,7 @@ describe("BrowserToolsBroker", () => {
|
||||
},
|
||||
]);
|
||||
|
||||
recentHost.resolveLatestWith(broker, {
|
||||
firstHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
@@ -264,7 +282,7 @@ describe("BrowserToolsBroker", () => {
|
||||
|
||||
await expect(newTabPromise).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "new_tab", browserId: BROWSER_ID },
|
||||
result: { command: "new_tab", browserId: BROWSER_ID, browserHost: selectedHost },
|
||||
});
|
||||
|
||||
const snapshotPromise = broker.execute({
|
||||
@@ -272,15 +290,15 @@ describe("BrowserToolsBroker", () => {
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
|
||||
expect(firstHost.receivedRequests).toEqual([]);
|
||||
expect(recentHost.receivedRequests.at(-1)).toEqual({
|
||||
expect(recentHost.receivedRequests).toEqual([]);
|
||||
expect(firstHost.receivedRequests.at(-1)).toEqual({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
});
|
||||
|
||||
recentHost.resolveLatestWith(broker, {
|
||||
firstHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
@@ -302,6 +320,102 @@ describe("BrowserToolsBroker", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("a disconnected selection stays invalid and the sole eligible host is automatic", async () => {
|
||||
const broker = createBroker();
|
||||
const formerHost = new FakeBrowserHostClient("former", { label: "Mac Mini · Window 1" });
|
||||
const remainingHost = new FakeBrowserHostClient("remaining");
|
||||
const inventoryHost = new FakeBrowserHostClient("inventory", {
|
||||
supportedCommands: ["list_tabs"],
|
||||
});
|
||||
const unregister = broker.registerClient(formerHost);
|
||||
broker.registerClient(remainingHost);
|
||||
broker.registerClient(inventoryHost);
|
||||
const [previous, remaining, inventoryOnly] = broker.listBrowserHosts();
|
||||
expect(inventoryOnly.canOpenTabs).toBe(false);
|
||||
unregister();
|
||||
expect(broker.listBrowserHosts()).toEqual([remaining, inventoryOnly]);
|
||||
|
||||
const input = {
|
||||
command: { command: "new_tab", args: {} },
|
||||
workspaceId: "workspace-1",
|
||||
} as const;
|
||||
await expect(
|
||||
broker.execute({ ...input, browserHostId: previous.browserHostId }),
|
||||
).resolves.toMatchObject({ ok: false, error: { code: "browser_no_host" } });
|
||||
await expect(
|
||||
broker.execute({ ...input, browserHostId: inventoryOnly.browserHostId }),
|
||||
).resolves.toMatchObject({ ok: false, error: { code: "browser_unsupported" } });
|
||||
expect(remainingHost.receivedRequests).toEqual([]);
|
||||
expect(inventoryHost.receivedRequests).toEqual([]);
|
||||
|
||||
const creation = broker.execute(input);
|
||||
remainingHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
},
|
||||
});
|
||||
await expect(creation).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { browserHost: remaining },
|
||||
});
|
||||
expect(inventoryHost.receivedRequests).toEqual([]);
|
||||
|
||||
broker.registerClient(new FakeBrowserHostClient("reconnected", { label: previous.label }));
|
||||
const reconnected = broker.listBrowserHosts().find(({ label }) => label === previous.label);
|
||||
expect(reconnected).toBeDefined();
|
||||
expect(reconnected?.browserHostId).not.toBe(previous.browserHostId);
|
||||
await expect(
|
||||
broker.execute({ ...input, browserHostId: previous.browserHostId }),
|
||||
).resolves.toMatchObject({ ok: false, error: { code: "browser_no_host" } });
|
||||
expect(formerHost.receivedRequests).toEqual([]);
|
||||
});
|
||||
|
||||
test("a reconnect replaces the old connection for the same browser host", async () => {
|
||||
const broker = createBroker();
|
||||
const identity = {
|
||||
instanceId: "33333333-3333-4333-8333-333333333333",
|
||||
label: "Desktop · Window 1",
|
||||
};
|
||||
const oldConnection = new FakeBrowserHostClient("old", identity);
|
||||
const releaseOld = broker.registerClient(oldConnection);
|
||||
const [target] = broker.listBrowserHosts();
|
||||
const pending = broker.execute({ command: { command: "new_tab", args: {} } });
|
||||
|
||||
const newConnection = new FakeBrowserHostClient("new", identity);
|
||||
broker.registerClient(newConnection);
|
||||
expect(broker.listBrowserHosts()).toEqual([target]);
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "browser_no_host" },
|
||||
});
|
||||
releaseOld();
|
||||
|
||||
const creation = broker.execute({
|
||||
command: { command: "new_tab", args: {} },
|
||||
browserHostId: target.browserHostId,
|
||||
});
|
||||
newConnection.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
},
|
||||
});
|
||||
await expect(creation).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { browserHost: target },
|
||||
});
|
||||
expect(oldConnection.receivedRequests).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("list tabs aggregates all hosts and seeds browser id affinity", async () => {
|
||||
const broker = createBroker();
|
||||
const firstHost = new FakeBrowserHostClient("host-1");
|
||||
@@ -375,6 +489,7 @@ describe("BrowserToolsBroker", () => {
|
||||
title: "One",
|
||||
isActive: false,
|
||||
isLoading: false,
|
||||
browserHost: broker.listBrowserHosts()[0],
|
||||
},
|
||||
{
|
||||
browserId: SECOND_BROWSER_ID,
|
||||
@@ -383,6 +498,7 @@ describe("BrowserToolsBroker", () => {
|
||||
title: "Two",
|
||||
isActive: false,
|
||||
isLoading: false,
|
||||
browserHost: broker.listBrowserHosts()[1],
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -479,7 +595,11 @@ describe("BrowserToolsBroker", () => {
|
||||
broker.registerClient(other);
|
||||
broker.registerClient(owner);
|
||||
|
||||
const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } });
|
||||
const [, selectedHost] = broker.listBrowserHosts();
|
||||
const newTabPromise = broker.execute({
|
||||
command: { command: "new_tab", args: {} },
|
||||
browserHostId: selectedHost.browserHostId,
|
||||
});
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
@@ -521,7 +641,11 @@ describe("BrowserToolsBroker", () => {
|
||||
broker.registerClient(other);
|
||||
broker.registerClient(owner);
|
||||
|
||||
const newTabPromise = broker.execute({ command: { command: "new_tab", args: {} } });
|
||||
const [, selectedHost] = broker.listBrowserHosts();
|
||||
const newTabPromise = broker.execute({
|
||||
command: { command: "new_tab", args: {} },
|
||||
browserHostId: selectedHost.browserHostId,
|
||||
});
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createHmac, randomBytes, randomUUID } from "node:crypto";
|
||||
import {
|
||||
BrowserAutomationExecuteRequestSchema,
|
||||
BrowserAutomationExecuteResponseSchema,
|
||||
@@ -7,11 +7,17 @@ import {
|
||||
type BrowserAutomationExecuteRequest,
|
||||
type BrowserAutomationExecuteResponse,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { browserToolsFailure, type BrowserToolsResponsePayload } from "./errors.js";
|
||||
import {
|
||||
browserToolsFailure,
|
||||
type BrowserHostTarget,
|
||||
type BrowserToolsResponsePayload,
|
||||
} from "./errors.js";
|
||||
|
||||
export interface BrowserHostClient {
|
||||
id: string;
|
||||
hostKind: string;
|
||||
label?: string;
|
||||
instanceId?: string;
|
||||
supportedCommands: readonly BrowserAutomationCommandName[];
|
||||
sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void | Promise<void>;
|
||||
}
|
||||
@@ -23,10 +29,11 @@ export interface BrowserToolsExecuteInput {
|
||||
workspaceId?: string;
|
||||
requestId?: string;
|
||||
timeoutMs?: number;
|
||||
browserHostId?: string;
|
||||
}
|
||||
|
||||
interface PendingBrowserToolsRequest {
|
||||
clientId: string;
|
||||
host: RegisteredBrowserHost;
|
||||
rememberAffinity: boolean;
|
||||
timeout: ReturnType<typeof setTimeout>;
|
||||
resolve: (payload: BrowserToolsResponsePayload) => void;
|
||||
@@ -34,10 +41,15 @@ interface PendingBrowserToolsRequest {
|
||||
|
||||
interface RegisteredBrowserHost {
|
||||
client: BrowserHostClient;
|
||||
target: BrowserHostTarget;
|
||||
registeredAt: number;
|
||||
supportedCommands: ReadonlySet<BrowserAutomationCommandName>;
|
||||
}
|
||||
|
||||
type HostSelection =
|
||||
| { ok: true; value: RegisteredBrowserHost }
|
||||
| { ok: false; payload: BrowserToolsResponsePayload };
|
||||
|
||||
export interface BrowserToolsBrokerOptions {
|
||||
defaultTimeoutMs?: number;
|
||||
createRequestId?: () => string;
|
||||
@@ -51,7 +63,7 @@ export class BrowserToolsBroker {
|
||||
private readonly clients = new Map<string, RegisteredBrowserHost>();
|
||||
private readonly pending = new Map<string, PendingBrowserToolsRequest>();
|
||||
private readonly browserHostByBrowserId = new Map<string, string>();
|
||||
private readonly strandedBrowserHostByBrowserId = new Map<string, string>();
|
||||
private readonly hostIdKey = randomBytes(32);
|
||||
private registrationSequence = 0;
|
||||
|
||||
public constructor(options: BrowserToolsBrokerOptions) {
|
||||
@@ -60,35 +72,36 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
|
||||
public registerClient(client: BrowserHostClient): () => void {
|
||||
this.unregisterClient(client.id);
|
||||
// The discovery ID must not allow another connection to claim this window.
|
||||
const browserHostId = createHmac("sha256", this.hostIdKey)
|
||||
.update(client.instanceId ? `instance:${client.instanceId}` : `connection:${client.id}`)
|
||||
.digest("hex");
|
||||
this.unregisterHost(browserHostId);
|
||||
const registeredAt = ++this.registrationSequence;
|
||||
this.clients.set(client.id, {
|
||||
const label = client.label?.trim() || `${client.hostKind} · Connection ${registeredAt}`;
|
||||
const supportedCommands = new Set(client.supportedCommands);
|
||||
this.clients.set(browserHostId, {
|
||||
client,
|
||||
target: Object.freeze({
|
||||
browserHostId,
|
||||
label,
|
||||
canOpenTabs: supportedCommands.has("new_tab"),
|
||||
}),
|
||||
registeredAt,
|
||||
supportedCommands: new Set(client.supportedCommands),
|
||||
supportedCommands,
|
||||
});
|
||||
return () => this.unregisterClient(client.id, registeredAt);
|
||||
return () => this.unregisterHost(browserHostId, registeredAt);
|
||||
}
|
||||
|
||||
public unregisterClient(clientId: string, registeredAt?: number): void {
|
||||
const current = this.clients.get(clientId);
|
||||
private unregisterHost(browserHostId: string, registeredAt?: number): void {
|
||||
const current = this.clients.get(browserHostId);
|
||||
if (!current || (registeredAt !== undefined && current.registeredAt !== registeredAt)) {
|
||||
return;
|
||||
}
|
||||
this.clients.delete(clientId);
|
||||
|
||||
for (const [browserId, ownerClientId] of this.browserHostByBrowserId) {
|
||||
if (ownerClientId !== clientId) {
|
||||
continue;
|
||||
}
|
||||
this.browserHostByBrowserId.delete(browserId);
|
||||
this.strandedBrowserHostByBrowserId.set(browserId, clientId);
|
||||
}
|
||||
this.clients.delete(browserHostId);
|
||||
|
||||
for (const [requestId, pending] of this.pending) {
|
||||
if (pending.clientId !== clientId) {
|
||||
continue;
|
||||
}
|
||||
if (pending.host !== current) continue;
|
||||
this.pending.delete(requestId);
|
||||
clearTimeout(pending.timeout);
|
||||
pending.resolve(
|
||||
@@ -110,6 +123,10 @@ export class BrowserToolsBroker {
|
||||
return this.clients.size;
|
||||
}
|
||||
|
||||
public listBrowserHosts(): BrowserHostTarget[] {
|
||||
return Array.from(this.clients.values(), ({ target }) => target);
|
||||
}
|
||||
|
||||
public async execute(input: BrowserToolsExecuteInput): Promise<BrowserToolsResponsePayload> {
|
||||
const requestId = input.requestId ?? this.createRequestId();
|
||||
|
||||
@@ -137,7 +154,11 @@ export class BrowserToolsBroker {
|
||||
});
|
||||
}
|
||||
|
||||
const host = this.selectHostForCommand(request.data.command, requestId);
|
||||
const command = request.data.command;
|
||||
const host =
|
||||
command.command === "new_tab"
|
||||
? this.selectHostForNewTab(requestId, input.browserHostId)
|
||||
: this.selectHostForTab(command.args.browserId, requestId);
|
||||
if (!host.ok) {
|
||||
return host.payload;
|
||||
}
|
||||
@@ -170,7 +191,7 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
|
||||
const pending = this.pending.get(requestId);
|
||||
if (!pending || (clientIds && !clientIds.includes(pending.clientId))) {
|
||||
if (!pending || (clientIds && !clientIds.includes(pending.host.client.id))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -187,16 +208,16 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
|
||||
const pending = this.pending.get(parsed.data.payload.requestId);
|
||||
if (!pending || (clientIds && !clientIds.includes(pending.clientId))) {
|
||||
if (!pending || (clientIds && !clientIds.includes(pending.host.client.id))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
this.pending.delete(parsed.data.payload.requestId);
|
||||
clearTimeout(pending.timeout);
|
||||
if (pending.rememberAffinity) {
|
||||
this.rememberBrowserHostForPayload(pending.clientId, parsed.data.payload);
|
||||
if (pending.rememberAffinity && parsed.data.payload.ok) {
|
||||
this.rememberBrowserHostForPayload(pending.host.target.browserHostId, parsed.data.payload);
|
||||
}
|
||||
pending.resolve(parsed.data.payload);
|
||||
pending.resolve(attachBrowserHost(parsed.data.payload, pending.host.target));
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -249,7 +270,9 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
|
||||
for (const { host, payload } of hostResponses) {
|
||||
this.rememberBrowserHostForPayload(host.client.id, payload);
|
||||
if (payload.ok && this.clients.get(host.target.browserHostId) === host) {
|
||||
this.rememberBrowserHostForPayload(host.target.browserHostId, payload);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -264,63 +287,60 @@ export class BrowserToolsBroker {
|
||||
};
|
||||
}
|
||||
|
||||
private selectHostForCommand(
|
||||
command: BrowserAutomationCommand,
|
||||
requestId: string,
|
||||
):
|
||||
| { ok: true; value: RegisteredBrowserHost }
|
||||
| { ok: false; payload: BrowserToolsResponsePayload } {
|
||||
if (command.command === "new_tab") {
|
||||
const host = this.selectMostRecentlyRegisteredHost();
|
||||
return host
|
||||
? { ok: true, value: host }
|
||||
: { ok: false, payload: this.noBrowserHostFailure(requestId) };
|
||||
}
|
||||
|
||||
const browserId = getBrowserIdForCommand(command);
|
||||
if (!browserId) {
|
||||
const host = this.selectMostRecentlyRegisteredHost();
|
||||
return host
|
||||
? { ok: true, value: host }
|
||||
: { ok: false, payload: this.noBrowserHostFailure(requestId) };
|
||||
}
|
||||
|
||||
const ownerClientId = this.browserHostByBrowserId.get(browserId);
|
||||
if (ownerClientId) {
|
||||
const host = this.clients.get(ownerClientId);
|
||||
if (host) {
|
||||
return { ok: true, value: host };
|
||||
}
|
||||
private selectHostForNewTab(requestId: string, browserHostId: string | undefined): HostSelection {
|
||||
if (browserHostId !== undefined) {
|
||||
const host = this.clients.get(browserHostId);
|
||||
if (host) return { ok: true, value: host };
|
||||
return {
|
||||
ok: false,
|
||||
payload: this.strandedBrowserTabFailure({ requestId, browserId }),
|
||||
payload: browserToolsFailure({
|
||||
requestId,
|
||||
code: "browser_no_host",
|
||||
message:
|
||||
"The selected browser host is not connected. Call browser_list_hosts to choose a connected host.",
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
const strandedOwnerClientId = this.strandedBrowserHostByBrowserId.get(browserId);
|
||||
if (strandedOwnerClientId) {
|
||||
const reconnectedHost = this.clients.get(strandedOwnerClientId);
|
||||
if (reconnectedHost) {
|
||||
this.strandedBrowserHostByBrowserId.delete(browserId);
|
||||
this.browserHostByBrowserId.set(browserId, strandedOwnerClientId);
|
||||
return { ok: true, value: reconnectedHost };
|
||||
}
|
||||
if (this.clients.size === 0)
|
||||
return { ok: false, payload: this.noBrowserHostFailure(requestId) };
|
||||
const eligible = Array.from(this.clients.values()).filter(({ target }) => target.canOpenTabs);
|
||||
if (eligible.length === 1) return { ok: true, value: eligible[0] };
|
||||
if (eligible.length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
payload: this.strandedBrowserTabFailure({ requestId, browserId }),
|
||||
payload: browserToolsFailure({
|
||||
requestId,
|
||||
code: "browser_unsupported",
|
||||
message: "None of the connected browser hosts support opening tabs.",
|
||||
}),
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
payload: browserToolsFailure({
|
||||
requestId,
|
||||
code: "browser_host_ambiguous",
|
||||
message: "Choose a browser host and retry with its browserHostId.",
|
||||
browserHosts: eligible.map(({ target }) => target),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
if (this.clients.size === 1) {
|
||||
const host = this.selectMostRecentlyRegisteredHost();
|
||||
if (host) {
|
||||
return { ok: true, value: host };
|
||||
}
|
||||
private selectHostForTab(browserId: string, requestId: string): HostSelection {
|
||||
const browserHostId = this.browserHostByBrowserId.get(browserId);
|
||||
if (browserHostId) {
|
||||
const host = this.clients.get(browserHostId);
|
||||
return host
|
||||
? { ok: true, value: host }
|
||||
: { ok: false, payload: this.strandedBrowserTabFailure({ requestId, browserId }) };
|
||||
}
|
||||
|
||||
if (this.clients.size === 0) {
|
||||
const firstHost = this.clients.values().next().value;
|
||||
if (!firstHost) {
|
||||
return { ok: false, payload: this.noBrowserHostFailure(requestId) };
|
||||
}
|
||||
if (this.clients.size === 1) return { ok: true, value: firstHost };
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
@@ -332,14 +352,6 @@ export class BrowserToolsBroker {
|
||||
};
|
||||
}
|
||||
|
||||
private selectMostRecentlyRegisteredHost(): RegisteredBrowserHost | null {
|
||||
let selected: RegisteredBrowserHost | null = null;
|
||||
for (const host of this.clients.values()) {
|
||||
selected = host;
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
private unsupportedCommandFailure(params: {
|
||||
host: RegisteredBrowserHost;
|
||||
commandName: BrowserAutomationCommandName;
|
||||
@@ -377,30 +389,23 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
|
||||
private rememberBrowserHostForPayload(
|
||||
clientId: string,
|
||||
payload: BrowserToolsResponsePayload,
|
||||
browserHostId: string,
|
||||
payload: Extract<BrowserAutomationExecuteResponse["payload"], { ok: true }>,
|
||||
): void {
|
||||
if (!payload.ok) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.result.command === "list_tabs") {
|
||||
for (const tab of payload.result.tabs) {
|
||||
this.browserHostByBrowserId.set(tab.browserId, clientId);
|
||||
this.strandedBrowserHostByBrowserId.delete(tab.browserId);
|
||||
this.browserHostByBrowserId.set(tab.browserId, browserHostId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.result.command === "close_tab") {
|
||||
this.browserHostByBrowserId.delete(payload.result.browserId);
|
||||
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
|
||||
return;
|
||||
}
|
||||
|
||||
if ("browserId" in payload.result) {
|
||||
this.browserHostByBrowserId.set(payload.result.browserId, clientId);
|
||||
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
|
||||
this.browserHostByBrowserId.set(payload.result.browserId, browserHostId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -429,7 +434,7 @@ export class BrowserToolsBroker {
|
||||
}, timeoutMs);
|
||||
|
||||
this.pending.set(request.requestId, {
|
||||
clientId: client.id,
|
||||
host,
|
||||
rememberAffinity: params.rememberAffinity ?? true,
|
||||
timeout,
|
||||
resolve,
|
||||
@@ -458,11 +463,20 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
}
|
||||
|
||||
function getBrowserIdForCommand(command: BrowserAutomationCommand): string | null {
|
||||
if (command.command === "list_tabs" || command.command === "new_tab") {
|
||||
return null;
|
||||
function attachBrowserHost(
|
||||
payload: BrowserAutomationExecuteResponse["payload"],
|
||||
target: BrowserHostTarget,
|
||||
): BrowserToolsResponsePayload {
|
||||
if (!payload.ok) return payload;
|
||||
const result = payload.result;
|
||||
if (result.command === "new_tab") {
|
||||
return { ...payload, result: { ...result, browserHost: target } };
|
||||
}
|
||||
return command.args.browserId;
|
||||
if (result.command === "list_tabs") {
|
||||
const tabs = result.tabs.map((tab) => ({ ...tab, browserHost: target }));
|
||||
return { ...payload, result: { ...result, tabs } };
|
||||
}
|
||||
return { ...payload, result };
|
||||
}
|
||||
|
||||
function describeBrowserHost(host: RegisteredBrowserHost): string {
|
||||
|
||||
@@ -3,14 +3,40 @@ import type {
|
||||
BrowserAutomationExecuteResponse,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
|
||||
export type BrowserToolsResponsePayload = BrowserAutomationExecuteResponse["payload"];
|
||||
export type BrowserToolsErrorPayload = Extract<BrowserToolsResponsePayload, { ok: false }>["error"];
|
||||
export interface BrowserHostTarget {
|
||||
readonly browserHostId: string;
|
||||
readonly label: string;
|
||||
readonly canOpenTabs: boolean;
|
||||
}
|
||||
|
||||
type BrowserHostResponse = BrowserAutomationExecuteResponse["payload"];
|
||||
type BrowserHostSuccess = Extract<BrowserHostResponse, { ok: true }>;
|
||||
type BrowserHostResult = BrowserHostSuccess["result"];
|
||||
type NewTabResult = Extract<BrowserHostResult, { command: "new_tab" }>;
|
||||
type ListTabsResult = Extract<BrowserHostResult, { command: "list_tabs" }>;
|
||||
type BrowserHostFailure = Extract<BrowserHostResponse, { ok: false }>;
|
||||
|
||||
export type BrowserToolsErrorPayload = Omit<BrowserHostFailure["error"], "code"> & {
|
||||
code: BrowserAutomationErrorCode | "browser_host_ambiguous";
|
||||
browserHosts?: BrowserHostTarget[];
|
||||
};
|
||||
|
||||
export type BrowserToolsResponsePayload =
|
||||
| (Omit<BrowserHostSuccess, "result"> & {
|
||||
result:
|
||||
| Exclude<BrowserHostResult, NewTabResult | ListTabsResult>
|
||||
| (NewTabResult & { browserHost: BrowserHostTarget })
|
||||
| (Omit<ListTabsResult, "tabs"> & {
|
||||
tabs: (ListTabsResult["tabs"][number] & { browserHost: BrowserHostTarget })[];
|
||||
});
|
||||
})
|
||||
| (Omit<BrowserHostFailure, "error"> & { error: BrowserToolsErrorPayload });
|
||||
|
||||
export class BrowserToolsRequestError extends Error {
|
||||
public readonly code: BrowserAutomationErrorCode;
|
||||
public readonly retryable: boolean;
|
||||
|
||||
public constructor(error: BrowserToolsErrorPayload) {
|
||||
public constructor(error: BrowserHostFailure["error"]) {
|
||||
super(error.message);
|
||||
this.name = "BrowserToolsRequestError";
|
||||
this.code = error.code;
|
||||
@@ -20,9 +46,10 @@ export class BrowserToolsRequestError extends Error {
|
||||
|
||||
export function browserToolsFailure(params: {
|
||||
requestId: string;
|
||||
code: BrowserAutomationErrorCode;
|
||||
code: BrowserToolsErrorPayload["code"];
|
||||
message: string;
|
||||
retryable?: boolean;
|
||||
browserHosts?: BrowserHostTarget[];
|
||||
}): BrowserToolsResponsePayload {
|
||||
return {
|
||||
requestId: params.requestId,
|
||||
@@ -31,6 +58,7 @@ export function browserToolsFailure(params: {
|
||||
code: params.code,
|
||||
message: params.message,
|
||||
retryable: params.retryable ?? false,
|
||||
...(params.browserHosts ? { browserHosts: params.browserHosts } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
import { z } from "zod";
|
||||
import type { BrowserToolsBroker, BrowserToolsExecuteInput } from "./broker.js";
|
||||
import type { BrowserToolsExecuteInput } from "./broker.js";
|
||||
import type { BrowserToolsResponsePayload } from "./errors.js";
|
||||
import { registerBrowserTools, type RegisterBrowserToolsOptions } from "./tools.js";
|
||||
import type {
|
||||
@@ -10,6 +10,11 @@ import type {
|
||||
} from "../agent/tools/types.js";
|
||||
|
||||
const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
|
||||
const BROWSER_HOST = {
|
||||
browserHostId: "22222222-2222-4222-8222-222222222222",
|
||||
label: "MacBook Pro · Window 1",
|
||||
canOpenTabs: true,
|
||||
};
|
||||
const BROWSER_ID_MESSAGE =
|
||||
"browserId must be a real id returned by browser_new_tab or browser_list_tabs";
|
||||
const WAIT_CONDITION_MESSAGE = "browser_wait requires exactly one of text or url";
|
||||
@@ -27,6 +32,10 @@ class FakeBrowserBroker {
|
||||
|
||||
public constructor(private response: BrowserToolsResponsePayload = listTabsPayload()) {}
|
||||
|
||||
public listBrowserHosts() {
|
||||
return [BROWSER_HOST];
|
||||
}
|
||||
|
||||
public setResponse(response: BrowserToolsResponsePayload): void {
|
||||
this.response = response;
|
||||
}
|
||||
@@ -53,7 +62,7 @@ class BrowserToolHarness {
|
||||
registerTool: (name, config, handler) => {
|
||||
this.tools.set(name, { config, handler });
|
||||
},
|
||||
broker: this.broker as Pick<BrowserToolsBroker, "execute">,
|
||||
broker: this.broker,
|
||||
...(this.callerAgentId ? { callerAgentId: this.callerAgentId } : {}),
|
||||
resolveCallerAgent: () => this.callerAgent,
|
||||
});
|
||||
@@ -100,6 +109,7 @@ function listTabsPayload(): Extract<BrowserToolsResponsePayload, { ok: true }> {
|
||||
tabs: [
|
||||
{
|
||||
browserId: BROWSER_ID,
|
||||
browserHost: BROWSER_HOST,
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
isActive: true,
|
||||
@@ -117,6 +127,7 @@ function newTabPayload(): Extract<BrowserToolsResponsePayload, { ok: true }> {
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
browserHost: BROWSER_HOST,
|
||||
workspaceId: "wks_workspace_a",
|
||||
url: "https://example.com",
|
||||
},
|
||||
@@ -565,6 +576,7 @@ describe("registerBrowserTools", () => {
|
||||
const harness = new BrowserToolHarness();
|
||||
|
||||
expect(harness.toolNames()).toEqual([
|
||||
"browser_list_hosts",
|
||||
"browser_list_tabs",
|
||||
"browser_new_tab",
|
||||
"browser_snapshot",
|
||||
@@ -606,7 +618,7 @@ describe("registerBrowserTools", () => {
|
||||
expect(response.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: `Found 1 Paseo browser tab. Use these browserId values for tab-scoped browser tools.\n- browserId=${BROWSER_ID} active title="Example" url=https://example.com`,
|
||||
text: `Found 1 Paseo browser tab. Use these browserId values for tab-scoped browser tools.\n- browserId=${BROWSER_ID} active title="Example" url=https://example.com host="${BROWSER_HOST.label}"`,
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -628,11 +640,48 @@ describe("registerBrowserTools", () => {
|
||||
expect(response.content).toEqual([
|
||||
{
|
||||
type: "text",
|
||||
text: `Created browser tab browserId=${BROWSER_ID} url=https://example.com. Use this browserId for tab-scoped browser tools.`,
|
||||
text: `Created browser tab browserId=${BROWSER_ID} url=https://example.com on ${BROWSER_HOST.label}. Use this browserId for tab-scoped browser tools.`,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test("lists labeled browser hosts and passes the selected ID for new tabs", async () => {
|
||||
const harness = new BrowserToolHarness();
|
||||
const [host] = harness.broker.listBrowserHosts();
|
||||
const hosts = await harness.execute("browser_list_hosts", {});
|
||||
expect(hosts.structuredContent).toEqual({ browserHosts: [host] });
|
||||
expect(hosts.content).toEqual([
|
||||
{ type: "text", text: `Connected browser hosts\n- ${host.label} (${host.browserHostId})` },
|
||||
]);
|
||||
expect(harness.broker.calls).toEqual([]);
|
||||
|
||||
harness.broker.setResponse({
|
||||
requestId: "ambiguous",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_host_ambiguous",
|
||||
message: "Choose a browser host.",
|
||||
retryable: false,
|
||||
browserHosts: [host],
|
||||
},
|
||||
});
|
||||
const ambiguous = await harness.execute("browser_new_tab", {});
|
||||
expect(ambiguous.structuredContent).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "browser_host_ambiguous", browserHosts: [host] },
|
||||
});
|
||||
expect(ambiguous.content).toEqual([
|
||||
{ type: "text", text: `Choose a browser host.\n- ${host.label} (${host.browserHostId})` },
|
||||
]);
|
||||
|
||||
harness.broker.setResponse(newTabPayload());
|
||||
await harness.execute("browser_new_tab", { browserHostId: host.browserHostId });
|
||||
expect(harness.broker.calls.at(-1)).toMatchObject({
|
||||
browserHostId: host.browserHostId,
|
||||
command: { command: "new_tab", args: {} },
|
||||
});
|
||||
});
|
||||
|
||||
test.each([
|
||||
{
|
||||
name: "navigate accepts localhost without a scheme as http",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
import { BrowserAutomationBrowserIdSchema } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import type { BrowserToolsBroker } from "./broker.js";
|
||||
import type { BrowserToolsResponsePayload } from "./errors.js";
|
||||
import type { BrowserHostTarget, BrowserToolsResponsePayload } from "./errors.js";
|
||||
import type {
|
||||
PaseoToolConfig,
|
||||
PaseoToolExecutionContext,
|
||||
@@ -24,7 +24,7 @@ export interface RegisterBrowserToolsOptions {
|
||||
context: PaseoToolExecutionContext,
|
||||
) => Promise<PaseoToolResult>,
|
||||
) => void;
|
||||
broker: Pick<BrowserToolsBroker, "execute">;
|
||||
broker: Pick<BrowserToolsBroker, "execute" | "listBrowserHosts">;
|
||||
callerAgentId?: string;
|
||||
resolveCallerAgent: () => CallerAgentContext | null;
|
||||
}
|
||||
@@ -64,6 +64,24 @@ const BrowserWaitInputSchema = z
|
||||
});
|
||||
|
||||
export function registerBrowserTools(options: RegisterBrowserToolsOptions): void {
|
||||
options.registerTool(
|
||||
"browser_list_hosts",
|
||||
{
|
||||
title: "List browser hosts",
|
||||
description:
|
||||
"List connected Paseo browser hosts, including hosts with no tabs. Use a browserHostId from this list to choose where browser_new_tab opens.",
|
||||
inputSchema: {},
|
||||
},
|
||||
async () => {
|
||||
const browserHosts = options.broker.listBrowserHosts();
|
||||
const text =
|
||||
browserHosts.length > 0
|
||||
? ["Connected browser hosts", ...browserHosts.map(formatBrowserHost)].join("\n")
|
||||
: "No browser hosts are connected.";
|
||||
return { content: [{ type: "text", text }], structuredContent: { browserHosts } };
|
||||
},
|
||||
);
|
||||
|
||||
options.registerTool(
|
||||
"browser_list_tabs",
|
||||
{
|
||||
@@ -96,12 +114,20 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Create browser tab",
|
||||
description:
|
||||
"Create a new Paseo browser tab in this agent's workspace on the most recently connected browser automation host, opened in the background without switching the user's view. Pass an http(s) URL or a scheme-less host URL, which is treated as http; the returned browserId is used by tab-scoped tools.",
|
||||
"Open a background browser tab in this agent's workspace. Choose a browserHostId when multiple browser hosts can open tabs. Use the returned browserId for tab commands.",
|
||||
inputSchema: {
|
||||
url: BrowserHttpUrlInputSchema.optional(),
|
||||
url: BrowserHttpUrlInputSchema.optional().describe(
|
||||
"An HTTP or HTTPS URL. A URL without a scheme uses HTTP.",
|
||||
),
|
||||
browserHostId: z
|
||||
.string()
|
||||
.min(1)
|
||||
.max(128)
|
||||
.optional()
|
||||
.describe("Destination ID from browser_list_hosts."),
|
||||
},
|
||||
},
|
||||
async ({ url }) => {
|
||||
async ({ url, browserHostId }) => {
|
||||
const context = resolveBrowserToolContext(options);
|
||||
const missingWorkspace = requireWorkspaceContext(context);
|
||||
if (missingWorkspace) {
|
||||
@@ -111,6 +137,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
agentId: context.agentId,
|
||||
cwd: context.cwd,
|
||||
...(context.workspaceId ? { workspaceId: context.workspaceId } : {}),
|
||||
...(browserHostId ? { browserHostId } : {}),
|
||||
command: {
|
||||
command: "new_tab",
|
||||
args: url ? { url } : {},
|
||||
@@ -875,7 +902,7 @@ function summarizeBrowserSuccess(
|
||||
}
|
||||
const tabLines = payload.result.tabs.map((tab) => {
|
||||
const active = tab.isActive ? " active" : "";
|
||||
return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url}`;
|
||||
return `- browserId=${tab.browserId}${active} title=${JSON.stringify(tab.title || "Untitled")} url=${tab.url} host=${JSON.stringify(tab.browserHost.label)}`;
|
||||
});
|
||||
return withDialogs(
|
||||
[
|
||||
@@ -887,7 +914,7 @@ function summarizeBrowserSuccess(
|
||||
|
||||
if (payload.result.command === "new_tab") {
|
||||
return withDialogs(
|
||||
`Created browser tab browserId=${payload.result.browserId} url=${payload.result.url}. Use this browserId for tab-scoped browser tools.`,
|
||||
`Created browser tab browserId=${payload.result.browserId} url=${payload.result.url} on ${payload.result.browserHost.label}. Use this browserId for tab-scoped browser tools.`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1041,7 +1068,14 @@ function summarizeBrowserError(
|
||||
return "Browser tools are disabled. Enable browser tools on the host, then try again.";
|
||||
case "browser_stale_ref":
|
||||
return "That browser element reference is stale. Take a new browser snapshot and try again.";
|
||||
case "browser_host_ambiguous":
|
||||
return [error.message, ...(error.browserHosts ?? []).map(formatBrowserHost)].join("\n");
|
||||
default:
|
||||
return error.message;
|
||||
}
|
||||
}
|
||||
|
||||
function formatBrowserHost(host: BrowserHostTarget): string {
|
||||
const capability = host.canOpenTabs ? "" : " (cannot open tabs)";
|
||||
return `- ${host.label} (${host.browserHostId})${capability}`;
|
||||
}
|
||||
|
||||
@@ -1407,6 +1407,8 @@ export class Session {
|
||||
type: "browser.host.register.request" as const,
|
||||
requestId: uuidv4(),
|
||||
hostKind: host.data.hostKind,
|
||||
label: host.data.label,
|
||||
instanceId: host.data.instanceId,
|
||||
supportedCommands: host.data.supportedCommands,
|
||||
};
|
||||
void this.delivery
|
||||
@@ -2571,6 +2573,8 @@ export class Session {
|
||||
unregister = this.browserToolsBroker.registerClient({
|
||||
id: owner.id,
|
||||
hostKind: request.hostKind,
|
||||
label: request.label,
|
||||
instanceId: request.instanceId,
|
||||
supportedCommands: request.supportedCommands,
|
||||
sendBrowserAutomationRequest: (message) => owner.emit(message),
|
||||
});
|
||||
|
||||
@@ -20,12 +20,15 @@ Arguments marked `?` are optional.
|
||||
|
||||
## Tabs
|
||||
|
||||
| Tool | Arguments | Purpose |
|
||||
| ------------------- | -------------------------- | ------------------------------------------------------------------------- |
|
||||
| `browser_list_tabs` | — | List open tabs in the agent's workspace across connected hosts. |
|
||||
| `browser_new_tab` | `url?` | Open a tab in the background and return its `browserId`. |
|
||||
| `browser_close_tab` | `browserId` | Close a tab and clean up its webview. |
|
||||
| `browser_resize` | `browserId, width, height` | Resize the tab's viewport — check a layout at phone or tablet dimensions. |
|
||||
| Tool | Arguments | Purpose |
|
||||
| -------------------- | -------------------------- | -------------------------------------------------------------------------- |
|
||||
| `browser_list_hosts` | — | List connected browser hosts by name and ID, including hosts with no tabs. |
|
||||
| `browser_list_tabs` | — | List workspace tabs and their browser hosts. |
|
||||
| `browser_new_tab` | `url?, browserHostId?` | Open a background tab and return its `browserId` and browser host. |
|
||||
| `browser_close_tab` | `browserId` | Close a tab and clean up its webview. |
|
||||
| `browser_resize` | `browserId, width, height` | Resize the tab's viewport — check a layout at phone or tablet dimensions. |
|
||||
|
||||
With one host that can open tabs, omit `browserHostId`. With several, pass an ID from `browser_list_hosts`. In Paseo Desktop, search for **Copy browser host name** in the Command Center. The result shows the same name the agent sees. Select it to copy the name. If the selected host is unavailable, list hosts again.
|
||||
|
||||
## Reading the page
|
||||
|
||||
@@ -69,10 +72,11 @@ Arguments marked `?` are optional.
|
||||
|
||||
Tools return structured errors rather than failing silently. The ones agents see most:
|
||||
|
||||
| Code | Meaning |
|
||||
| ------------------- | ------------------------------------------------------------------------------- |
|
||||
| `browser_disabled` | Browser tools are turned off on this host. |
|
||||
| `browser_no_host` | No browser host (desktop app) is connected. Retryable. |
|
||||
| `browser_stale_ref` | The ref no longer matches the page — take a new snapshot. |
|
||||
| `browser_timeout` | The element never became actionable, or the wait condition never held. |
|
||||
| `browser_denied` | The action isn't allowed — e.g. a non-`http(s)` URL, or no history to navigate. |
|
||||
| Code | Meaning |
|
||||
| ------------------------ | ------------------------------------------------------------------------------- |
|
||||
| `browser_disabled` | Browser tools are turned off on this host. |
|
||||
| `browser_no_host` | There is no connected browser host, or the selected host disconnected. |
|
||||
| `browser_host_ambiguous` | Several hosts can open tabs. Choose one from the error and retry with its ID. |
|
||||
| `browser_stale_ref` | The ref no longer matches the page — take a new snapshot. |
|
||||
| `browser_timeout` | The element never became actionable, or the wait condition never held. |
|
||||
| `browser_denied` | The action isn't allowed — e.g. a non-`http(s)` URL, or no history to navigate. |
|
||||
|
||||
@@ -18,12 +18,15 @@ nav: 工具参考
|
||||
|
||||
## 标签页 {#tabs}
|
||||
|
||||
| 工具 | 参数 | 用途 |
|
||||
| ------------------- | -------------------------- | ------------------------------------------------- |
|
||||
| `browser_list_tabs` | — | 列出 Agent 工作区在所有已连接宿主上打开的标签页。 |
|
||||
| `browser_new_tab` | `url?` | 在后台打开一个标签页并返回它的 `browserId`。 |
|
||||
| `browser_close_tab` | `browserId` | 关闭标签页并清理它的 webview。 |
|
||||
| `browser_resize` | `browserId, width, height` | 调整标签页视口大小,用手机或平板的尺寸检查布局。 |
|
||||
| 工具 | 参数 | 用途 |
|
||||
| -------------------- | -------------------------- | -------------------------------------------------------------- |
|
||||
| `browser_list_hosts` | — | 按名称和 ID 列出已连接的浏览器宿主,包括尚未打开标签页的宿主。 |
|
||||
| `browser_list_tabs` | — | 列出工作区中的标签页及其浏览器宿主。 |
|
||||
| `browser_new_tab` | `url?, browserHostId?` | 在后台打开标签页,并返回它的 `browserId` 和浏览器宿主。 |
|
||||
| `browser_close_tab` | `browserId` | 关闭标签页并清理它的 webview。 |
|
||||
| `browser_resize` | `browserId, width, height` | 调整标签页视口大小,用手机或平板的尺寸检查布局。 |
|
||||
|
||||
如果只有一个宿主支持打开标签页,可以省略 `browserHostId`。如果有多个,请传入 `browser_list_hosts` 返回的 ID。在 Paseo Desktop 的命令中心搜索 **复制浏览器宿主名称**。结果会显示 Agent 看到的名称,选择该结果即可复制名称。如果选定的宿主不可用,请重新列出宿主。
|
||||
|
||||
## 读取页面 {#reading-the-page}
|
||||
|
||||
@@ -67,10 +70,11 @@ nav: 工具参考
|
||||
|
||||
工具会返回结构化的错误,而不是静默失败。Agent 最常遇到的错误:
|
||||
|
||||
| 错误码 | 含义 |
|
||||
| ------------------- | --------------------------------------------------------------- |
|
||||
| `browser_disabled` | 该主机上的浏览器工具已关闭。 |
|
||||
| `browser_no_host` | 没有已连接的浏览器宿主(桌面应用)。可以重试。 |
|
||||
| `browser_stale_ref` | 引用与页面不再匹配,请重新获取快照。 |
|
||||
| `browser_timeout` | 元素始终无法操作,或等待条件始终未满足。 |
|
||||
| `browser_denied` | 不允许该操作,例如非 `http(s)` 的 URL,或没有可导航的历史记录。 |
|
||||
| 错误码 | 含义 |
|
||||
| ------------------------ | ------------------------------------------------------------------------------ |
|
||||
| `browser_disabled` | 该主机上的浏览器工具已关闭。 |
|
||||
| `browser_no_host` | 没有已连接的浏览器宿主,或选定的宿主已断开连接。 |
|
||||
| `browser_host_ambiguous` | 多个宿主支持打开标签页。请从错误返回的列表中选择一个宿主,并使用它的 ID 重试。 |
|
||||
| `browser_stale_ref` | 引用与页面不再匹配,请重新获取快照。 |
|
||||
| `browser_timeout` | 元素始终无法操作,或等待条件始终未满足。 |
|
||||
| `browser_denied` | 不允许该操作,例如非 `http(s)` 的 URL,或没有可导航的历史记录。 |
|
||||
|
||||
Reference in New Issue
Block a user