mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Generalize browser automation hosting beyond the desktop app (#1878)
* feat(browser): generalize browser automation hosts Rename the browser automation client capability to a host-neutral payload, advertise supported commands, and route tab-scoped commands by browserId affinity so future browser hosts can register without another protocol shape change. * fix(browser): harden browser host registration Reject unusable browser host capabilities, clean up pending requests when a host registration is replaced, and defer list_tabs affinity updates until every host in the aggregation succeeds. * fix(browser): preserve host tab affinity across reconnects Keep stranded tab ownership tied to the disconnected host so the same host can reclaim open tabs after reconnect, and tolerate future command names in browser_host capability payloads while retaining only known commands.
This commit is contained in:
@@ -396,7 +396,7 @@ describe("mountBrowserAutomationHandler", () => {
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: "Desktop browser automation is not available in this app runtime.",
|
||||
message: "Browser automation is not available in this app runtime.",
|
||||
retryable: false,
|
||||
},
|
||||
},
|
||||
@@ -450,7 +450,7 @@ describe("mountBrowserAutomationHandler", () => {
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: "Desktop browser automation is not implemented by this desktop build yet.",
|
||||
message: "Browser automation is not implemented by this app build yet.",
|
||||
retryable: false,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -120,7 +120,7 @@ async function handleBrowserAutomationRequest(params: {
|
||||
payload: browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_unsupported",
|
||||
message: "Desktop browser automation is not available in this app runtime.",
|
||||
message: "Browser automation is not available in this app runtime.",
|
||||
}),
|
||||
});
|
||||
return;
|
||||
@@ -202,7 +202,7 @@ async function openBrowserTabForRequest(params: {
|
||||
return browserAutomationFailure({
|
||||
requestId: request.requestId,
|
||||
code: "browser_timeout",
|
||||
message: `Timed out waiting for browser tab ${browserId} to register with desktop automation. Try browser_new_tab again.`,
|
||||
message: `Timed out waiting for browser tab ${browserId} to register with the browser automation host. Try browser_new_tab again.`,
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
@@ -270,14 +270,14 @@ function normalizeThrownBridgeError(
|
||||
return browserAutomationFailure({
|
||||
requestId,
|
||||
code: "browser_unsupported",
|
||||
message: "Desktop browser automation is not implemented by this desktop build yet.",
|
||||
message: "Browser automation is not implemented by this app build yet.",
|
||||
});
|
||||
}
|
||||
|
||||
return browserAutomationFailure({
|
||||
requestId,
|
||||
code: "browser_unknown_error",
|
||||
message: message || "Desktop browser automation failed.",
|
||||
message: message || "Browser automation failed.",
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ import {
|
||||
} from "@/desktop/daemon/desktop-daemon-transport";
|
||||
import { getDesktopHost } from "@/desktop/host";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { replaceFetchedAgentDirectory } from "@/utils/agent-directory-sync";
|
||||
import { useSessionStore } from "@/stores/session-store";
|
||||
import {
|
||||
@@ -522,10 +523,15 @@ function probeIntervalForConnection(
|
||||
}
|
||||
|
||||
function createDefaultDeps(): HostRuntimeControllerDeps {
|
||||
const desktopBrowserAutomationAvailable =
|
||||
const browserHostAvailable =
|
||||
typeof getDesktopHost()?.browser?.executeAutomationCommand === "function";
|
||||
const browserAutomationCapabilities = desktopBrowserAutomationAvailable
|
||||
? { [CLIENT_CAPS.desktopBrowserAutomation]: true }
|
||||
const browserAutomationCapabilities = browserHostAvailable
|
||||
? {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { MutableDaemonConfig } from "@getpaseo/protocol/messages";
|
||||
|
||||
export const BROWSER_TOOLS_TITLE = "Browser tools";
|
||||
export const BROWSER_TOOLS_WARNING =
|
||||
"Allow agents to access and control Paseo desktop browser tabs, including logged-in browser state. Only enable this for agents you trust.";
|
||||
"Allow agents to access and control Paseo browser tabs, including logged-in browser state. Only enable this for agents you trust.";
|
||||
|
||||
export interface BrowserToolsCardState {
|
||||
isVisible: boolean;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { afterEach, expect, expectTypeOf, test, vi } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { DaemonClient, type DaemonTransport, type Logger } from "./daemon-client";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import {
|
||||
decodeFileTransferFrame,
|
||||
encodeFileTransferFrame,
|
||||
@@ -164,7 +165,7 @@ test("does not infer browser automation capabilities from Electron runtime", asy
|
||||
capabilities: z.record(z.unknown()),
|
||||
})
|
||||
.parse(JSON.parse(assertStr(mock.sent[0])));
|
||||
expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBeUndefined();
|
||||
expect(hello.capabilities[CLIENT_CAPS.browserHost]).toBeUndefined();
|
||||
});
|
||||
|
||||
test("advertises consumer-provided browser automation capabilities", async () => {
|
||||
@@ -174,7 +175,12 @@ test("advertises consumer-provided browser automation capabilities", async () =>
|
||||
clientId: "browser_capability_unit_test",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
|
||||
capabilities: {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
},
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
@@ -188,7 +194,10 @@ test("advertises consumer-provided browser automation capabilities", async () =>
|
||||
capabilities: z.record(z.unknown()),
|
||||
})
|
||||
.parse(JSON.parse(assertStr(mock.sent[0])));
|
||||
expect(hello.capabilities[CLIENT_CAPS.desktopBrowserAutomation]).toBe(true);
|
||||
expect(hello.capabilities[CLIENT_CAPS.browserHost]).toEqual({
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
hostKind: "desktop app",
|
||||
});
|
||||
});
|
||||
|
||||
const noopLogger: Logger = {
|
||||
@@ -506,7 +515,12 @@ test("advertises client capabilities in hello", async () => {
|
||||
logger,
|
||||
reconnect: { enabled: false },
|
||||
transportFactory: () => mock.transport,
|
||||
capabilities: { desktop_browser_automation: true },
|
||||
capabilities: {
|
||||
browser_host: {
|
||||
supportedCommands: ["list_tabs"],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
},
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
@@ -524,7 +538,10 @@ test("advertises client capabilities in hello", async () => {
|
||||
custom_mode_icons: true,
|
||||
reasoning_merge_enum: true,
|
||||
terminal_reflowable_snapshot: true,
|
||||
desktop_browser_automation: true,
|
||||
browser_host: {
|
||||
supportedCommands: ["list_tabs"],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -252,7 +252,7 @@ export interface DaemonClientConfig {
|
||||
};
|
||||
runtimeMetricsIntervalMs?: number;
|
||||
runtimeMetricsWindowMs?: number;
|
||||
capabilities?: Partial<Record<ClientCapability, boolean>>;
|
||||
capabilities?: Partial<Record<ClientCapability, unknown>>;
|
||||
}
|
||||
|
||||
export interface SendMessageOptions {
|
||||
|
||||
41
packages/protocol/src/browser-automation/capabilities.ts
Normal file
41
packages/protocol/src/browser-automation/capabilities.ts
Normal file
@@ -0,0 +1,41 @@
|
||||
import { z } from "zod";
|
||||
import {
|
||||
BROWSER_AUTOMATION_COMMAND_NAMES,
|
||||
type BrowserAutomationCommandName,
|
||||
} from "./rpc-schemas.js";
|
||||
|
||||
const KNOWN_BROWSER_AUTOMATION_COMMAND_NAMES = new Set<string>(BROWSER_AUTOMATION_COMMAND_NAMES);
|
||||
|
||||
export const BrowserAutomationHostCapabilitySchema = z
|
||||
.object({
|
||||
supportedCommands: z.array(z.string().min(1)).transform((commands, context) => {
|
||||
const supportedCommands: BrowserAutomationCommandName[] = [];
|
||||
const seen = new Set<BrowserAutomationCommandName>();
|
||||
|
||||
for (const command of commands) {
|
||||
if (!isKnownBrowserAutomationCommandName(command) || seen.has(command)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(command);
|
||||
supportedCommands.push(command);
|
||||
}
|
||||
|
||||
if (supportedCommands.length === 0) {
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "supportedCommands must include at least one known browser automation command",
|
||||
});
|
||||
return z.NEVER;
|
||||
}
|
||||
|
||||
return supportedCommands;
|
||||
}),
|
||||
hostKind: z.string().min(1).default("browser host"),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
export type BrowserAutomationHostCapability = z.infer<typeof BrowserAutomationHostCapabilitySchema>;
|
||||
|
||||
function isKnownBrowserAutomationCommandName(value: string): value is BrowserAutomationCommandName {
|
||||
return KNOWN_BROWSER_AUTOMATION_COMMAND_NAMES.has(value);
|
||||
}
|
||||
@@ -564,8 +564,8 @@ describe("browser automation execute RPC schemas", () => {
|
||||
requestId: "req-error",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_no_desktop",
|
||||
message: "No desktop browser automation client is connected.",
|
||||
code: "browser_no_host",
|
||||
message: "No browser automation host is connected.",
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -574,8 +574,8 @@ describe("browser automation execute RPC schemas", () => {
|
||||
requestId: "req-error",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_no_desktop",
|
||||
message: "No desktop browser automation client is connected.",
|
||||
code: "browser_no_host",
|
||||
message: "No browser automation host is connected.",
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -2,7 +2,7 @@ import { z } from "zod";
|
||||
|
||||
export const BrowserAutomationErrorCodeSchema = z.enum([
|
||||
"browser_disabled",
|
||||
"browser_no_desktop",
|
||||
"browser_no_host",
|
||||
"browser_tab_not_found",
|
||||
"browser_tab_closed",
|
||||
"browser_timeout",
|
||||
@@ -20,6 +20,29 @@ const BROWSER_AUTOMATION_BROWSER_ID_MESSAGE =
|
||||
const BROWSER_AUTOMATION_WAIT_CONDITION_MESSAGE =
|
||||
"browser_wait requires exactly one of text or url";
|
||||
|
||||
export const BROWSER_AUTOMATION_COMMAND_NAMES = [
|
||||
"list_tabs",
|
||||
"new_tab",
|
||||
"snapshot",
|
||||
"click",
|
||||
"fill",
|
||||
"wait",
|
||||
"type",
|
||||
"keypress",
|
||||
"navigate",
|
||||
"back",
|
||||
"forward",
|
||||
"reload",
|
||||
"screenshot",
|
||||
"upload",
|
||||
"select",
|
||||
"hover",
|
||||
"drag",
|
||||
"logs",
|
||||
] as const;
|
||||
|
||||
export const BrowserAutomationCommandNameSchema = z.enum(BROWSER_AUTOMATION_COMMAND_NAMES);
|
||||
|
||||
export const BrowserAutomationBrowserIdSchema = z
|
||||
.string({ error: () => BROWSER_AUTOMATION_BROWSER_ID_MESSAGE })
|
||||
.min(1, BROWSER_AUTOMATION_BROWSER_ID_MESSAGE)
|
||||
@@ -399,6 +422,7 @@ export const BrowserAutomationExecuteResponseSchema = z.object({
|
||||
});
|
||||
|
||||
export type BrowserAutomationErrorCode = z.infer<typeof BrowserAutomationErrorCodeSchema>;
|
||||
export type BrowserAutomationCommandName = z.infer<typeof BrowserAutomationCommandNameSchema>;
|
||||
export type BrowserAutomationCommand = z.infer<typeof BrowserAutomationCommandSchema>;
|
||||
export type BrowserAutomationResult = z.infer<typeof BrowserAutomationResultSchema>;
|
||||
export type BrowserAutomationConsoleLogEntry = z.infer<
|
||||
|
||||
@@ -11,7 +11,7 @@ export const CLIENT_CAPS = {
|
||||
// Old clients use a strict TerminalState schema and would reject the extra fields.
|
||||
// Drop the gate (always send the flags) when floor >= v0.1.88.
|
||||
terminalReflowableSnapshot: "terminal_reflowable_snapshot",
|
||||
desktopBrowserAutomation: "desktop_browser_automation",
|
||||
browserHost: "browser_host",
|
||||
} as const;
|
||||
|
||||
export type ClientCapability = (typeof CLIENT_CAPS)[keyof typeof CLIENT_CAPS];
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, test } from "vitest";
|
||||
|
||||
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "./browser-automation/rpc-schemas.js";
|
||||
import { CLIENT_CAPS } from "./client-capabilities.js";
|
||||
import {
|
||||
MutableDaemonConfigPatchSchema,
|
||||
@@ -12,7 +13,7 @@ import {
|
||||
describe("browser automation protocol integration", () => {
|
||||
const browserId = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
test("desktop automation capability parses in hello without narrowing old clients", () => {
|
||||
test("browser host capability parses supported commands in hello", () => {
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
@@ -20,13 +21,86 @@ describe("browser automation protocol integration", () => {
|
||||
clientType: "mobile",
|
||||
protocolVersion: 1,
|
||||
capabilities: {
|
||||
[CLIENT_CAPS.desktopBrowserAutomation]: true,
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
},
|
||||
}).capabilities,
|
||||
).toMatchObject({
|
||||
[CLIENT_CAPS.desktopBrowserAutomation]: true,
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [...BROWSER_AUTOMATION_COMMAND_NAMES],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("browser host capability requires at least one supported command", () => {
|
||||
expect(() =>
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
clientId: "client-1",
|
||||
clientType: "mobile",
|
||||
protocolVersion: 1,
|
||||
capabilities: {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toThrow();
|
||||
|
||||
expect(() =>
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
clientId: "client-2",
|
||||
clientType: "mobile",
|
||||
protocolVersion: 1,
|
||||
capabilities: {
|
||||
[CLIENT_CAPS.browserHost]: {},
|
||||
},
|
||||
}),
|
||||
).toThrow();
|
||||
|
||||
expect(() =>
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
clientId: "client-3",
|
||||
clientType: "mobile",
|
||||
protocolVersion: 1,
|
||||
capabilities: {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: ["future_command"],
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
test("browser host capability ignores unknown future commands when known commands remain", () => {
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
clientId: "client-1",
|
||||
clientType: "mobile",
|
||||
protocolVersion: 1,
|
||||
capabilities: {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: ["list_tabs", "future_command", "list_tabs"],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
},
|
||||
}).capabilities,
|
||||
).toMatchObject({
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: ["list_tabs"],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("hello remains valid when no browser host capability is advertised", () => {
|
||||
expect(
|
||||
WSHelloMessageSchema.parse({
|
||||
type: "hello",
|
||||
@@ -37,7 +111,7 @@ describe("browser automation protocol integration", () => {
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
test("daemon to desktop execute request is an outbound session message", () => {
|
||||
test("daemon to browser host execute request is an outbound session message", () => {
|
||||
const parsed = SessionOutboundMessageSchema.parse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
@@ -47,7 +121,7 @@ describe("browser automation protocol integration", () => {
|
||||
expect(parsed.type).toBe("browser.automation.execute.request");
|
||||
});
|
||||
|
||||
test("desktop to daemon execute response is an inbound session message", () => {
|
||||
test("browser host to daemon execute response is an inbound session message", () => {
|
||||
const parsed = SessionInboundMessageSchema.parse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
|
||||
@@ -57,6 +57,7 @@ import {
|
||||
BrowserAutomationExecuteRequestSchema,
|
||||
BrowserAutomationExecuteResponseSchema,
|
||||
} from "./browser-automation/rpc-schemas.js";
|
||||
import { BrowserAutomationHostCapabilitySchema } from "./browser-automation/capabilities.js";
|
||||
import {
|
||||
PaseoConfigRawSchema,
|
||||
PaseoLifecycleCommandRawSchema,
|
||||
@@ -4672,7 +4673,7 @@ export const WSHelloMessageSchema = z.object({
|
||||
[CLIENT_CAPS.reasoningMergeEnum]: z.boolean().optional(),
|
||||
[CLIENT_CAPS.customModeIcons]: z.boolean().optional(),
|
||||
[CLIENT_CAPS.terminalReflowableSnapshot]: z.boolean().optional(),
|
||||
[CLIENT_CAPS.desktopBrowserAutomation]: z.boolean().optional(),
|
||||
[CLIENT_CAPS.browserHost]: BrowserAutomationHostCapabilitySchema.optional(),
|
||||
})
|
||||
.passthrough()
|
||||
.optional(),
|
||||
|
||||
@@ -1,18 +1,32 @@
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
import type {
|
||||
BrowserAutomationCommand,
|
||||
BrowserAutomationCommandName,
|
||||
BrowserAutomationExecuteRequest,
|
||||
BrowserAutomationExecuteResponse,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { BrowserToolsBroker, type BrowserToolsDesktopClient } from "./broker.js";
|
||||
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { BrowserToolsBroker, type BrowserHostClient } from "./broker.js";
|
||||
import { StaticBrowserToolsPolicy } from "./policy.js";
|
||||
|
||||
const BROWSER_ID = "11111111-1111-4111-8111-111111111111";
|
||||
const SECOND_BROWSER_ID = "22222222-2222-4222-8222-222222222222";
|
||||
|
||||
class FakeDesktopClient implements BrowserToolsDesktopClient {
|
||||
class FakeBrowserHostClient implements BrowserHostClient {
|
||||
public readonly receivedRequests: BrowserAutomationExecuteRequest[] = [];
|
||||
public readonly hostKind: string;
|
||||
public readonly supportedCommands: readonly BrowserAutomationCommandName[];
|
||||
|
||||
public constructor(public readonly id: string) {}
|
||||
public constructor(
|
||||
public readonly id: string,
|
||||
options: {
|
||||
hostKind?: string;
|
||||
supportedCommands?: readonly BrowserAutomationCommandName[];
|
||||
} = {},
|
||||
) {
|
||||
this.hostKind = options.hostKind ?? "desktop app";
|
||||
this.supportedCommands = options.supportedCommands ?? [...BROWSER_AUTOMATION_COMMAND_NAMES];
|
||||
}
|
||||
|
||||
public sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void {
|
||||
this.receivedRequests.push(request);
|
||||
@@ -21,16 +35,30 @@ class FakeDesktopClient implements BrowserToolsDesktopClient {
|
||||
public resolveLatestWith(
|
||||
broker: BrowserToolsBroker,
|
||||
responsePayload: BrowserAutomationExecuteResponse["payload"],
|
||||
): boolean {
|
||||
const latest = this.receivedRequests.at(-1);
|
||||
if (!latest) {
|
||||
throw new Error(`Host ${this.id} has not received a browser request.`);
|
||||
}
|
||||
return this.resolveRequestWith(broker, latest, responsePayload);
|
||||
}
|
||||
|
||||
public resolveRequestWith(
|
||||
broker: BrowserToolsBroker,
|
||||
request: BrowserAutomationExecuteRequest,
|
||||
responsePayload: BrowserAutomationExecuteResponse["payload"],
|
||||
): boolean {
|
||||
return broker.receiveResponse({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: responsePayload,
|
||||
payload: { ...responsePayload, requestId: request.requestId },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class FailingDesktopClient implements BrowserToolsDesktopClient {
|
||||
public readonly id = "desktop-1";
|
||||
class FailingBrowserHostClient implements BrowserHostClient {
|
||||
public readonly id = "host-1";
|
||||
public readonly hostKind = "desktop app";
|
||||
public readonly supportedCommands = [...BROWSER_AUTOMATION_COMMAND_NAMES];
|
||||
|
||||
public sendBrowserAutomationRequest(): void {
|
||||
throw new Error("websocket send failed");
|
||||
@@ -68,23 +96,23 @@ describe("BrowserToolsBroker", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("no capable desktop returns browser_no_desktop", async () => {
|
||||
test("no connected browser host returns a retryable browser_no_host error", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
|
||||
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_no_desktop",
|
||||
message: "No desktop browser automation client is connected.",
|
||||
code: "browser_no_host",
|
||||
message: "No browser automation host is connected.",
|
||||
retryable: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("invalid browser requests return structured failures without contacting desktop", async () => {
|
||||
test("invalid browser requests return structured failures without contacting a host", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
const client = new FakeBrowserHostClient("host-1");
|
||||
broker.registerClient(client);
|
||||
|
||||
await expect(
|
||||
@@ -107,9 +135,9 @@ describe("BrowserToolsBroker", () => {
|
||||
expect(broker.getPendingRequestCount()).toBe(0);
|
||||
});
|
||||
|
||||
test("capable fake desktop receives request and returns response", async () => {
|
||||
test("capable browser host receives request and returns response", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
const client = new FakeBrowserHostClient("host-1");
|
||||
broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({
|
||||
@@ -165,9 +193,9 @@ describe("BrowserToolsBroker", () => {
|
||||
expect(broker.getPendingRequestCount()).toBe(0);
|
||||
});
|
||||
|
||||
test("desktop receives snapshot requests", async () => {
|
||||
test("single browser host receives snapshot requests", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
const client = new FakeBrowserHostClient("host-1");
|
||||
broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({
|
||||
@@ -211,10 +239,404 @@ describe("BrowserToolsBroker", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("new tabs target the most recently registered host and tab commands stay with that host", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const firstHost = new FakeBrowserHostClient("host-1");
|
||||
const recentHost = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(firstHost);
|
||||
broker.registerClient(recentHost);
|
||||
|
||||
const newTabPromise = broker.execute({
|
||||
command: { command: "new_tab", args: { url: "https://example.com" } },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
|
||||
expect(firstHost.receivedRequests).toEqual([]);
|
||||
expect(recentHost.receivedRequests).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: { command: "new_tab", args: { url: "https://example.com" } },
|
||||
},
|
||||
]);
|
||||
|
||||
recentHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
},
|
||||
});
|
||||
|
||||
await expect(newTabPromise).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "new_tab", browserId: BROWSER_ID },
|
||||
});
|
||||
|
||||
const snapshotPromise = broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
|
||||
expect(firstHost.receivedRequests).toEqual([]);
|
||||
expect(recentHost.receivedRequests.at(-1)).toEqual({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
});
|
||||
|
||||
recentHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "snapshot",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
},
|
||||
});
|
||||
|
||||
await expect(snapshotPromise).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "snapshot", browserId: BROWSER_ID },
|
||||
});
|
||||
});
|
||||
|
||||
test("list tabs aggregates all hosts and seeds browser id affinity", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const firstHost = new FakeBrowserHostClient("host-1");
|
||||
const secondHost = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(firstHost);
|
||||
broker.registerClient(secondHost);
|
||||
|
||||
const listPromise = broker.execute({
|
||||
command: { command: "list_tabs", args: {} },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
|
||||
expect(firstHost.receivedRequests).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1:host-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: { command: "list_tabs", args: {} },
|
||||
},
|
||||
]);
|
||||
expect(secondHost.receivedRequests).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1:host-2",
|
||||
workspaceId: "workspace-1",
|
||||
command: { command: "list_tabs", args: {} },
|
||||
},
|
||||
]);
|
||||
|
||||
firstHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1:host-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "list_tabs",
|
||||
tabs: [
|
||||
{
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
title: "One",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
secondHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1:host-2",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "list_tabs",
|
||||
tabs: [
|
||||
{
|
||||
browserId: SECOND_BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://two.example",
|
||||
title: "Two",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
await expect(listPromise).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "list_tabs",
|
||||
tabs: [
|
||||
{
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
title: "One",
|
||||
isActive: false,
|
||||
isLoading: false,
|
||||
},
|
||||
{
|
||||
browserId: SECOND_BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://two.example",
|
||||
title: "Two",
|
||||
isActive: false,
|
||||
isLoading: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const firstSnapshot = broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
const secondSnapshot = broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: SECOND_BROWSER_ID } },
|
||||
workspaceId: "workspace-1",
|
||||
requestId: "req-2",
|
||||
});
|
||||
|
||||
expect(firstHost.receivedRequests.at(-1)?.command).toEqual({
|
||||
command: "snapshot",
|
||||
args: { browserId: BROWSER_ID },
|
||||
});
|
||||
expect(secondHost.receivedRequests.at(-1)?.command).toEqual({
|
||||
command: "snapshot",
|
||||
args: { browserId: SECOND_BROWSER_ID },
|
||||
});
|
||||
|
||||
firstHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "snapshot",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
title: "One",
|
||||
elements: [],
|
||||
},
|
||||
});
|
||||
secondHost.resolveLatestWith(broker, {
|
||||
requestId: "req-2",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "snapshot",
|
||||
browserId: SECOND_BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://two.example",
|
||||
title: "Two",
|
||||
elements: [],
|
||||
},
|
||||
});
|
||||
|
||||
await expect(firstSnapshot).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "snapshot", browserId: BROWSER_ID },
|
||||
});
|
||||
await expect(secondSnapshot).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "snapshot", browserId: SECOND_BROWSER_ID },
|
||||
});
|
||||
});
|
||||
|
||||
test("failed list tabs aggregation does not seed browser id affinity", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const firstHost = new FakeBrowserHostClient("host-1");
|
||||
const secondHost = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(firstHost);
|
||||
broker.registerClient(secondHost);
|
||||
|
||||
const listPromise = broker.execute({
|
||||
command: { command: "list_tabs", args: {} },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
|
||||
firstHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1:host-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "list_tabs",
|
||||
tabs: [
|
||||
{
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://one.example",
|
||||
title: "One",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
secondHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1:host-2",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_timeout",
|
||||
message: "Host did not answer list_tabs.",
|
||||
retryable: true,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(listPromise).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_timeout",
|
||||
message: "Host did not answer list_tabs.",
|
||||
retryable: true,
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
workspaceId: "workspace-1",
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_tab_not_found",
|
||||
message: `Browser tab ${BROWSER_ID} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`,
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(firstHost.receivedRequests).toHaveLength(1);
|
||||
expect(secondHost.receivedRequests).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("unsupported commands are rejected before sending to the routed host", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const client = new FakeBrowserHostClient("host-1", {
|
||||
supportedCommands: ["list_tabs"],
|
||||
hostKind: "desktop app",
|
||||
});
|
||||
broker.registerClient(client);
|
||||
|
||||
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_unsupported",
|
||||
message: 'Browser automation command "snapshot" is not supported by the desktop app.',
|
||||
retryable: false,
|
||||
},
|
||||
});
|
||||
expect(client.receivedRequests).toEqual([]);
|
||||
});
|
||||
|
||||
test("unregistering a host strands its browser ids instead of routing them to another host", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const owner = new FakeBrowserHostClient("host-1");
|
||||
const unregisterOwner = broker.registerClient(owner);
|
||||
|
||||
const newTabPromise = broker.execute({
|
||||
command: { command: "new_tab", args: {} },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
},
|
||||
});
|
||||
await newTabPromise;
|
||||
|
||||
unregisterOwner();
|
||||
const replacement = new FakeBrowserHostClient("host-2");
|
||||
broker.registerClient(replacement);
|
||||
|
||||
await expect(
|
||||
broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
workspaceId: "workspace-1",
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_no_host",
|
||||
message: `The app hosting browser tab ${BROWSER_ID} disconnected.`,
|
||||
retryable: true,
|
||||
},
|
||||
});
|
||||
expect(replacement.receivedRequests).toEqual([]);
|
||||
});
|
||||
|
||||
test("reconnecting the same host reclaims its stranded browser ids", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const owner = new FakeBrowserHostClient("host-1");
|
||||
const unregisterOwner = broker.registerClient(owner);
|
||||
|
||||
const newTabPromise = broker.execute({
|
||||
command: { command: "new_tab", args: {} },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
owner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "new_tab",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
},
|
||||
});
|
||||
await newTabPromise;
|
||||
|
||||
unregisterOwner();
|
||||
const reconnectedOwner = new FakeBrowserHostClient("host-1");
|
||||
broker.registerClient(reconnectedOwner);
|
||||
|
||||
const snapshotPromise = broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
workspaceId: "workspace-1",
|
||||
});
|
||||
|
||||
expect(reconnectedOwner.receivedRequests).toEqual([
|
||||
{
|
||||
type: "browser.automation.execute.request",
|
||||
requestId: "req-1",
|
||||
workspaceId: "workspace-1",
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
},
|
||||
]);
|
||||
reconnectedOwner.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: {
|
||||
command: "snapshot",
|
||||
browserId: BROWSER_ID,
|
||||
workspaceId: "workspace-1",
|
||||
url: "https://example.com",
|
||||
title: "Example",
|
||||
elements: [],
|
||||
},
|
||||
});
|
||||
|
||||
await expect(snapshotPromise).resolves.toMatchObject({
|
||||
ok: true,
|
||||
result: { command: "snapshot", browserId: BROWSER_ID },
|
||||
});
|
||||
});
|
||||
|
||||
test("timeout resolves browser_timeout and clears pending state", async () => {
|
||||
vi.useFakeTimers();
|
||||
const broker = createBroker({ enabled: true, timeoutMs: 50 });
|
||||
broker.registerClient(new FakeDesktopClient("desktop-1"));
|
||||
broker.registerClient(new FakeBrowserHostClient("host-1"));
|
||||
|
||||
const resultPromise = broker.execute({ command: snapshotCommand() });
|
||||
expect(broker.getPendingRequestCount()).toBe(1);
|
||||
@@ -235,7 +657,7 @@ describe("BrowserToolsBroker", () => {
|
||||
|
||||
test("disconnect resolves retryable failure and clears pending request", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
const client = new FakeBrowserHostClient("host-1");
|
||||
const unregister = broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({ command: snapshotCommand() });
|
||||
@@ -247,17 +669,55 @@ describe("BrowserToolsBroker", () => {
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_no_desktop",
|
||||
message: "The desktop browser automation client disconnected before responding.",
|
||||
code: "browser_no_host",
|
||||
message: "The browser automation host disconnected before responding.",
|
||||
retryable: true,
|
||||
},
|
||||
});
|
||||
expect(broker.getPendingRequestCount()).toBe(0);
|
||||
});
|
||||
|
||||
test("desktop send failure resolves structured failure and clears pending request", async () => {
|
||||
test("replacing a host registration resolves pending requests from the old registration", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
broker.registerClient(new FailingDesktopClient());
|
||||
const oldHost = new FakeBrowserHostClient("host-1");
|
||||
const newHost = new FakeBrowserHostClient("host-1");
|
||||
broker.registerClient(oldHost);
|
||||
|
||||
const pendingResult = broker.execute({ command: snapshotCommand() });
|
||||
expect(oldHost.receivedRequests).toHaveLength(1);
|
||||
expect(broker.getPendingRequestCount()).toBe(1);
|
||||
|
||||
broker.registerClient(newHost);
|
||||
|
||||
await expect(pendingResult).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: false,
|
||||
error: {
|
||||
code: "browser_no_host",
|
||||
message: "The browser automation host disconnected before responding.",
|
||||
retryable: true,
|
||||
},
|
||||
});
|
||||
expect(broker.getPendingRequestCount()).toBe(0);
|
||||
|
||||
const listResult = broker.execute({ command: { command: "list_tabs", args: {} } });
|
||||
expect(newHost.receivedRequests).toHaveLength(1);
|
||||
newHost.resolveLatestWith(broker, {
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
});
|
||||
|
||||
await expect(listResult).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
ok: true,
|
||||
result: { command: "list_tabs", tabs: [] },
|
||||
});
|
||||
});
|
||||
|
||||
test("browser host send failure resolves structured failure and clears pending request", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
broker.registerClient(new FailingBrowserHostClient());
|
||||
|
||||
await expect(broker.execute({ command: snapshotCommand() })).resolves.toEqual({
|
||||
requestId: "req-1",
|
||||
@@ -273,7 +733,7 @@ describe("BrowserToolsBroker", () => {
|
||||
|
||||
test("explicit browser failure response propagates typed error", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
const client = new FakeBrowserHostClient("host-1");
|
||||
broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({ command: snapshotCommand() });
|
||||
@@ -302,7 +762,7 @@ describe("BrowserToolsBroker", () => {
|
||||
|
||||
test("invalid browser response resolves a structured failure and clears pending state", async () => {
|
||||
const broker = createBroker({ enabled: true });
|
||||
const client = new FakeDesktopClient("desktop-1");
|
||||
const client = new FakeBrowserHostClient("host-1");
|
||||
broker.registerClient(client);
|
||||
|
||||
const resultPromise = broker.execute({ command: snapshotCommand() });
|
||||
|
||||
@@ -3,14 +3,17 @@ import {
|
||||
BrowserAutomationExecuteRequestSchema,
|
||||
BrowserAutomationExecuteResponseSchema,
|
||||
type BrowserAutomationCommand,
|
||||
type BrowserAutomationCommandName,
|
||||
type BrowserAutomationExecuteRequest,
|
||||
type BrowserAutomationExecuteResponse,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { browserToolsFailure, type BrowserToolsResponsePayload } from "./errors.js";
|
||||
import type { BrowserToolsPolicy } from "./policy.js";
|
||||
|
||||
export interface BrowserToolsDesktopClient {
|
||||
export interface BrowserHostClient {
|
||||
id: string;
|
||||
hostKind: string;
|
||||
supportedCommands: readonly BrowserAutomationCommandName[];
|
||||
sendBrowserAutomationRequest(request: BrowserAutomationExecuteRequest): void | Promise<void>;
|
||||
}
|
||||
|
||||
@@ -25,10 +28,17 @@ export interface BrowserToolsExecuteInput {
|
||||
|
||||
interface PendingBrowserToolsRequest {
|
||||
clientId: string;
|
||||
rememberAffinity: boolean;
|
||||
timeout: ReturnType<typeof setTimeout>;
|
||||
resolve: (payload: BrowserToolsResponsePayload) => void;
|
||||
}
|
||||
|
||||
interface RegisteredBrowserHost {
|
||||
client: BrowserHostClient;
|
||||
registeredAt: number;
|
||||
supportedCommands: ReadonlySet<BrowserAutomationCommandName>;
|
||||
}
|
||||
|
||||
export interface BrowserToolsBrokerOptions {
|
||||
policy: BrowserToolsPolicy;
|
||||
defaultTimeoutMs?: number;
|
||||
@@ -41,8 +51,11 @@ export class BrowserToolsBroker {
|
||||
private readonly policy: BrowserToolsPolicy;
|
||||
private readonly defaultTimeoutMs: number;
|
||||
private readonly createRequestId: () => string;
|
||||
private readonly clients = new Map<string, BrowserToolsDesktopClient>();
|
||||
private readonly clients = new Map<string, RegisteredBrowserHost>();
|
||||
private readonly pending = new Map<string, PendingBrowserToolsRequest>();
|
||||
private readonly browserHostByBrowserId = new Map<string, string>();
|
||||
private readonly strandedBrowserHostByBrowserId = new Map<string, string>();
|
||||
private registrationSequence = 0;
|
||||
|
||||
public constructor(options: BrowserToolsBrokerOptions) {
|
||||
this.policy = options.policy;
|
||||
@@ -50,16 +63,31 @@ export class BrowserToolsBroker {
|
||||
this.createRequestId = options.createRequestId ?? (() => `browser_${randomUUID()}`);
|
||||
}
|
||||
|
||||
public registerClient(client: BrowserToolsDesktopClient): () => void {
|
||||
this.clients.set(client.id, client);
|
||||
return () => this.unregisterClient(client.id);
|
||||
public registerClient(client: BrowserHostClient): () => void {
|
||||
this.unregisterClient(client.id);
|
||||
const registeredAt = ++this.registrationSequence;
|
||||
this.clients.set(client.id, {
|
||||
client,
|
||||
registeredAt,
|
||||
supportedCommands: new Set(client.supportedCommands),
|
||||
});
|
||||
return () => this.unregisterClient(client.id, registeredAt);
|
||||
}
|
||||
|
||||
public unregisterClient(clientId: string): void {
|
||||
const deleted = this.clients.delete(clientId);
|
||||
if (!deleted) {
|
||||
public unregisterClient(clientId: string, registeredAt?: number): void {
|
||||
const current = this.clients.get(clientId);
|
||||
if (!current || (registeredAt !== undefined && current.registeredAt !== registeredAt)) {
|
||||
return;
|
||||
}
|
||||
this.clients.delete(clientId);
|
||||
|
||||
for (const [browserId, ownerClientId] of this.browserHostByBrowserId) {
|
||||
if (ownerClientId !== clientId) {
|
||||
continue;
|
||||
}
|
||||
this.browserHostByBrowserId.delete(browserId);
|
||||
this.strandedBrowserHostByBrowserId.set(browserId, clientId);
|
||||
}
|
||||
|
||||
for (const [requestId, pending] of this.pending) {
|
||||
if (pending.clientId !== clientId) {
|
||||
@@ -70,8 +98,8 @@ export class BrowserToolsBroker {
|
||||
pending.resolve(
|
||||
browserToolsFailure({
|
||||
requestId,
|
||||
code: "browser_no_desktop",
|
||||
message: "The desktop browser automation client disconnected before responding.",
|
||||
code: "browser_no_host",
|
||||
message: "The browser automation host disconnected before responding.",
|
||||
retryable: true,
|
||||
}),
|
||||
);
|
||||
@@ -97,16 +125,6 @@ export class BrowserToolsBroker {
|
||||
});
|
||||
}
|
||||
|
||||
const client = this.selectClient();
|
||||
if (!client) {
|
||||
return browserToolsFailure({
|
||||
requestId,
|
||||
code: "browser_no_desktop",
|
||||
message: "No desktop browser automation client is connected.",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
const request = BrowserAutomationExecuteRequestSchema.safeParse({
|
||||
type: "browser.automation.execute.request",
|
||||
requestId,
|
||||
@@ -124,8 +142,29 @@ export class BrowserToolsBroker {
|
||||
});
|
||||
}
|
||||
|
||||
if (request.data.command.command === "list_tabs") {
|
||||
return this.executeListTabs({
|
||||
request: request.data,
|
||||
timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
const host = this.selectHostForCommand(request.data.command, requestId);
|
||||
if (!host.ok) {
|
||||
return host.payload;
|
||||
}
|
||||
|
||||
const unsupported = this.unsupportedCommandFailure({
|
||||
host: host.value,
|
||||
commandName: request.data.command.command,
|
||||
requestId,
|
||||
});
|
||||
if (unsupported) {
|
||||
return unsupported;
|
||||
}
|
||||
|
||||
return this.sendRequest({
|
||||
client,
|
||||
host: host.value,
|
||||
request: request.data,
|
||||
timeoutMs: input.timeoutMs ?? this.defaultTimeoutMs,
|
||||
});
|
||||
@@ -163,23 +202,219 @@ export class BrowserToolsBroker {
|
||||
|
||||
this.pending.delete(parsed.data.payload.requestId);
|
||||
clearTimeout(pending.timeout);
|
||||
if (pending.rememberAffinity) {
|
||||
this.rememberBrowserHostForPayload(pending.clientId, parsed.data.payload);
|
||||
}
|
||||
pending.resolve(parsed.data.payload);
|
||||
return true;
|
||||
}
|
||||
|
||||
private selectClient(): BrowserToolsDesktopClient | null {
|
||||
for (const client of this.clients.values()) {
|
||||
return client;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private sendRequest(params: {
|
||||
client: BrowserToolsDesktopClient;
|
||||
private async executeListTabs(params: {
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
timeoutMs: number;
|
||||
}): Promise<BrowserToolsResponsePayload> {
|
||||
const { client, request, timeoutMs } = params;
|
||||
const hosts = Array.from(this.clients.values());
|
||||
if (hosts.length === 0) {
|
||||
return this.noBrowserHostFailure(params.request.requestId);
|
||||
}
|
||||
|
||||
for (const host of hosts) {
|
||||
const unsupported = this.unsupportedCommandFailure({
|
||||
host,
|
||||
commandName: "list_tabs",
|
||||
requestId: params.request.requestId,
|
||||
});
|
||||
if (unsupported) {
|
||||
return unsupported;
|
||||
}
|
||||
}
|
||||
|
||||
if (hosts.length === 1) {
|
||||
return this.sendRequest({
|
||||
host: hosts[0],
|
||||
request: params.request,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
}
|
||||
|
||||
const hostResponses = await Promise.all(
|
||||
hosts.map(async (host) => ({
|
||||
host,
|
||||
payload: await this.sendRequest({
|
||||
host,
|
||||
request: {
|
||||
...params.request,
|
||||
requestId: `${params.request.requestId}:${host.client.id}`,
|
||||
},
|
||||
rememberAffinity: false,
|
||||
timeoutMs: params.timeoutMs,
|
||||
}),
|
||||
})),
|
||||
);
|
||||
|
||||
const failed = hostResponses.find(({ payload }) => !payload.ok);
|
||||
if (failed) {
|
||||
return withBrowserToolsRequestId(failed.payload, params.request.requestId);
|
||||
}
|
||||
|
||||
for (const { host, payload } of hostResponses) {
|
||||
this.rememberBrowserHostForPayload(host.client.id, payload);
|
||||
}
|
||||
|
||||
return {
|
||||
requestId: params.request.requestId,
|
||||
ok: true,
|
||||
result: {
|
||||
command: "list_tabs",
|
||||
tabs: hostResponses.flatMap(({ payload }) =>
|
||||
payload.ok && payload.result.command === "list_tabs" ? payload.result.tabs : [],
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private selectHostForCommand(
|
||||
command: BrowserAutomationCommand,
|
||||
requestId: string,
|
||||
):
|
||||
| { ok: true; value: RegisteredBrowserHost }
|
||||
| { ok: false; payload: BrowserToolsResponsePayload } {
|
||||
if (command.command === "new_tab") {
|
||||
const host = this.selectMostRecentlyRegisteredHost();
|
||||
return host
|
||||
? { ok: true, value: host }
|
||||
: { ok: false, payload: this.noBrowserHostFailure(requestId) };
|
||||
}
|
||||
|
||||
const browserId = getBrowserIdForCommand(command);
|
||||
if (!browserId) {
|
||||
const host = this.selectMostRecentlyRegisteredHost();
|
||||
return host
|
||||
? { ok: true, value: host }
|
||||
: { ok: false, payload: this.noBrowserHostFailure(requestId) };
|
||||
}
|
||||
|
||||
const ownerClientId = this.browserHostByBrowserId.get(browserId);
|
||||
if (ownerClientId) {
|
||||
const host = this.clients.get(ownerClientId);
|
||||
if (host) {
|
||||
return { ok: true, value: host };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
payload: this.strandedBrowserTabFailure({ requestId, browserId }),
|
||||
};
|
||||
}
|
||||
|
||||
const strandedOwnerClientId = this.strandedBrowserHostByBrowserId.get(browserId);
|
||||
if (strandedOwnerClientId) {
|
||||
const reconnectedHost = this.clients.get(strandedOwnerClientId);
|
||||
if (reconnectedHost) {
|
||||
this.strandedBrowserHostByBrowserId.delete(browserId);
|
||||
this.browserHostByBrowserId.set(browserId, strandedOwnerClientId);
|
||||
return { ok: true, value: reconnectedHost };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
payload: this.strandedBrowserTabFailure({ requestId, browserId }),
|
||||
};
|
||||
}
|
||||
|
||||
if (this.clients.size === 1) {
|
||||
const host = this.selectMostRecentlyRegisteredHost();
|
||||
if (host) {
|
||||
return { ok: true, value: host };
|
||||
}
|
||||
}
|
||||
|
||||
if (this.clients.size === 0) {
|
||||
return { ok: false, payload: this.noBrowserHostFailure(requestId) };
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
payload: browserToolsFailure({
|
||||
requestId,
|
||||
code: "browser_tab_not_found",
|
||||
message: `Browser tab ${browserId} is not associated with a connected browser automation host. Call browser_list_tabs and use one of the returned browserId values.`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
private selectMostRecentlyRegisteredHost(): RegisteredBrowserHost | null {
|
||||
let selected: RegisteredBrowserHost | null = null;
|
||||
for (const host of this.clients.values()) {
|
||||
selected = host;
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
private unsupportedCommandFailure(params: {
|
||||
host: RegisteredBrowserHost;
|
||||
commandName: BrowserAutomationCommandName;
|
||||
requestId: string;
|
||||
}): BrowserToolsResponsePayload | null {
|
||||
if (params.host.supportedCommands.has(params.commandName)) {
|
||||
return null;
|
||||
}
|
||||
return browserToolsFailure({
|
||||
requestId: params.requestId,
|
||||
code: "browser_unsupported",
|
||||
message: `Browser automation command "${params.commandName}" is not supported by the ${describeBrowserHost(params.host)}.`,
|
||||
});
|
||||
}
|
||||
|
||||
private noBrowserHostFailure(requestId: string): BrowserToolsResponsePayload {
|
||||
return browserToolsFailure({
|
||||
requestId,
|
||||
code: "browser_no_host",
|
||||
message: "No browser automation host is connected.",
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
private strandedBrowserTabFailure(params: {
|
||||
requestId: string;
|
||||
browserId: string;
|
||||
}): BrowserToolsResponsePayload {
|
||||
return browserToolsFailure({
|
||||
requestId: params.requestId,
|
||||
code: "browser_no_host",
|
||||
message: `The app hosting browser tab ${params.browserId} disconnected.`,
|
||||
retryable: true,
|
||||
});
|
||||
}
|
||||
|
||||
private rememberBrowserHostForPayload(
|
||||
clientId: string,
|
||||
payload: BrowserToolsResponsePayload,
|
||||
): void {
|
||||
if (!payload.ok) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (payload.result.command === "list_tabs") {
|
||||
for (const tab of payload.result.tabs) {
|
||||
this.browserHostByBrowserId.set(tab.browserId, clientId);
|
||||
this.strandedBrowserHostByBrowserId.delete(tab.browserId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if ("browserId" in payload.result) {
|
||||
this.browserHostByBrowserId.set(payload.result.browserId, clientId);
|
||||
this.strandedBrowserHostByBrowserId.delete(payload.result.browserId);
|
||||
}
|
||||
}
|
||||
|
||||
private sendRequest(params: {
|
||||
host: RegisteredBrowserHost;
|
||||
request: BrowserAutomationExecuteRequest;
|
||||
rememberAffinity?: boolean;
|
||||
timeoutMs: number;
|
||||
}): Promise<BrowserToolsResponsePayload> {
|
||||
const { host, request, timeoutMs } = params;
|
||||
const client = host.client;
|
||||
|
||||
return new Promise<BrowserToolsResponsePayload>((resolve) => {
|
||||
const timeout = setTimeout(() => {
|
||||
@@ -198,6 +433,7 @@ export class BrowserToolsBroker {
|
||||
|
||||
this.pending.set(request.requestId, {
|
||||
clientId: client.id,
|
||||
rememberAffinity: params.rememberAffinity ?? true,
|
||||
timeout,
|
||||
resolve,
|
||||
});
|
||||
@@ -225,6 +461,43 @@ export class BrowserToolsBroker {
|
||||
}
|
||||
}
|
||||
|
||||
function getBrowserIdForCommand(command: BrowserAutomationCommand): string | null {
|
||||
switch (command.command) {
|
||||
case "list_tabs":
|
||||
case "new_tab":
|
||||
return null;
|
||||
case "snapshot":
|
||||
case "click":
|
||||
case "fill":
|
||||
case "wait":
|
||||
case "type":
|
||||
case "keypress":
|
||||
case "navigate":
|
||||
case "back":
|
||||
case "forward":
|
||||
case "reload":
|
||||
case "screenshot":
|
||||
case "upload":
|
||||
case "select":
|
||||
case "hover":
|
||||
case "drag":
|
||||
case "logs":
|
||||
return command.args.browserId;
|
||||
}
|
||||
}
|
||||
|
||||
function describeBrowserHost(host: RegisteredBrowserHost): string {
|
||||
const hostKind = host.client.hostKind.trim();
|
||||
return hostKind || "browser host";
|
||||
}
|
||||
|
||||
function withBrowserToolsRequestId(
|
||||
payload: BrowserToolsResponsePayload,
|
||||
requestId: string,
|
||||
): BrowserToolsResponsePayload {
|
||||
return { ...payload, requestId } as BrowserToolsResponsePayload;
|
||||
}
|
||||
|
||||
function resolveSendFailure(params: {
|
||||
requestId: string;
|
||||
pending: Map<string, PendingBrowserToolsRequest>;
|
||||
|
||||
@@ -370,7 +370,7 @@ const brokerErrorCases = [
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "Browser tools are disabled. Enable desktop browser tools on the host, then try again.",
|
||||
text: "Browser tools are disabled. Enable browser tools on the host, then try again.",
|
||||
},
|
||||
],
|
||||
context: { agentId: "agent-1", cwd: "/repo", workspaceId: "wks_workspace_a" },
|
||||
@@ -391,7 +391,7 @@ const brokerErrorCases = [
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "The browser did not respond before the timeout. Try again or check the desktop app.",
|
||||
text: "The browser did not respond before the timeout. Try again or check the browser host.",
|
||||
},
|
||||
],
|
||||
context: {
|
||||
|
||||
@@ -87,7 +87,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "List browser tabs",
|
||||
description:
|
||||
"List open Paseo desktop browser tabs for this agent's workspace. Use returned browserId values with tab-scoped tools.",
|
||||
"List open Paseo browser tabs for this agent's workspace across connected browser automation hosts. Use returned browserId values with tab-scoped tools.",
|
||||
inputSchema: {},
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
@@ -115,7 +115,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Create browser tab",
|
||||
description:
|
||||
"Create a new Paseo desktop browser tab in this agent's workspace, opened in the background without switching the user's view. Pass an http(s) URL or a scheme-less host URL, which is treated as http; the returned browserId is used by tab-scoped tools.",
|
||||
"Create a new Paseo browser tab in this agent's workspace on the most recently connected browser automation host, opened in the background without switching the user's view. Pass an http(s) URL or a scheme-less host URL, which is treated as http; the returned browserId is used by tab-scoped tools.",
|
||||
inputSchema: {
|
||||
url: BrowserHttpUrlInputSchema.optional(),
|
||||
},
|
||||
@@ -145,7 +145,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Snapshot browser page",
|
||||
description:
|
||||
"Return a model-readable snapshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
"Return a model-readable snapshot of a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
inputSchema: {
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
},
|
||||
@@ -174,7 +174,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Click browser element",
|
||||
description:
|
||||
"Click an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
"Click an element in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
inputSchema: {
|
||||
ref: BrowserRefInputSchema,
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
@@ -205,7 +205,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Fill browser element",
|
||||
description:
|
||||
"Fill an input-like element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
"Fill an input-like element in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
inputSchema: {
|
||||
ref: BrowserRefInputSchema,
|
||||
value: z.string(),
|
||||
@@ -238,7 +238,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Wait for browser condition",
|
||||
description:
|
||||
"Wait until a Paseo desktop browser tab contains text or reaches a URL fragment. Use browserId from browser_new_tab or browser_list_tabs; waits up to 5s by default on the desktop side.",
|
||||
"Wait until a Paseo browser tab contains text or reaches a URL fragment. Use browserId from browser_new_tab or browser_list_tabs; waits up to 5s by default on the browser host.",
|
||||
inputSchema: BrowserWaitInputSchema,
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
@@ -335,7 +335,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Navigate browser",
|
||||
description:
|
||||
"Navigate a Paseo desktop browser tab to a URL. Use browserId from browser_new_tab or browser_list_tabs; pass an http(s) URL or a scheme-less host URL, which is treated as http.",
|
||||
"Navigate a Paseo browser tab to a URL. Use browserId from browser_new_tab or browser_list_tabs; pass an http(s) URL or a scheme-less host URL, which is treated as http.",
|
||||
inputSchema: { url: BrowserHttpUrlInputSchema, browserId: BrowserAutomationBrowserIdSchema },
|
||||
outputSchema: BrowserToolOutputSchema,
|
||||
},
|
||||
@@ -364,21 +364,21 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
command: "back",
|
||||
title: "Browser back",
|
||||
description:
|
||||
"Go back in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
"Go back in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
},
|
||||
{
|
||||
name: "browser_forward",
|
||||
command: "forward",
|
||||
title: "Browser forward",
|
||||
description:
|
||||
"Go forward in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
"Go forward in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
},
|
||||
{
|
||||
name: "browser_reload",
|
||||
command: "reload",
|
||||
title: "Browser reload",
|
||||
description:
|
||||
"Reload a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
"Reload a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs.",
|
||||
},
|
||||
] as const) {
|
||||
options.registerTool(
|
||||
@@ -413,7 +413,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Capture browser screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs. Set fullPage to true to capture the full page.",
|
||||
"Capture a PNG screenshot of a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs. Set fullPage to true to capture the full page.",
|
||||
inputSchema: {
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
fullPage: z.boolean().default(false),
|
||||
@@ -444,7 +444,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Upload files in browser",
|
||||
description:
|
||||
"Set workspace files on a file input in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
"Set workspace files on a file input in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
inputSchema: {
|
||||
ref: BrowserRefInputSchema,
|
||||
filePaths: z.array(z.string().min(1)).min(1),
|
||||
@@ -478,7 +478,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
command: "hover",
|
||||
title: "Hover browser element",
|
||||
description:
|
||||
"Hover an element in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
"Hover an element in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
},
|
||||
] as const) {
|
||||
options.registerTool(
|
||||
@@ -514,7 +514,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Select browser option",
|
||||
description:
|
||||
"Set a select element in a Paseo desktop browser tab to a value. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
"Set a select element in a Paseo browser tab to a value. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
inputSchema: {
|
||||
ref: BrowserRefInputSchema,
|
||||
value: z.string(),
|
||||
@@ -547,7 +547,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Drag browser element",
|
||||
description:
|
||||
"Drag one element onto another in a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
"Drag one element onto another in a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; refs come from the latest browser_snapshot of the same tab and expire when the page changes.",
|
||||
inputSchema: {
|
||||
sourceRef: BrowserRefInputSchema,
|
||||
targetRef: BrowserRefInputSchema,
|
||||
@@ -580,7 +580,7 @@ export function registerBrowserTools(options: RegisterBrowserToolsOptions): void
|
||||
{
|
||||
title: "Read browser logs",
|
||||
description:
|
||||
"Read recent console messages and browser performance network entries for a Paseo desktop browser tab. Use browserId from browser_new_tab or browser_list_tabs; maxEntries defaults to 50.",
|
||||
"Read recent console messages and browser performance network entries for a Paseo browser tab. Use browserId from browser_new_tab or browser_list_tabs; maxEntries defaults to 50.",
|
||||
inputSchema: {
|
||||
maxEntries: z.number().int().positive().max(200).optional(),
|
||||
browserId: BrowserAutomationBrowserIdSchema,
|
||||
@@ -886,15 +886,15 @@ function summarizeBrowserError(
|
||||
): string {
|
||||
switch (error.code) {
|
||||
case "browser_disabled":
|
||||
return "Browser tools are disabled. Enable desktop browser tools on the host, then try again.";
|
||||
case "browser_no_desktop":
|
||||
return "No desktop browser automation client is connected. Open the Paseo desktop app and try again.";
|
||||
return "Browser tools are disabled. Enable browser tools on the host, then try again.";
|
||||
case "browser_no_host":
|
||||
return error.message;
|
||||
case "browser_timeout":
|
||||
return "The browser did not respond before the timeout. Try again or check the desktop app.";
|
||||
return "The browser did not respond before the timeout. Try again or check the browser host.";
|
||||
case "screenshot_no_frame":
|
||||
return error.message;
|
||||
case "browser_unsupported":
|
||||
return "This desktop build does not support that browser automation request yet.";
|
||||
return error.message;
|
||||
case "browser_stale_ref":
|
||||
return "That browser element reference is stale. Take a new browser snapshot and try again.";
|
||||
default:
|
||||
|
||||
@@ -2,9 +2,11 @@ import { createServer, type Server as HTTPServer } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
|
||||
import type {
|
||||
BrowserAutomationCommandName,
|
||||
BrowserAutomationExecuteRequest,
|
||||
BrowserAutomationExecuteResponse,
|
||||
} from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import type pino from "pino";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
@@ -26,18 +28,18 @@ import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
|
||||
|
||||
interface BrowserToolsDaemonHarness {
|
||||
broker: BrowserToolsBroker;
|
||||
connectDesktopBrowserClient(
|
||||
options?: ConnectDesktopBrowserClientOptions,
|
||||
): Promise<DesktopBrowserClientHandle>;
|
||||
connectBrowserHostClient(
|
||||
options?: ConnectBrowserHostClientOptions,
|
||||
): Promise<BrowserHostClientHandle>;
|
||||
stop(): Promise<void>;
|
||||
}
|
||||
|
||||
interface ConnectDesktopBrowserClientOptions {
|
||||
interface ConnectBrowserHostClientOptions {
|
||||
clientId?: string;
|
||||
capabilities?: Partial<Record<string, boolean>>;
|
||||
capabilities?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
interface DesktopBrowserClientHandle {
|
||||
interface BrowserHostClientHandle {
|
||||
clientId: string;
|
||||
nextBrowserRequest(): Promise<BrowserAutomationExecuteRequest>;
|
||||
respondToBrowserRequest(response: BrowserAutomationExecuteResponse): void;
|
||||
@@ -57,15 +59,26 @@ afterEach(async () => {
|
||||
await Promise.all(harnesses.splice(0).map((harness) => harness.stop()));
|
||||
});
|
||||
|
||||
function browserHostCapabilities(
|
||||
supportedCommands: readonly BrowserAutomationCommandName[] = BROWSER_AUTOMATION_COMMAND_NAMES,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
[CLIENT_CAPS.browserHost]: {
|
||||
supportedCommands: [...supportedCommands],
|
||||
hostKind: "desktop app",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("WebSocketServer browser tools wiring", () => {
|
||||
it("registers capable clients and dispatches broker requests over the real WebSocket path", async () => {
|
||||
const harness = await startBrowserToolsDaemonHarness();
|
||||
const desktop = await harness.connectDesktopBrowserClient();
|
||||
const browserHost = await harness.connectBrowserHostClient();
|
||||
|
||||
const resultPromise = harness.broker.execute({
|
||||
command: { command: "list_tabs", args: {} },
|
||||
});
|
||||
const request = await desktop.nextBrowserRequest();
|
||||
const request = await browserHost.nextBrowserRequest();
|
||||
|
||||
expect(request).toMatchObject({
|
||||
type: "browser.automation.execute.request",
|
||||
@@ -73,7 +86,7 @@ describe("WebSocketServer browser tools wiring", () => {
|
||||
command: { command: "list_tabs", args: {} },
|
||||
});
|
||||
|
||||
desktop.respondToBrowserRequest({
|
||||
browserHost.respondToBrowserRequest({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: request.requestId,
|
||||
@@ -91,20 +104,20 @@ describe("WebSocketServer browser tools wiring", () => {
|
||||
|
||||
it("unregisters capable clients on disconnect and clears pending browser commands", async () => {
|
||||
const harness = await startBrowserToolsDaemonHarness();
|
||||
const desktop = await harness.connectDesktopBrowserClient();
|
||||
const browserHost = await harness.connectBrowserHostClient();
|
||||
|
||||
const pendingResult = harness.broker.execute({
|
||||
command: { command: "list_tabs", args: {} },
|
||||
});
|
||||
const pendingExpectation = expect(pendingResult).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "browser_no_desktop", retryable: true },
|
||||
error: { code: "browser_no_host", retryable: true },
|
||||
});
|
||||
await desktop.nextBrowserRequest();
|
||||
await browserHost.nextBrowserRequest();
|
||||
|
||||
expect(harness.broker.getPendingRequestCount()).toBe(1);
|
||||
|
||||
await desktop.disconnect();
|
||||
await browserHost.disconnect();
|
||||
|
||||
expect(harness.broker.getRegisteredClientCount()).toBe(0);
|
||||
expect(harness.broker.getPendingRequestCount()).toBe(0);
|
||||
@@ -114,28 +127,28 @@ describe("WebSocketServer browser tools wiring", () => {
|
||||
harness.broker.execute({ command: { command: "list_tabs", args: {} } }),
|
||||
).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "browser_no_desktop" },
|
||||
error: { code: "browser_no_host" },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps browser automation registered when a desktop browser client resumes", async () => {
|
||||
it("keeps browser automation registered when a browser host client resumes", async () => {
|
||||
const harness = await startBrowserToolsDaemonHarness();
|
||||
const clientId = "desktop-client-1";
|
||||
await harness.connectDesktopBrowserClient({
|
||||
const clientId = "browser-host-client-1";
|
||||
await harness.connectBrowserHostClient({
|
||||
clientId,
|
||||
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
|
||||
capabilities: browserHostCapabilities(),
|
||||
});
|
||||
|
||||
const resumedDesktop = await harness.connectDesktopBrowserClient({
|
||||
const resumedBrowserHost = await harness.connectBrowserHostClient({
|
||||
clientId,
|
||||
capabilities: { [CLIENT_CAPS.desktopBrowserAutomation]: true },
|
||||
capabilities: browserHostCapabilities(),
|
||||
});
|
||||
|
||||
const resultPromise = harness.broker.execute({
|
||||
command: { command: "click", args: { browserId: BROWSER_ID, ref: "@e1" } },
|
||||
});
|
||||
const request = await resumedDesktop.nextBrowserRequest();
|
||||
resumedDesktop.respondToBrowserRequest({
|
||||
const request = await resumedBrowserHost.nextBrowserRequest();
|
||||
resumedBrowserHost.respondToBrowserRequest({
|
||||
type: "browser.automation.execute.response",
|
||||
payload: {
|
||||
requestId: request.requestId,
|
||||
@@ -149,6 +162,33 @@ describe("WebSocketServer browser tools wiring", () => {
|
||||
result: { command: "click", browserId: BROWSER_ID, ref: "@e1" },
|
||||
});
|
||||
});
|
||||
|
||||
it("clears pending browser commands when a browser host changes capabilities", async () => {
|
||||
const harness = await startBrowserToolsDaemonHarness();
|
||||
const clientId = "browser-host-client-1";
|
||||
const browserHost = await harness.connectBrowserHostClient({
|
||||
clientId,
|
||||
capabilities: browserHostCapabilities(),
|
||||
});
|
||||
|
||||
const pendingResult = harness.broker.execute({
|
||||
command: { command: "snapshot", args: { browserId: BROWSER_ID } },
|
||||
});
|
||||
await browserHost.nextBrowserRequest();
|
||||
expect(harness.broker.getPendingRequestCount()).toBe(1);
|
||||
|
||||
await harness.connectBrowserHostClient({
|
||||
clientId,
|
||||
capabilities: browserHostCapabilities(["list_tabs"]),
|
||||
});
|
||||
|
||||
await expect(pendingResult).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "browser_no_host", retryable: true },
|
||||
});
|
||||
expect(harness.broker.getRegisteredClientCount()).toBe(1);
|
||||
expect(harness.broker.getPendingRequestCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
async function startBrowserToolsDaemonHarness(): Promise<BrowserToolsDaemonHarness> {
|
||||
@@ -162,7 +202,7 @@ async function startBrowserToolsDaemonHarness(): Promise<BrowserToolsDaemonHarne
|
||||
|
||||
const harness: BrowserToolsDaemonHarness = {
|
||||
broker,
|
||||
async connectDesktopBrowserClient(options = {}) {
|
||||
async connectBrowserHostClient(options = {}) {
|
||||
const clientId = options.clientId;
|
||||
const client = new DaemonClient({
|
||||
url,
|
||||
@@ -170,7 +210,7 @@ async function startBrowserToolsDaemonHarness(): Promise<BrowserToolsDaemonHarne
|
||||
clientType: "browser",
|
||||
connectTimeoutMs: 500,
|
||||
reconnect: { enabled: false },
|
||||
capabilities: options.capabilities ?? { [CLIENT_CAPS.desktopBrowserAutomation]: true },
|
||||
capabilities: options.capabilities ?? browserHostCapabilities(),
|
||||
});
|
||||
clients.add(client);
|
||||
|
||||
|
||||
@@ -71,6 +71,10 @@ import {
|
||||
} from "./lifecycle-reasons.js";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import type { BrowserAutomationExecuteResponse } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
import {
|
||||
BrowserAutomationHostCapabilitySchema,
|
||||
type BrowserAutomationHostCapability,
|
||||
} from "@getpaseo/protocol/browser-automation/capabilities";
|
||||
import type { BrowserToolsBroker } from "./browser-tools/broker.js";
|
||||
|
||||
const WS_CLOSE_DAEMON_AUTH_FAILED = 4401;
|
||||
@@ -307,10 +311,13 @@ function bufferFromWsData(data: Buffer | ArrayBuffer | Buffer[] | string): Buffe
|
||||
return Buffer.from(data);
|
||||
}
|
||||
|
||||
function hasDesktopBrowserAutomationCapability(
|
||||
function getBrowserHostCapability(
|
||||
capabilities: Record<string, unknown> | null,
|
||||
): boolean {
|
||||
return capabilities?.[CLIENT_CAPS.desktopBrowserAutomation] === true;
|
||||
): BrowserAutomationHostCapability | null {
|
||||
const parsed = BrowserAutomationHostCapabilitySchema.safeParse(
|
||||
capabilities?.[CLIENT_CAPS.browserHost],
|
||||
);
|
||||
return parsed.success ? parsed.data : null;
|
||||
}
|
||||
|
||||
interface WebSocketLike {
|
||||
@@ -333,6 +340,7 @@ interface SessionConnection {
|
||||
}
|
||||
|
||||
interface BrowserToolsRegistration {
|
||||
capabilitySignature: string;
|
||||
unregister: () => void;
|
||||
}
|
||||
|
||||
@@ -1412,22 +1420,31 @@ export class VoiceAssistantWebSocketServer {
|
||||
if (!this.browserToolsBroker) {
|
||||
return;
|
||||
}
|
||||
if (!hasDesktopBrowserAutomationCapability(connection.clientCapabilities)) {
|
||||
const browserHostCapability = getBrowserHostCapability(connection.clientCapabilities);
|
||||
if (!browserHostCapability) {
|
||||
this.unregisterBrowserToolsClient(connection.clientId);
|
||||
return;
|
||||
}
|
||||
const capabilitySignature = JSON.stringify(browserHostCapability);
|
||||
const existing = this.browserToolsRegistrations.get(connection.clientId);
|
||||
if (existing) {
|
||||
if (existing?.capabilitySignature === capabilitySignature) {
|
||||
return;
|
||||
}
|
||||
if (existing) {
|
||||
this.browserToolsRegistrations.delete(connection.clientId);
|
||||
existing.unregister();
|
||||
}
|
||||
|
||||
const unregister = this.browserToolsBroker.registerClient({
|
||||
id: connection.clientId,
|
||||
hostKind: browserHostCapability.hostKind,
|
||||
supportedCommands: browserHostCapability.supportedCommands,
|
||||
sendBrowserAutomationRequest: (request) => {
|
||||
this.sendToConnection(connection, wrapSessionMessage(request));
|
||||
},
|
||||
});
|
||||
this.browserToolsRegistrations.set(connection.clientId, {
|
||||
capabilitySignature,
|
||||
unregister,
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user