diff --git a/packages/desktop/capture-harness/main.js b/packages/desktop/capture-harness/main.js index f657bfb2b6..5501ff8ba4 100644 --- a/packages/desktop/capture-harness/main.js +++ b/packages/desktop/capture-harness/main.js @@ -1574,9 +1574,8 @@ async function automationDrag(guest, sourceRef, targetRef) { } async function automationType(guest, refEntry, text) { - const send = await attachAutomationDebugger(guest); await automationClick(guest, refEntry); - await send("Input.insertText", { text }); + await guest.insertText(text); } function sendContainedEnter(guest) { diff --git a/packages/desktop/e2e/browser-tabs.e2e.mjs b/packages/desktop/e2e/browser-tabs.e2e.mjs index 6647af7b31..b96129e317 100644 --- a/packages/desktop/e2e/browser-tabs.e2e.mjs +++ b/packages/desktop/e2e/browser-tabs.e2e.mjs @@ -272,7 +272,7 @@ async function waitForGuestActiveElement(client, browserId, elementId) { return false; } -async function createCallerAgent(daemonPort) { +async function createCallerAgent(daemonPort, workspaceId = workspaceIds[0]) { const transport = new StreamableHTTPClientTransport( new URL(`http://127.0.0.1:${daemonPort}/mcp/agents`), ); @@ -282,7 +282,7 @@ async function createCallerAgent(daemonPort) { name: "create_agent", args: { relationship: { kind: "detached" }, - workspace: { kind: "existing", workspaceId: workspaceIds[0] }, + workspace: { kind: "existing", workspaceId }, title: "Browser desktop browser E2E caller", provider: "mock/ten-second-stream", settings: { modeId: "load-test" }, @@ -297,7 +297,7 @@ async function createCallerAgent(daemonPort) { ); assert(typeof result.agentId === "string", "create_agent returned no caller agent id"); assert( - result.workspaceId === workspaceIds[0], + result.workspaceId === workspaceId, `MCP caller attached to unexpected workspace ${result.workspaceId}`, ); return result.agentId; @@ -562,12 +562,66 @@ async function verifyHiddenBrowserScreenshots({ await expectAnimation(); } +async function verifyBackgroundTypingStaysInGuest({ page, client, serverId, browserId, agent }) { + await page + .getByTestId(`workspace-deck-entry-${serverId}:${agent.workspaceId}`) + .getByTestId(`workspace-tab-agent_${agent.agentId}`) + .click({ timeout: timeoutMs }); + const composer = page + .getByRole("textbox", { name: "Message agent..." }) + .filter({ visible: true }) + .first(); + const draft = "draft the user is typing"; + await composer.click(); + await composer.pressSequentially(draft); + assert( + await composer.evaluate((element) => element === document.activeElement), + "Composer did not keep focus while the user typed", + ); + + const snapshot = await callBrowserTool(client, "browser_snapshot", { browserId }); + const ref = snapshot.snapshot.match(/textbox "Typing target" \[ref=(@e\d+)\]/)?.[1]; + assert(ref, `browser_snapshot did not expose the typing target: ${snapshot.snapshot}`); + await callBrowserTool(client, "browser_type", { browserId, ref, text: "agent text" }); + await callBrowserTool(client, "browser_keypress", { browserId, ref, key: "!" }); + + // The user keeps typing in the composer while the agent types into the tab's focused field. + await composer.click(); + await composer.press("End"); + assert( + await composer.evaluate((element) => element === document.activeElement), + "Composer did not take focus back from the browser tab", + ); + await callBrowserTool(client, "browser_type", { browserId, text: " more" }); + await callBrowserTool(client, "browser_keypress", { browserId, key: "?" }); + + const guestValue = JSON.parse( + ( + await callBrowserTool(client, "browser_evaluate", { + browserId, + function: "() => document.getElementById('typing-target').value", + }) + ).resultJson, + ); + const composerValue = await composer.inputValue(); + assert( + composerValue === draft, + `browser_type and browser_keypress in a background tab changed the focused composer: ${JSON.stringify(composerValue)}`, + ); + assert( + guestValue === "agent text! more?", + `browser_type and browser_keypress did not reach the browser tab: ${JSON.stringify(guestValue)}`, + ); + await composer.fill(""); +} + async function runRegression({ page, client, serverId, targetUrl, callerAgentId, + typingAgent, artifactDir, inspectorPort, }) { @@ -857,6 +911,13 @@ async function runRegression({ text: "Clicked", timeoutMs: 5_000, }); + await verifyBackgroundTypingStaysInGuest({ + page, + client, + serverId, + browserId, + agent: typingAgent, + }); await originalWorkspaceRow.click(); await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).click(); @@ -1157,6 +1218,10 @@ async function main() { await runAppearanceFontSizeRegression(page); const callerAgentId = await createCallerAgent(daemonPort); + const typingAgent = { + workspaceId: workspaceIds.at(-1), + agentId: await createCallerAgent(daemonPort, workspaceIds.at(-1)), + }; const transport = new StreamableHTTPClientTransport( new URL( `http://127.0.0.1:${daemonPort}/mcp/agents?callerAgentId=${encodeURIComponent(callerAgentId)}`, @@ -1170,6 +1235,7 @@ async function main() { targetUrl: target.url, inspectorPort, callerAgentId, + typingAgent, artifactDir, }); const pluginLinks = await checkPluginLinks(); diff --git a/packages/desktop/src/features/browser-automation/ipc.test.ts b/packages/desktop/src/features/browser-automation/ipc.test.ts index ee7143adc6..427e86e1b1 100644 --- a/packages/desktop/src/features/browser-automation/ipc.test.ts +++ b/packages/desktop/src/features/browser-automation/ipc.test.ts @@ -190,6 +190,8 @@ class FakeWebContents { this.inputEvents.push(event); } + public async insertText(_text: string): Promise {} + public on(event: "console-message", listener: ConsoleMessageListener): void { expect(event).toBe("console-message"); this.consoleMessageListener = listener; diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 42c38d2e2c..18e5b84544 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -111,6 +111,7 @@ interface BrowserAutomationWebContents extends ConsoleMessageEmitter { getBackgroundThrottling(): boolean; setBackgroundThrottling(allowed: boolean): void; sendInputEvent(event: IsolatedKeyboardInputEvent): void; + insertText(text: string): Promise; } export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents { @@ -143,6 +144,7 @@ export function adaptWebContents(contents: BrowserAutomationWebContents): TabCon } }, sendInputEvent: (event) => contents.sendInputEvent(event), + insertText: (text) => contents.insertText(text), getConsoleMessages: () => consoleMessagesByContentsId.get(contentsId) ?? [], captureDialogs: (task) => dialogMonitor.capture(task), sendDebugCommand: (command: string, params?: Record) => diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index dee624a3f7..6bc054c6a5 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -41,6 +41,7 @@ class FakeTab implements TabContents { public readonly actions: string[] = []; public readonly capturedViewports: AbortSignal[] = []; public readonly debugCommands: Array<{ command: string; params?: Record }> = []; + public readonly insertedText: string[] = []; public readonly inputEvents: IsolatedKeyboardInputEvent[] = []; private readonly captureStartWaiters: Array<() => void> = []; private readonly deferredCaptures: Array<(image: TabImage) => void> = []; @@ -231,6 +232,10 @@ class FakeTab implements TabContents { this.inputEvents.push(event); } + public async insertText(text: string): Promise { + this.insertedText.push(text); + } + public waitForCaptureStart(count: number): Promise { if (this.capturedViewports.length >= count) { return Promise.resolve(); @@ -1104,17 +1109,31 @@ describe("executeAutomationCommand", () => { ok: true, result: { command: "type", browserId: BROWSER_A, ref: "@e1", x: 40, y: 30 }, }); - expect(browser.tab.debugCommands.at(-1)).toEqual({ - command: "Input.insertText", - params: { text: "Ada" }, - }); - expect(browser.tab.debugCommands.slice(0, 3).map((entry) => entry.command)).toEqual([ + expect(browser.tab.insertedText).toEqual(["Ada"]); + expect(browser.tab.debugCommands.map((entry) => entry.command)).toEqual([ "Input.dispatchMouseEvent", "Input.dispatchMouseEvent", "Input.dispatchMouseEvent", ]); }); + test("type inserts text through the tab, not the window-wide debugger input", async () => { + const browser = new BrowserAutomationHarness(); + + const action = await browser.execute({ + command: "type", + args: { browserId: BROWSER_A, text: "Ada" }, + }); + + expect(action).toEqual({ + requestId: "req-type", + ok: true, + result: { command: "type", browserId: BROWSER_A }, + }); + expect(browser.tab.insertedText).toEqual(["Ada"]); + expect(browser.tab.debugCommands).toEqual([]); + }); + test("keypress dispatches a trusted key to the focused element when ref is omitted", async () => { const browser = new BrowserAutomationHarness(); diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index ca6247e543..fe07adecd3 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -17,7 +17,6 @@ import { dispatchTrustedHover, dispatchTrustedKey, dispatchTrustedScroll, - dispatchTrustedText, type ClickInputOptions, type IsolatedKeyboardInputEvent, } from "./trusted-input.js"; @@ -39,6 +38,9 @@ export interface TabContents { invalidate(): void; withFrameProduction(capture: () => Promise): Promise; sendInputEvent(event: IsolatedKeyboardInputEvent): void; + // Commits text into this tab's focused element. CDP Input.insertText would + // commit into whichever element has focus in the Paseo window instead. + insertText(text: string): Promise; getConsoleMessages?(): BrowserAutomationConsoleLogEntry[]; captureDialogs?( task: () => Promise, @@ -1076,7 +1078,7 @@ async function executeType( } await dispatchTrustedClick(cdpSender(target.contents), actionable.target.point); } - await dispatchTrustedText(cdpSender(target.contents), text); + await target.contents.insertText(text); return { requestId, ok: true, diff --git a/packages/desktop/src/features/browser-automation/trusted-input.ts b/packages/desktop/src/features/browser-automation/trusted-input.ts index 671c5dbda1..8473263a1c 100644 --- a/packages/desktop/src/features/browser-automation/trusted-input.ts +++ b/packages/desktop/src/features/browser-automation/trusted-input.ts @@ -154,13 +154,6 @@ export async function dispatchTrustedScroll( }); } -export async function dispatchTrustedText(send: CdpCommandSender, text: string): Promise { - if (text.length === 0) { - return; - } - await send("Input.insertText", { text }); -} - export function dispatchTrustedKey(send: KeyboardInputSender, key: string): void { const keyCode = ELECTRON_KEY_CODE_ALIASES[key] ?? key; let character: string | null = null;