From 2974d7c960167541d18c10939a0db5726d2a7896 Mon Sep 17 00:00:00 2001 From: "paseo-bot[bot]" <266920839+paseo-bot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:59:59 +0200 Subject: [PATCH] fix(desktop): keep browser_type text out of the focused Paseo composer (#6207) browser_type committed its text with CDP Input.insertText on the tab's debugger. Chromium commits that text into the focused element of the whole Paseo window, so when the user was typing in an agent composer and the browser tab did not hold focus, the agent's text landed in the composer instead of the page. Insert the text with the tab's own webContents.insertText, which commits into the tab's focused element regardless of window focus. Fixes #6182 Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com> --- packages/desktop/capture-harness/main.js | 3 +- packages/desktop/e2e/browser-tabs.e2e.mjs | 72 ++++++++++++++++++- .../features/browser-automation/ipc.test.ts | 2 + .../src/features/browser-automation/ipc.ts | 2 + .../browser-automation/service.test.ts | 29 ++++++-- .../features/browser-automation/service.ts | 6 +- .../browser-automation/trusted-input.ts | 7 -- 7 files changed, 102 insertions(+), 19 deletions(-) diff --git a/packages/desktop/capture-harness/main.js b/packages/desktop/capture-harness/main.js index f657bfb2b6..5501ff8ba4 100644 --- a/packages/desktop/capture-harness/main.js +++ b/packages/desktop/capture-harness/main.js @@ -1574,9 +1574,8 @@ async function automationDrag(guest, sourceRef, targetRef) { } async function automationType(guest, refEntry, text) { - const send = await attachAutomationDebugger(guest); await automationClick(guest, refEntry); - await send("Input.insertText", { text }); + await guest.insertText(text); } function sendContainedEnter(guest) { diff --git a/packages/desktop/e2e/browser-tabs.e2e.mjs b/packages/desktop/e2e/browser-tabs.e2e.mjs index 6647af7b31..b96129e317 100644 --- a/packages/desktop/e2e/browser-tabs.e2e.mjs +++ b/packages/desktop/e2e/browser-tabs.e2e.mjs @@ -272,7 +272,7 @@ async function waitForGuestActiveElement(client, browserId, elementId) { return false; } -async function createCallerAgent(daemonPort) { +async function createCallerAgent(daemonPort, workspaceId = workspaceIds[0]) { const transport = new StreamableHTTPClientTransport( new URL(`http://127.0.0.1:${daemonPort}/mcp/agents`), ); @@ -282,7 +282,7 @@ async function createCallerAgent(daemonPort) { name: "create_agent", args: { relationship: { kind: "detached" }, - workspace: { kind: "existing", workspaceId: workspaceIds[0] }, + workspace: { kind: "existing", workspaceId }, title: "Browser desktop browser E2E caller", provider: "mock/ten-second-stream", settings: { modeId: "load-test" }, @@ -297,7 +297,7 @@ async function createCallerAgent(daemonPort) { ); assert(typeof result.agentId === "string", "create_agent returned no caller agent id"); assert( - result.workspaceId === workspaceIds[0], + result.workspaceId === workspaceId, `MCP caller attached to unexpected workspace ${result.workspaceId}`, ); return result.agentId; @@ -562,12 +562,66 @@ async function verifyHiddenBrowserScreenshots({ await expectAnimation(); } +async function verifyBackgroundTypingStaysInGuest({ page, client, serverId, browserId, agent }) { + await page + .getByTestId(`workspace-deck-entry-${serverId}:${agent.workspaceId}`) + .getByTestId(`workspace-tab-agent_${agent.agentId}`) + .click({ timeout: timeoutMs }); + const composer = page + .getByRole("textbox", { name: "Message agent..." }) + .filter({ visible: true }) + .first(); + const draft = "draft the user is typing"; + await composer.click(); + await composer.pressSequentially(draft); + assert( + await composer.evaluate((element) => element === document.activeElement), + "Composer did not keep focus while the user typed", + ); + + const snapshot = await callBrowserTool(client, "browser_snapshot", { browserId }); + const ref = snapshot.snapshot.match(/textbox "Typing target" \[ref=(@e\d+)\]/)?.[1]; + assert(ref, `browser_snapshot did not expose the typing target: ${snapshot.snapshot}`); + await callBrowserTool(client, "browser_type", { browserId, ref, text: "agent text" }); + await callBrowserTool(client, "browser_keypress", { browserId, ref, key: "!" }); + + // The user keeps typing in the composer while the agent types into the tab's focused field. + await composer.click(); + await composer.press("End"); + assert( + await composer.evaluate((element) => element === document.activeElement), + "Composer did not take focus back from the browser tab", + ); + await callBrowserTool(client, "browser_type", { browserId, text: " more" }); + await callBrowserTool(client, "browser_keypress", { browserId, key: "?" }); + + const guestValue = JSON.parse( + ( + await callBrowserTool(client, "browser_evaluate", { + browserId, + function: "() => document.getElementById('typing-target').value", + }) + ).resultJson, + ); + const composerValue = await composer.inputValue(); + assert( + composerValue === draft, + `browser_type and browser_keypress in a background tab changed the focused composer: ${JSON.stringify(composerValue)}`, + ); + assert( + guestValue === "agent text! more?", + `browser_type and browser_keypress did not reach the browser tab: ${JSON.stringify(guestValue)}`, + ); + await composer.fill(""); +} + async function runRegression({ page, client, serverId, targetUrl, callerAgentId, + typingAgent, artifactDir, inspectorPort, }) { @@ -857,6 +911,13 @@ async function runRegression({ text: "Clicked", timeoutMs: 5_000, }); + await verifyBackgroundTypingStaysInGuest({ + page, + client, + serverId, + browserId, + agent: typingAgent, + }); await originalWorkspaceRow.click(); await originalDeck.getByTestId(`workspace-tab-browser_${browserId}`).click(); @@ -1157,6 +1218,10 @@ async function main() { await runAppearanceFontSizeRegression(page); const callerAgentId = await createCallerAgent(daemonPort); + const typingAgent = { + workspaceId: workspaceIds.at(-1), + agentId: await createCallerAgent(daemonPort, workspaceIds.at(-1)), + }; const transport = new StreamableHTTPClientTransport( new URL( `http://127.0.0.1:${daemonPort}/mcp/agents?callerAgentId=${encodeURIComponent(callerAgentId)}`, @@ -1170,6 +1235,7 @@ async function main() { targetUrl: target.url, inspectorPort, callerAgentId, + typingAgent, artifactDir, }); const pluginLinks = await checkPluginLinks(); diff --git a/packages/desktop/src/features/browser-automation/ipc.test.ts b/packages/desktop/src/features/browser-automation/ipc.test.ts index ee7143adc6..427e86e1b1 100644 --- a/packages/desktop/src/features/browser-automation/ipc.test.ts +++ b/packages/desktop/src/features/browser-automation/ipc.test.ts @@ -190,6 +190,8 @@ class FakeWebContents { this.inputEvents.push(event); } + public async insertText(_text: string): Promise {} + public on(event: "console-message", listener: ConsoleMessageListener): void { expect(event).toBe("console-message"); this.consoleMessageListener = listener; diff --git a/packages/desktop/src/features/browser-automation/ipc.ts b/packages/desktop/src/features/browser-automation/ipc.ts index 42c38d2e2c..18e5b84544 100644 --- a/packages/desktop/src/features/browser-automation/ipc.ts +++ b/packages/desktop/src/features/browser-automation/ipc.ts @@ -111,6 +111,7 @@ interface BrowserAutomationWebContents extends ConsoleMessageEmitter { getBackgroundThrottling(): boolean; setBackgroundThrottling(allowed: boolean): void; sendInputEvent(event: IsolatedKeyboardInputEvent): void; + insertText(text: string): Promise; } export function adaptWebContents(contents: BrowserAutomationWebContents): TabContents { @@ -143,6 +144,7 @@ export function adaptWebContents(contents: BrowserAutomationWebContents): TabCon } }, sendInputEvent: (event) => contents.sendInputEvent(event), + insertText: (text) => contents.insertText(text), getConsoleMessages: () => consoleMessagesByContentsId.get(contentsId) ?? [], captureDialogs: (task) => dialogMonitor.capture(task), sendDebugCommand: (command: string, params?: Record) => diff --git a/packages/desktop/src/features/browser-automation/service.test.ts b/packages/desktop/src/features/browser-automation/service.test.ts index dee624a3f7..6bc054c6a5 100644 --- a/packages/desktop/src/features/browser-automation/service.test.ts +++ b/packages/desktop/src/features/browser-automation/service.test.ts @@ -41,6 +41,7 @@ class FakeTab implements TabContents { public readonly actions: string[] = []; public readonly capturedViewports: AbortSignal[] = []; public readonly debugCommands: Array<{ command: string; params?: Record }> = []; + public readonly insertedText: string[] = []; public readonly inputEvents: IsolatedKeyboardInputEvent[] = []; private readonly captureStartWaiters: Array<() => void> = []; private readonly deferredCaptures: Array<(image: TabImage) => void> = []; @@ -231,6 +232,10 @@ class FakeTab implements TabContents { this.inputEvents.push(event); } + public async insertText(text: string): Promise { + this.insertedText.push(text); + } + public waitForCaptureStart(count: number): Promise { if (this.capturedViewports.length >= count) { return Promise.resolve(); @@ -1104,17 +1109,31 @@ describe("executeAutomationCommand", () => { ok: true, result: { command: "type", browserId: BROWSER_A, ref: "@e1", x: 40, y: 30 }, }); - expect(browser.tab.debugCommands.at(-1)).toEqual({ - command: "Input.insertText", - params: { text: "Ada" }, - }); - expect(browser.tab.debugCommands.slice(0, 3).map((entry) => entry.command)).toEqual([ + expect(browser.tab.insertedText).toEqual(["Ada"]); + expect(browser.tab.debugCommands.map((entry) => entry.command)).toEqual([ "Input.dispatchMouseEvent", "Input.dispatchMouseEvent", "Input.dispatchMouseEvent", ]); }); + test("type inserts text through the tab, not the window-wide debugger input", async () => { + const browser = new BrowserAutomationHarness(); + + const action = await browser.execute({ + command: "type", + args: { browserId: BROWSER_A, text: "Ada" }, + }); + + expect(action).toEqual({ + requestId: "req-type", + ok: true, + result: { command: "type", browserId: BROWSER_A }, + }); + expect(browser.tab.insertedText).toEqual(["Ada"]); + expect(browser.tab.debugCommands).toEqual([]); + }); + test("keypress dispatches a trusted key to the focused element when ref is omitted", async () => { const browser = new BrowserAutomationHarness(); diff --git a/packages/desktop/src/features/browser-automation/service.ts b/packages/desktop/src/features/browser-automation/service.ts index ca6247e543..fe07adecd3 100644 --- a/packages/desktop/src/features/browser-automation/service.ts +++ b/packages/desktop/src/features/browser-automation/service.ts @@ -17,7 +17,6 @@ import { dispatchTrustedHover, dispatchTrustedKey, dispatchTrustedScroll, - dispatchTrustedText, type ClickInputOptions, type IsolatedKeyboardInputEvent, } from "./trusted-input.js"; @@ -39,6 +38,9 @@ export interface TabContents { invalidate(): void; withFrameProduction(capture: () => Promise): Promise; sendInputEvent(event: IsolatedKeyboardInputEvent): void; + // Commits text into this tab's focused element. CDP Input.insertText would + // commit into whichever element has focus in the Paseo window instead. + insertText(text: string): Promise; getConsoleMessages?(): BrowserAutomationConsoleLogEntry[]; captureDialogs?( task: () => Promise, @@ -1076,7 +1078,7 @@ async function executeType( } await dispatchTrustedClick(cdpSender(target.contents), actionable.target.point); } - await dispatchTrustedText(cdpSender(target.contents), text); + await target.contents.insertText(text); return { requestId, ok: true, diff --git a/packages/desktop/src/features/browser-automation/trusted-input.ts b/packages/desktop/src/features/browser-automation/trusted-input.ts index 671c5dbda1..8473263a1c 100644 --- a/packages/desktop/src/features/browser-automation/trusted-input.ts +++ b/packages/desktop/src/features/browser-automation/trusted-input.ts @@ -154,13 +154,6 @@ export async function dispatchTrustedScroll( }); } -export async function dispatchTrustedText(send: CdpCommandSender, text: string): Promise { - if (text.length === 0) { - return; - } - await send("Input.insertText", { text }); -} - export function dispatchTrustedKey(send: KeyboardInputSender, key: string): void { const keyCode = ELECTRON_KEY_CODE_ALIASES[key] ?? key; let character: string | null = null;