M2: terminal dictation: mic in tab-bar slot and key bar, recording pill and overlay, per-terminal recording lifetime incl. pending mic acquisition, Cmd+D on macOS by keyboard focus, i18n

packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts

@@ -1,6 +1,10 @@
 import { expect, test, type Page } from "../support/fixtures";
 import { gotoAppShell } from "../support/helpers/app";
-import { daemonWsRoutePattern } from "../support/helpers/daemon-port";
+import {
+  installDictationWsHarness,
+  installSyntheticMicrophone,
+  sendSessionMessage,
+} from "../support/helpers/dictation-ws";
 import {
   openNewWorkspaceComposer,
   selectWorkspaceIsolation,
@@ -17,141 +21,39 @@ import {
 } from "../support/helpers/composer";
 import { waitForSidebarHydration } from "../support/helpers/workspace-ui";
 
-type WebSocketMessage = string | Buffer;
-
 const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace";
 const CREATE_FAILURE = "Synthetic workspace creation failure";
 
-function parseEnvelope(message: WebSocketMessage): {
-  type?: unknown;
-  message?: Record<string, unknown>;
-} | null {
-  const raw = typeof message === "string" ? message : message.toString("utf8");
-  try {
-    return JSON.parse(raw) as { type?: unknown; message?: Record<string, unknown> };
-  } catch {
-    return null;
-  }
-}
-
-function sessionMessage(message: WebSocketMessage): Record<string, unknown> | null {
-  const envelope = parseEnvelope(message);
-  return envelope?.type === "session" && envelope.message ? envelope.message : null;
-}
-
-function sendSessionMessage(
-  ws: { send(message: string): void },
-  message: Record<string, unknown>,
-): void {
-  ws.send(JSON.stringify({ type: "session", message }));
-}
-
-function enableDictationCapability(message: WebSocketMessage): WebSocketMessage {
-  const envelope = parseEnvelope(message);
-  const payload = envelope?.message?.payload;
-  if (
-    envelope?.message?.type !== "status" ||
-    !payload ||
-    typeof payload !== "object" ||
-    (payload as { status?: unknown }).status !== "server_info"
-  ) {
-    return message;
-  }
-
-  (payload as Record<string, unknown>).capabilities = {
-    voice: {
-      dictation: { enabled: true, reason: "" },
-      voice: { enabled: false, reason: "Realtime voice is disabled in this test." },
-    },
-  };
-  return JSON.stringify(envelope);
-}
-
-async function installSyntheticMicrophone(page: Page): Promise<void> {
-  await page.addInitScript(() => {
-    const mediaDevices = navigator.mediaDevices;
-    Object.defineProperty(mediaDevices, "getUserMedia", {
-      configurable: true,
-      value: async () => {
-        const context = new AudioContext();
-        const oscillator = context.createOscillator();
-        const destination = context.createMediaStreamDestination();
-        oscillator.connect(destination);
-        oscillator.start();
-        return destination.stream;
-      },
-    });
-  });
-}
-
 async function installDictationFailureHarness(page: Page) {
-  let resolveAudioChunk!: () => void;
   let resolveCreateRequest!: () => void;
-  const audioChunk = new Promise<void>((resolve) => {
-    resolveAudioChunk = resolve;
-  });
   const createRequest = new Promise<void>((resolve) => {
     resolveCreateRequest = resolve;
   });
   let failCreate: (() => void) | null = null;
 
-  await page.routeWebSocket(daemonWsRoutePattern(), (ws) => {
-    const server = ws.connectToServer();
-
-    ws.onMessage((message) => {
-      const request = sessionMessage(message);
-      const type = request?.type;
-      const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null;
-
-      if (type === "dictation_stream_start" && dictationId) {
-        sendSessionMessage(ws, {
-          type: "dictation_stream_ack",
-          payload: { dictationId, ackSeq: -1 },
-        });
-        return;
+  const dictation = await installDictationWsHarness(page, {
+    transcript: TRANSCRIPT,
+    interceptClientMessage: (request, ws) => {
+      if (request.type !== "workspace.create.request" || typeof request.requestId !== "string") {
+        return false;
       }
-      if (type === "dictation_stream_chunk" && dictationId) {
-        const seq = typeof request?.seq === "number" ? request.seq : 0;
+      const requestId = request.requestId;
+      failCreate = () => {
         sendSessionMessage(ws, {
-          type: "dictation_stream_ack",
-          payload: { dictationId, ackSeq: seq },
+          type: "workspace.create.response",
+          payload: {
+            requestId,
+            workspace: null,
+            setupTerminalId: null,
+            error: CREATE_FAILURE,
+          },
         });
-        resolveAudioChunk();
-        return;
-      }
-      if (type === "dictation_stream_finish" && dictationId) {
-        sendSessionMessage(ws, {
-          type: "dictation_stream_finish_accepted",
-          payload: { dictationId, timeoutMs: 5_000 },
-        });
-        sendSessionMessage(ws, {
-          type: "dictation_stream_final",
-          payload: { dictationId, text: TRANSCRIPT },
-        });
-        return;
-      }
-      if (request && type === "workspace.create.request" && typeof request.requestId === "string") {
-        const requestId = request.requestId;
-        failCreate = () => {
-          sendSessionMessage(ws, {
-            type: "workspace.create.response",
-            payload: {
-              requestId,
-              workspace: null,
-              setupTerminalId: null,
-              error: CREATE_FAILURE,
-            },
-          });
-        };
-        resolveCreateRequest();
-        return;
-      }
-
-      server.send(message);
-    });
-
-    server.onMessage((message) => ws.send(enableDictationCapability(message)));
+      };
+      resolveCreateRequest();
+      return true;
+    },
   });
+  const audioChunk = dictation.nextChunk();
 
   return {
     waitForAudio: () => audioChunk,

packages/app/e2e/browser/terminal-dictation.spec.ts

@@ -0,0 +1,598 @@
+import { writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { Locator, Page } from "@playwright/test";
+import { expect, test } from "../support/fixtures";
+import { runWorkspaceActionFromCommandCenter } from "../support/helpers/command-center-workspace-actions";
+import {
+  holdMicrophone,
+  installDictationWsHarness,
+  installSyntheticMicrophone,
+  microphoneStats,
+  releaseMicrophone,
+  type DictationCapability,
+  type DictationWsHarness,
+} from "../support/helpers/dictation-ws";
+import { TerminalE2EHarness, type TerminalInstance } from "../support/helpers/terminal-dsl";
+import { ensureExplorerSidebar } from "../support/helpers/workspace-tabs";
+
+const MAC_USER_AGENT =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
+const LINUX_USER_AGENT =
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
+
+// Payloads are echoed as JSON with whitespace escaped, because the PTY capture
+// below drops whitespace when it joins wrapped lines.
+const ENCODE = String.raw`
+const encode = (text) =>
+  JSON.stringify(text).replace(
+    /\s/g,
+    (space) => "\\u" + space.charCodeAt(0).toString(16).padStart(4, "0"),
+  );
+`;
+
+// A TUI with bracketed paste on: echoes each paste body as INPUT:<json>.
+const BRACKETED_RECEIVER_SCRIPT = String.raw`
+${ENCODE}
+let buffer = "";
+process.stdin.setRawMode(true);
+process.stdin.resume();
+process.stdout.write("\x1b[?2004hDICTATION_RECEIVER_READY\r\n");
+process.stdin.on("data", (chunk) => {
+  buffer += chunk.toString();
+  for (;;) {
+    const paste = /\x1b\[200~([^]*?)\x1b\[201~/.exec(buffer);
+    if (!paste) break;
+    process.stdout.write("INPUT:" + encode(paste[1]) + "\r\n");
+    buffer = buffer.slice(0, paste.index) + buffer.slice(paste.index + paste[0].length);
+  }
+});
+`;
+
+// A program without bracketed paste: echoes every stdin byte, in raw mode so
+// Ctrl+D arrives as \x04, as RAW_INPUT:<json> after a quiet period.
+const RAW_RECEIVER_SCRIPT = String.raw`
+${ENCODE}
+let pending = "";
+let timer = null;
+process.stdin.setRawMode(true);
+process.stdin.resume();
+process.stdout.write("RAW_RECEIVER_READY\r\n");
+process.stdin.on("data", (chunk) => {
+  pending += chunk.toString();
+  clearTimeout(timer);
+  timer = setTimeout(() => {
+    process.stdout.write("RAW_INPUT:" + encode(pending) + "\r\n");
+    pending = "";
+  }, 200);
+});
+`;
+
+type ReceiverKind = "bracketed" | "raw";
+
+function visibleTestId(scope: Page | Locator, testId: string): Locator {
+  return scope.getByTestId(testId).filter({ visible: true }).first();
+}
+
+// Click the surface, not its helper textarea: the textarea sits at the left edge,
+// under a split or Explorer resize handle.
+function terminalSurface(scope: Page | Locator): Locator {
+  return scope.getByTestId("terminal-surface").filter({ visible: true });
+}
+
+function terminalInput(scope: Page | Locator): Locator {
+  return scope
+    .getByTestId("terminal-surface")
+    .filter({ visible: true })
+    .locator(".xterm-helper-textarea");
+}
+
+function splitChild(page: Page, terminal: TerminalInstance): Locator {
+  return page
+    .getByTestId("split-group-child")
+    .filter({ has: page.getByTestId(`workspace-tab-terminal_${terminal.id}`) });
+}
+
+function explorerSidebar(page: Page): Locator {
+  return page.getByTestId("workspace-explorer-sidebar").filter({ visible: true }).first();
+}
+
+async function dragTabIntoExplorer(page: Page, terminal: TerminalInstance): Promise<void> {
+  const source = page
+    .getByTestId("workspace-pane-main")
+    .getByTestId(`workspace-tab-terminal_${terminal.id}`);
+  const destination = explorerSidebar(page).getByTestId("workspace-tab-files");
+  const sourceBox = await source.boundingBox();
+  const destinationBox = await destination.boundingBox();
+  if (!sourceBox || !destinationBox) {
+    throw new Error("Tab drag requires visible source and destination");
+  }
+  const start = { x: sourceBox.x + 12, y: sourceBox.y + sourceBox.height / 2 };
+  await page.mouse.move(start.x, start.y);
+  await page.mouse.down();
+  await page.mouse.move(start.x + 12, start.y + 4);
+  await page.mouse.move(destinationBox.x + 12, destinationBox.y + destinationBox.height / 2, {
+    steps: 20,
+  });
+  await page.mouse.up();
+}
+
+async function expectMicrophoneStat(
+  page: Page,
+  stat: "trackStops" | "getUserMediaCalls",
+  count: number,
+): Promise<void> {
+  await expect.poll(async () => (await microphoneStats(page))[stat]).toBe(count);
+}
+
+test.describe("Terminal dictation", () => {
+  test.describe.configure({ timeout: 180_000 });
+
+  let harness: TerminalE2EHarness;
+
+  test.beforeAll(async () => {
+    harness = await TerminalE2EHarness.create({ tempPrefix: "terminal-dictation-" });
+    await writeFile(
+      path.join(harness.tempRepo.path, "bracketed-receiver.cjs"),
+      BRACKETED_RECEIVER_SCRIPT,
+    );
+    await writeFile(path.join(harness.tempRepo.path, "raw-receiver.cjs"), RAW_RECEIVER_SCRIPT);
+  });
+
+  test.afterAll(async () => {
+    await harness?.cleanup();
+  });
+
+  const terminals: TerminalInstance[] = [];
+
+  test.afterEach(async () => {
+    for (const terminal of terminals.splice(0)) {
+      await harness.killTerminal(terminal.id);
+    }
+  });
+
+  async function createReceiver(kind: ReceiverKind, name: string): Promise<TerminalInstance> {
+    const terminal = await harness.createTerminal({
+      name,
+      command: process.execPath,
+      args: [kind === "bracketed" ? "bracketed-receiver.cjs" : "raw-receiver.cjs"],
+    });
+    terminals.push(terminal);
+    await expectTerminalOutput(
+      terminal.id,
+      kind === "bracketed" ? "DICTATION_RECEIVER_READY" : "RAW_RECEIVER_READY",
+    );
+    return terminal;
+  }
+
+  async function setup(
+    page: Page,
+    options: { transcript?: string; capability?: DictationCapability } = {},
+  ): Promise<DictationWsHarness> {
+    await installSyntheticMicrophone(page);
+    return await installDictationWsHarness(page, {
+      transcript: options.transcript ?? "first\nsecond",
+      capability: options.capability,
+    });
+  }
+
+  // Reads the PTY itself, so output from one pane cannot satisfy another's check.
+  async function terminalOutput(terminalId: string): Promise<string> {
+    const capture = await harness.client.captureTerminal(terminalId);
+    return capture.lines.join("").replace(/\s/g, "");
+  }
+
+  async function expectTerminalOutput(terminalId: string, text: string): Promise<void> {
+    await expect.poll(() => terminalOutput(terminalId), { timeout: 10_000 }).toContain(text);
+  }
+
+  /** Everything the receiver got, in order and joined. */
+  async function received(terminalId: string, kind: ReceiverKind): Promise<string> {
+    const label = kind === "bracketed" ? "INPUT" : "RAW_INPUT";
+    const output = await terminalOutput(terminalId);
+    return [
+      ...output.matchAll(
+        new RegExp(
+          `${kind === "bracketed" ? "(?<!RAW_)" : ""}${label}:("(?:[^"\\\\]|\\\\.)*")`,
+          "g",
+        ),
+      ),
+    ]
+      .map(([, json]) => JSON.parse(json) as string)
+      .join("");
+  }
+
+  async function expectReceived(
+    terminalId: string,
+    kind: ReceiverKind,
+    text: string,
+  ): Promise<void> {
+    await expect.poll(() => received(terminalId, kind), { timeout: 15_000 }).toBe(text);
+  }
+
+  /** Gives a late transcript time to land, then checks that none did. */
+  async function expectNothingReceived(
+    page: Page,
+    ...receivers: Array<[TerminalInstance, ReceiverKind]>
+  ): Promise<void> {
+    await page.waitForTimeout(1_500);
+    for (const [terminal, kind] of receivers) {
+      expect(await received(terminal.id, kind)).toBe("");
+    }
+  }
+
+  async function startFromToolbar(
+    page: Page,
+    ws: DictationWsHarness,
+    scope: Page | Locator = page,
+  ) {
+    const chunk = ws.nextChunk();
+    await visibleTestId(scope, "terminal-toolbar-dictation").click();
+    await chunk;
+  }
+
+  // The route helper needs the desktop tab row, so open there, then go compact.
+  async function openCompact(page: Page, terminal: TerminalInstance) {
+    await harness.openTerminal(page, { terminalId: terminal.id });
+    await page.setViewportSize({ width: 390, height: 844 });
+    await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
+  }
+
+  // The compact switcher labels terminals by their command, so pick by position.
+  async function selectCompactTab(page: Page, index: number) {
+    await page
+      .getByRole("button", { name: /Switch tabs/ })
+      .filter({ visible: true })
+      .click();
+    await expect(page.getByText("Workspace tabs", { exact: true })).toBeVisible();
+    const options = page
+      .getByText("bracketed-receiver.cjs", { exact: true })
+      .filter({ visible: true });
+    await expect(options).toHaveCount(2);
+    await options.nth(index).click();
+    await expect(page.getByRole("button", { name: "Bottom sheet backdrop" })).toHaveCount(0);
+  }
+
+  async function selectTerminalTab(scope: Page | Locator, terminal: TerminalInstance) {
+    await visibleTestId(scope, `workspace-tab-terminal_${terminal.id}`).click();
+  }
+
+  test("confirm pastes the transcript on one line with a trailing space and no Enter", async ({
+    page,
+  }) => {
+    const ws = await setup(page);
+    const bracketed = await createReceiver("bracketed", "dictation-bracketed");
+    const raw = await createReceiver("raw", "dictation-raw");
+
+    await harness.openTerminal(page, { terminalId: bracketed.id });
+    await startFromToolbar(page, ws);
+    await visibleTestId(page, "terminal-dictation-confirm").click();
+    await expectReceived(bracketed.id, "bracketed", "first second ");
+    await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
+
+    await selectTerminalTab(page, raw);
+    await startFromToolbar(page, ws);
+    await visibleTestId(page, "terminal-dictation-confirm").click();
+    await expectReceived(raw.id, "raw", "first second ");
+    expect(await received(bracketed.id, "bracketed")).toBe("first second ");
+  });
+
+  test.describe("on macOS", () => {
+    test.use({ userAgent: MAC_USER_AGENT });
+
+    test("Cmd+D starts and then confirms dictation in the focused terminal", async ({ page }) => {
+      const ws = await setup(page, { transcript: "shortcut words" });
+      const terminal = await createReceiver("bracketed", "dictation-shortcut");
+
+      await harness.openTerminal(page, { terminalId: terminal.id });
+      const chunk = ws.nextChunk();
+      await terminalInput(page).press("Meta+d");
+      await expect(visibleTestId(page, "terminal-dictation-confirm")).toBeVisible();
+      await chunk;
+      await terminalInput(page).press("Meta+d");
+
+      await expectReceived(terminal.id, "bracketed", "shortcut words ");
+      await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
+    });
+
+    test("Cmd+D goes to the terminal with keyboard focus, main pane or Explorer", async ({
+      page,
+    }) => {
+      const ws = await setup(page, { transcript: "main pane words" });
+      const explorerTerminal = await createReceiver("bracketed", "dictation-explorer");
+      const mainTerminal = await createReceiver("bracketed", "dictation-main");
+
+      await harness.openTerminal(page, { terminalId: explorerTerminal.id });
+      await ensureExplorerSidebar(page);
+      await dragTabIntoExplorer(page, explorerTerminal);
+      const explorer = explorerSidebar(page);
+      await expect(
+        explorer.getByTestId(`workspace-tab-terminal_${explorerTerminal.id}`),
+      ).toBeVisible();
+      const main = page.getByTestId("workspace-pane-main");
+      await selectTerminalTab(main, mainTerminal);
+      await expect(explorer.getByTestId("terminal-surface")).toBeVisible();
+
+      let chunk = ws.nextChunk();
+      await terminalSurface(main).click();
+      await terminalInput(main).press("Meta+d");
+      await expect(visibleTestId(main, "terminal-dictation-confirm")).toBeVisible();
+      await expect(explorer.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
+      await chunk;
+      await terminalInput(main).press("Meta+d");
+      await expectReceived(mainTerminal.id, "bracketed", "main pane words ");
+      await expectNothingReceived(page, [explorerTerminal, "bracketed"]);
+
+      ws.setTranscript("explorer words");
+      chunk = ws.nextChunk();
+      await terminalSurface(explorer).click();
+      await terminalInput(explorer).press("Meta+d");
+      await expect(visibleTestId(explorer, "terminal-dictation-confirm")).toBeVisible();
+      await expect(main.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
+      await chunk;
+      await terminalInput(explorer).press("Meta+d");
+      await expectReceived(explorerTerminal.id, "bracketed", "explorer words ");
+      expect(await received(mainTerminal.id, "bracketed")).toBe("main pane words ");
+    });
+
+    test("Cmd+D records only in the focused split pane", async ({ page }) => {
+      const ws = await setup(page, { transcript: "right pane words" });
+      const left = await createReceiver("bracketed", "dictation-split-left");
+      const right = await createReceiver("bracketed", "dictation-split-right");
+
+      await harness.openTerminal(page, { terminalId: left.id });
+      await runWorkspaceActionFromCommandCenter(page, "Split pane right");
+      await visibleTestId(page, `workspace-tab-terminal_${right.id}`).click();
+      await runWorkspaceActionFromCommandCenter(page, "Move tab right");
+      const leftPane = splitChild(page, left);
+      const rightPane = splitChild(page, right);
+      await expect(terminalInput(leftPane)).toHaveCount(1);
+      await expect(terminalInput(rightPane)).toHaveCount(1);
+
+      const chunk = ws.nextChunk();
+      await terminalSurface(rightPane).click();
+      await terminalInput(rightPane).press("Meta+d");
+      await expect(visibleTestId(rightPane, "terminal-dictation-confirm")).toBeVisible();
+      await expect(leftPane.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
+      await chunk;
+      await terminalInput(rightPane).press("Meta+d");
+
+      await expectReceived(right.id, "bracketed", "right pane words ");
+      await expectNothingReceived(page, [left, "bracketed"]);
+    });
+  });
+
+  test("Ctrl+D off macOS reaches the PTY as EOF and starts nothing", async ({ page }) => {
+    // Chromium keeps the host's navigator.platform unless the override sets it too.
+    const session = await page.context().newCDPSession(page);
+    await session.send("Emulation.setUserAgentOverride", {
+      userAgent: LINUX_USER_AGENT,
+      platform: "Linux x86_64",
+    });
+    const ws = await setup(page);
+    const terminal = await createReceiver("raw", "dictation-ctrl-d");
+
+    await harness.openTerminal(page, { terminalId: terminal.id });
+    await terminalInput(page).press("Control+d");
+
+    await expectReceived(terminal.id, "raw", "\u0004");
+    expect(ws.streamStarts()).toBe(0);
+    expect((await microphoneStats(page)).getUserMediaCalls).toBe(0);
+    await expect(page.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
+  });
+
+  test("switching the pane to another terminal while recording stops it and delivers nothing", async ({
+    page,
+  }) => {
+    const ws = await setup(page);
+    const first = await createReceiver("bracketed", "dictation-switch-a");
+    const second = await createReceiver("bracketed", "dictation-switch-b");
+
+    await harness.openTerminal(page, { terminalId: first.id });
+    await startFromToolbar(page, ws);
+    await selectTerminalTab(page, second);
+
+    await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
+    await expect(
+      page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
+    ).toHaveCount(0);
+    await expectMicrophoneStat(page, "trackStops", 1);
+    const chunksAfterSwitch = ws.chunks();
+    const startsAfterSwitch = ws.streamStarts();
+    await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
+    expect(ws.chunks()).toBe(chunksAfterSwitch);
+    expect(ws.streamStarts()).toBe(startsAfterSwitch);
+
+    await selectTerminalTab(page, first);
+    await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
+    await expect(
+      page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
+    ).toHaveCount(0);
+  });
+
+  test("switching the pane to another terminal while processing drops the late transcript", async ({
+    page,
+  }) => {
+    const ws = await setup(page);
+    const first = await createReceiver("bracketed", "dictation-processing-a");
+    const second = await createReceiver("bracketed", "dictation-processing-b");
+
+    await harness.openTerminal(page, { terminalId: first.id });
+    await startFromToolbar(page, ws);
+    ws.holdFinal();
+    await visibleTestId(page, "terminal-dictation-confirm").click();
+    await expect(visibleTestId(page, "terminal-dictation-confirm")).toHaveCount(0);
+    await expect(visibleTestId(page, "terminal-dictation-cancel")).toBeVisible();
+    await selectTerminalTab(page, second);
+    await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
+    ws.releaseFinal();
+
+    await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
+  });
+
+  test.describe("on compact web", () => {
+    // Keyboard shortcuts are off on compact form factors (keyboardShortcutsAvailable), so
+    // the key-bar mic is how compact web dictates.
+    test("the key-bar mic shows the overlay and confirm delivers", async ({ page }) => {
+      const ws = await setup(page, { transcript: "compact words" });
+      const terminal = await createReceiver("bracketed", "dictation-compact-confirm");
+
+      await openCompact(page, terminal);
+      const chunk = ws.nextChunk();
+      await visibleTestId(page, "terminal-dictation").click();
+      const overlay = visibleTestId(page, "terminal-dictation-overlay");
+      await expect(overlay).toBeVisible();
+      await expect(visibleTestId(page, "terminal-dictation")).toHaveCount(0);
+      await expect(
+        overlay.getByRole("button", { name: "Insert transcription and send", exact: true }),
+      ).toHaveCount(0);
+      await chunk;
+      await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
+
+      await expectReceived(terminal.id, "bracketed", "compact words ");
+      await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
+      await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
+    });
+
+    test("switching tabs while recording from the key bar stops it and delivers nothing", async ({
+      page,
+    }) => {
+      const ws = await setup(page);
+      const first = await createReceiver("bracketed", "dictation-compact-a");
+      const second = await createReceiver("bracketed", "dictation-compact-b");
+
+      await openCompact(page, first);
+      const chunk = ws.nextChunk();
+      await visibleTestId(page, "terminal-dictation").click();
+      await expect(visibleTestId(page, "terminal-dictation-overlay")).toBeVisible();
+      await chunk;
+
+      await selectCompactTab(page, 1);
+      await expectMicrophoneStat(page, "trackStops", 1);
+      await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
+
+      await selectCompactTab(page, 0);
+      await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
+      await expect(
+        page.getByTestId("terminal-dictation-overlay").filter({ visible: true }),
+      ).toHaveCount(0);
+      expect(await received(first.id, "bracketed")).toBe("");
+    });
+
+    test("a failed dictation keeps the overlay with retry and discard", async ({ page }) => {
+      const ws = await setup(page, { transcript: "retried words" });
+      const terminal = await createReceiver("bracketed", "dictation-compact-failure");
+
+      await openCompact(page, terminal);
+      const overlay = visibleTestId(page, "terminal-dictation-overlay");
+
+      ws.failNextFinish("Synthetic transcription failure");
+      let chunk = ws.nextChunk();
+      await visibleTestId(page, "terminal-dictation").click();
+      await chunk;
+      await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
+      await expect(overlay).toContainText("Dictation failed: Synthetic transcription failure");
+      await overlay.getByRole("button", { name: "Retry dictation", exact: true }).click();
+      await expectReceived(terminal.id, "bracketed", "retried words ");
+      await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
+      await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
+
+      ws.failNextFinish("Synthetic transcription failure");
+      chunk = ws.nextChunk();
+      await visibleTestId(page, "terminal-dictation").click();
+      await chunk;
+      await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
+      await expect(overlay).toContainText("Dictation failed: Synthetic transcription failure");
+      await overlay.getByRole("button", { name: "Cancel dictation", exact: true }).click();
+      await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
+      await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
+      await page.waitForTimeout(1_500);
+      expect(await received(terminal.id, "bracketed")).toBe("retried words ");
+    });
+  });
+
+  for (const scenario of [
+    {
+      name: "a host without the dictation capability asks to update the host",
+      capability: "missing" as const,
+      message: "Update the host to use dictation in terminals.",
+    },
+    {
+      name: "disabled dictation with an empty reason shows the fallback message",
+      capability: { enabled: false, reason: "  " },
+      message: "Dictation is not available on this host.",
+    },
+    {
+      name: "disabled dictation shows the host's reason",
+      capability: { enabled: false, reason: "No speech model installed" },
+      message: "No speech model installed",
+    },
+  ]) {
+    test(scenario.name, async ({ page }) => {
+      const ws = await setup(page, { capability: scenario.capability });
+      const terminal = await createReceiver("bracketed", "dictation-capability");
+
+      await harness.openTerminal(page, { terminalId: terminal.id });
+      await visibleTestId(page, "terminal-toolbar-dictation").click();
+
+      await expect(page.getByTestId("app-toast-message")).toHaveText(scenario.message);
+      await expect(page.getByTestId("terminal-dictation-cancel")).toHaveCount(0);
+      expect((await microphoneStats(page)).getUserMediaCalls).toBe(0);
+      expect(ws.streamStarts()).toBe(0);
+    });
+  }
+
+  test("hiding a terminal while its microphone is still being acquired releases it", async ({
+    page,
+  }) => {
+    const ws = await setup(page);
+    const first = await createReceiver("bracketed", "dictation-held-hide-a");
+    const second = await createReceiver("bracketed", "dictation-held-hide-b");
+
+    await harness.openTerminal(page, { terminalId: first.id });
+    await holdMicrophone(page);
+    await visibleTestId(page, "terminal-toolbar-dictation").click();
+    await expectMicrophoneStat(page, "getUserMediaCalls", 1);
+    await expect(visibleTestId(page, "terminal-dictation-cancel")).toBeVisible();
+
+    await selectTerminalTab(page, second);
+    await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
+    await releaseMicrophone(page);
+
+    await expectMicrophoneStat(page, "trackStops", 1);
+    await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
+    expect(ws.streamStarts()).toBe(0);
+    expect(ws.chunks()).toBe(0);
+
+    await selectTerminalTab(page, first);
+    await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
+    await expect(
+      page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
+    ).toHaveCount(0);
+    await expect(
+      page.getByTestId("terminal-dictation-confirm").filter({ visible: true }),
+    ).toHaveCount(0);
+  });
+
+  test("closing a terminal while its microphone is still being acquired releases it", async ({
+    page,
+  }) => {
+    const ws = await setup(page);
+    const first = await createReceiver("bracketed", "dictation-held-close-a");
+    const second = await createReceiver("bracketed", "dictation-held-close-b");
+
+    await harness.openTerminal(page, { terminalId: first.id });
+    await holdMicrophone(page);
+    await visibleTestId(page, "terminal-toolbar-dictation").click();
+    await expectMicrophoneStat(page, "getUserMediaCalls", 1);
+
+    // Closing the terminal closes its tab, so its pane unmounts with the start pending.
+    await harness.killTerminal(first.id);
+    await expect(page.getByTestId(`workspace-tab-terminal_${first.id}`)).toHaveCount(0);
+    await releaseMicrophone(page);
+
+    await expectMicrophoneStat(page, "trackStops", 1);
+    await expectNothingReceived(page, [second, "bracketed"]);
+    expect(ws.streamStarts()).toBe(0);
+    expect(ws.chunks()).toBe(0);
+  });
+});

packages/app/e2e/browser/terminal-protocol-query.spec.ts

@@ -306,7 +306,11 @@ test.describe("Terminal protocol queries", () => {
     try {
       const upload = visibleTestId(page, "terminal-toolbar-upload");
       await expect(upload).toHaveCount(1);
+      // The terminal slot sits right before "...": upload, then the mic.
       expect(await controlBefore(visibleTestId(page, "workspace-split-pane-menu"))).toBe(
+        "terminal-toolbar-dictation",
+      );
+      expect(await controlBefore(visibleTestId(page, "terminal-toolbar-dictation"))).toBe(
         "terminal-toolbar-upload",
       );
 

packages/app/e2e/support/helpers/dictation-ws.ts

@@ -0,0 +1,278 @@
+import type { Page } from "@playwright/test";
+import { daemonWsRoutePattern } from "./daemon-port";
+
+type WebSocketMessage = string | Buffer;
+
+interface SessionSocket {
+  send(message: string): void;
+}
+
+/** What the rewritten `server_info` reports for dictation; `missing` drops the voice field. */
+export type DictationCapability = { enabled: boolean; reason: string } | "missing";
+
+function parseEnvelope(message: WebSocketMessage): {
+  type?: unknown;
+  message?: Record<string, unknown>;
+} | null {
+  const raw = typeof message === "string" ? message : message.toString("utf8");
+  try {
+    return JSON.parse(raw) as { type?: unknown; message?: Record<string, unknown> };
+  } catch {
+    return null;
+  }
+}
+
+export function sessionMessage(message: WebSocketMessage): Record<string, unknown> | null {
+  const envelope = parseEnvelope(message);
+  return envelope?.type === "session" && envelope.message ? envelope.message : null;
+}
+
+export function sendSessionMessage(ws: SessionSocket, message: Record<string, unknown>): void {
+  ws.send(JSON.stringify({ type: "session", message }));
+}
+
+function rewriteDictationCapability(
+  message: WebSocketMessage,
+  capability: DictationCapability,
+): WebSocketMessage {
+  const envelope = parseEnvelope(message);
+  const payload = envelope?.message?.payload;
+  if (
+    envelope?.message?.type !== "status" ||
+    !payload ||
+    typeof payload !== "object" ||
+    (payload as { status?: unknown }).status !== "server_info"
+  ) {
+    return message;
+  }
+
+  const serverInfo = payload as Record<string, unknown>;
+  if (capability === "missing") {
+    const capabilities = serverInfo.capabilities as Record<string, unknown> | undefined;
+    if (capabilities) {
+      delete capabilities.voice;
+    }
+  } else {
+    serverInfo.capabilities = {
+      voice: {
+        dictation: capability,
+        voice: { enabled: false, reason: "Realtime voice is disabled in this test." },
+      },
+    };
+  }
+  return JSON.stringify(envelope);
+}
+
+interface SyntheticMicrophoneState {
+  getUserMediaCalls: number;
+  trackStops: number;
+  hold: boolean;
+  pending: Array<() => void>;
+}
+
+/**
+ * Replaces getUserMedia with an oscillator stream, so the app's real web audio
+ * source runs end to end. Counts calls and stopped tracks, and can hold an
+ * acquisition until the test releases it (a permission prompt left open).
+ */
+export async function installSyntheticMicrophone(page: Page): Promise<void> {
+  await page.addInitScript(() => {
+    const state: SyntheticMicrophoneState = {
+      getUserMediaCalls: 0,
+      trackStops: 0,
+      hold: false,
+      pending: [],
+    };
+    (
+      window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState }
+    ).__paseoSyntheticMicrophone = state;
+    Object.defineProperty(navigator.mediaDevices, "getUserMedia", {
+      configurable: true,
+      value: async () => {
+        state.getUserMediaCalls += 1;
+        if (state.hold) {
+          await new Promise<void>((resolve) => state.pending.push(resolve));
+        }
+        const context = new AudioContext();
+        const oscillator = context.createOscillator();
+        const destination = context.createMediaStreamDestination();
+        oscillator.connect(destination);
+        oscillator.start();
+        for (const track of destination.stream.getTracks()) {
+          const stop = track.stop.bind(track);
+          track.stop = () => {
+            state.trackStops += 1;
+            stop();
+          };
+        }
+        return destination.stream;
+      },
+    });
+  });
+}
+
+export async function microphoneStats(
+  page: Page,
+): Promise<{ getUserMediaCalls: number; trackStops: number }> {
+  return await page.evaluate(() => {
+    const state = (window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState })
+      .__paseoSyntheticMicrophone;
+    return { getUserMediaCalls: state.getUserMediaCalls, trackStops: state.trackStops };
+  });
+}
+
+/** Makes the next getUserMedia calls wait until `releaseMicrophone`. */
+export async function holdMicrophone(page: Page): Promise<void> {
+  await page.evaluate(() => {
+    (
+      window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState }
+    ).__paseoSyntheticMicrophone.hold = true;
+  });
+}
+
+export async function releaseMicrophone(page: Page): Promise<void> {
+  await page.evaluate(() => {
+    const state = (window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState })
+      .__paseoSyntheticMicrophone;
+    state.hold = false;
+    for (const resolve of state.pending.splice(0)) {
+      resolve();
+    }
+  });
+}
+
+export interface DictationWsHarness {
+  /** Every `dictation_stream_start` the app sent. */
+  streamStarts(): number;
+  /** Audio chunks the app sent, across all dictation ids. */
+  chunks(): number;
+  /** Resolves on the first chunk the app sends after this call. */
+  nextChunk(): Promise<void>;
+  /** The final transcript the next finish replies with. */
+  setTranscript(text: string): void;
+  /** Keeps the next final reply until `releaseFinal`. */
+  holdFinal(): void;
+  releaseFinal(): void;
+  /** Answers the next finish with a stream error instead of a final. */
+  failNextFinish(error: string): void;
+}
+
+/**
+ * Answers the app's dictation stream in place of the daemon: acknowledges starts
+ * and chunks, accepts finishes and replies with the scenario's transcript. Rewrites
+ * `server_info` to the given dictation capability and forwards everything else, so
+ * terminals, agents and workspaces keep using the real daemon.
+ * `interceptClientMessage` lets a spec answer other requests itself; return true
+ * when it handled one.
+ */
+export async function installDictationWsHarness(
+  page: Page,
+  options: {
+    transcript?: string;
+    capability?: DictationCapability;
+    interceptClientMessage?: (request: Record<string, unknown>, ws: SessionSocket) => boolean;
+  } = {},
+): Promise<DictationWsHarness> {
+  const capability = options.capability ?? { enabled: true, reason: "" };
+  let transcript = options.transcript ?? "";
+  let starts = 0;
+  let chunkCount = 0;
+  let chunkWaiters: Array<() => void> = [];
+  let holdNextFinal = false;
+  let heldFinal: (() => void) | null = null;
+  let nextFinishError: string | null = null;
+
+  await page.routeWebSocket(daemonWsRoutePattern(), (ws) => {
+    const server = ws.connectToServer();
+
+    ws.onMessage((message) => {
+      const request = sessionMessage(message);
+      const type = request?.type;
+      const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null;
+
+      if (type === "dictation_stream_start" && dictationId) {
+        starts += 1;
+        sendSessionMessage(ws, {
+          type: "dictation_stream_ack",
+          payload: { dictationId, ackSeq: -1 },
+        });
+        return;
+      }
+      if (type === "dictation_stream_chunk" && dictationId) {
+        chunkCount += 1;
+        const seq = typeof request?.seq === "number" ? request.seq : 0;
+        sendSessionMessage(ws, {
+          type: "dictation_stream_ack",
+          payload: { dictationId, ackSeq: seq },
+        });
+        const waiters = chunkWaiters;
+        chunkWaiters = [];
+        for (const resolve of waiters) {
+          resolve();
+        }
+        return;
+      }
+      if (type === "dictation_stream_finish" && dictationId) {
+        sendSessionMessage(ws, {
+          type: "dictation_stream_finish_accepted",
+          payload: { dictationId, timeoutMs: 30_000 },
+        });
+        if (nextFinishError !== null) {
+          const error = nextFinishError;
+          nextFinishError = null;
+          sendSessionMessage(ws, {
+            type: "dictation_stream_error",
+            payload: { dictationId, error, retryable: true },
+          });
+          return;
+        }
+        const text = transcript;
+        const reply = () =>
+          sendSessionMessage(ws, {
+            type: "dictation_stream_final",
+            payload: { dictationId, text },
+          });
+        if (holdNextFinal) {
+          holdNextFinal = false;
+          heldFinal = reply;
+        } else {
+          reply();
+        }
+        return;
+      }
+      if (type === "dictation_stream_cancel") {
+        return;
+      }
+      if (request && options.interceptClientMessage?.(request, ws)) {
+        return;
+      }
+
+      server.send(message);
+    });
+
+    server.onMessage((message) => ws.send(rewriteDictationCapability(message, capability)));
+  });
+
+  return {
+    streamStarts: () => starts,
+    chunks: () => chunkCount,
+    nextChunk: () => new Promise<void>((resolve) => chunkWaiters.push(resolve)),
+    setTranscript: (text) => {
+      transcript = text;
+    },
+    holdFinal: () => {
+      holdNextFinal = true;
+    },
+    releaseFinal: () => {
+      const reply = heldFinal;
+      heldFinal = null;
+      if (!reply) {
+        throw new Error("No final reply is being held");
+      }
+      reply();
+    },
+    failNextFinish: (error) => {
+      nextFinishError = error;
+    },
+  };
+}