From 113dd83d95d3ec40aaa7eadba26f266dfd6dfdb3 Mon Sep 17 00:00:00 2001 From: Sebastian Arena Date: Mon, 26 Oct 2026 21:59:03 -0400 Subject: [PATCH] M2: terminal dictation: mic in tab-bar slot and key bar, recording pill and overlay, per-terminal recording lifetime incl. pending mic acquisition, Cmd+D on macOS by keyboard focus, i18n --- .../new-workspace-dictation-submit.spec.ts | 146 +---- .../e2e/browser/terminal-dictation.spec.ts | 598 ++++++++++++++++++ .../browser/terminal-protocol-query.spec.ts | 4 + .../app/e2e/support/helpers/dictation-ws.ts | 278 ++++++++ .../app/src/components/dictation-controls.tsx | 33 +- packages/app/src/components/terminal-pane.tsx | 75 ++- .../hooks/use-dictation-audio-source.web.ts | 21 +- packages/app/src/hooks/use-dictation.test.tsx | 203 ++++++ packages/app/src/hooks/use-dictation.ts | 63 +- packages/app/src/i18n/resources/ar.ts | 8 + packages/app/src/i18n/resources/en.ts | 8 + packages/app/src/i18n/resources/es.ts | 8 + packages/app/src/i18n/resources/fr.ts | 8 + packages/app/src/i18n/resources/ja.ts | 8 + packages/app/src/i18n/resources/ko.ts | 8 + packages/app/src/i18n/resources/pt-BR.ts | 8 + packages/app/src/i18n/resources/ru.ts | 8 + packages/app/src/i18n/resources/zh-CN.ts | 8 + packages/app/src/keyboard/actions.ts | 1 + .../keyboard/keyboard-action-dispatcher.ts | 2 + .../src/keyboard/keyboard-shortcuts.test.ts | 15 +- .../app/src/keyboard/keyboard-shortcuts.ts | 14 + .../app/src/keyboard/route-shortcut.test.ts | 1 + packages/app/src/keyboard/route-shortcut.ts | 1 + .../workspace/workspace-desktop-tabs-row.tsx | 5 +- .../format-terminal-transcript.test.ts | 18 + .../dictation/format-terminal-transcript.ts | 10 + .../terminal-dictation-availability.test.ts | 70 ++ .../terminal-dictation-availability.ts | 35 + .../terminal-dictation-lifecycle.test.ts | 89 +++ .../dictation/terminal-dictation-lifecycle.ts | 39 ++ .../dictation/terminal-dictation-owner.tsx | 198 ++++++ .../src/terminal/input-sink/index.test.tsx | 76 ++- packages/app/src/terminal/input-sink/index.ts | 88 ++- .../runtime/terminal-virtual-keyboard.test.ts | 14 +- .../runtime/terminal-virtual-keyboard.ts | 13 +- .../toolbar/terminal-dictation-control.tsx | 160 +++++ .../toolbar/terminal-key-bar-dictation.tsx | 66 ++ .../toolbar/terminal-toolbar-slot.tsx | 24 +- .../src/utils/format-dictation-duration.ts | 6 + 40 files changed, 2264 insertions(+), 174 deletions(-) create mode 100644 packages/app/e2e/browser/terminal-dictation.spec.ts create mode 100644 packages/app/e2e/support/helpers/dictation-ws.ts create mode 100644 packages/app/src/hooks/use-dictation.test.tsx create mode 100644 packages/app/src/terminal/dictation/format-terminal-transcript.test.ts create mode 100644 packages/app/src/terminal/dictation/format-terminal-transcript.ts create mode 100644 packages/app/src/terminal/dictation/terminal-dictation-availability.test.ts create mode 100644 packages/app/src/terminal/dictation/terminal-dictation-availability.ts create mode 100644 packages/app/src/terminal/dictation/terminal-dictation-lifecycle.test.ts create mode 100644 packages/app/src/terminal/dictation/terminal-dictation-lifecycle.ts create mode 100644 packages/app/src/terminal/dictation/terminal-dictation-owner.tsx create mode 100644 packages/app/src/terminal/toolbar/terminal-dictation-control.tsx create mode 100644 packages/app/src/terminal/toolbar/terminal-key-bar-dictation.tsx create mode 100644 packages/app/src/utils/format-dictation-duration.ts diff --git a/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts b/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts index 22b2b8a6c5..cf27d2b367 100644 --- a/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts +++ b/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts @@ -1,6 +1,10 @@ import { expect, test, type Page } from "../support/fixtures"; import { gotoAppShell } from "../support/helpers/app"; -import { daemonWsRoutePattern } from "../support/helpers/daemon-port"; +import { + installDictationWsHarness, + installSyntheticMicrophone, + sendSessionMessage, +} from "../support/helpers/dictation-ws"; import { openNewWorkspaceComposer, selectWorkspaceIsolation, @@ -17,141 +21,39 @@ import { } from "../support/helpers/composer"; import { waitForSidebarHydration } from "../support/helpers/workspace-ui"; -type WebSocketMessage = string | Buffer; - const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace"; const CREATE_FAILURE = "Synthetic workspace creation failure"; -function parseEnvelope(message: WebSocketMessage): { - type?: unknown; - message?: Record; -} | null { - const raw = typeof message === "string" ? message : message.toString("utf8"); - try { - return JSON.parse(raw) as { type?: unknown; message?: Record }; - } catch { - return null; - } -} - -function sessionMessage(message: WebSocketMessage): Record | null { - const envelope = parseEnvelope(message); - return envelope?.type === "session" && envelope.message ? envelope.message : null; -} - -function sendSessionMessage( - ws: { send(message: string): void }, - message: Record, -): void { - ws.send(JSON.stringify({ type: "session", message })); -} - -function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { - const envelope = parseEnvelope(message); - const payload = envelope?.message?.payload; - if ( - envelope?.message?.type !== "status" || - !payload || - typeof payload !== "object" || - (payload as { status?: unknown }).status !== "server_info" - ) { - return message; - } - - (payload as Record).capabilities = { - voice: { - dictation: { enabled: true, reason: "" }, - voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, - }, - }; - return JSON.stringify(envelope); -} - -async function installSyntheticMicrophone(page: Page): Promise { - await page.addInitScript(() => { - const mediaDevices = navigator.mediaDevices; - Object.defineProperty(mediaDevices, "getUserMedia", { - configurable: true, - value: async () => { - const context = new AudioContext(); - const oscillator = context.createOscillator(); - const destination = context.createMediaStreamDestination(); - oscillator.connect(destination); - oscillator.start(); - return destination.stream; - }, - }); - }); -} - async function installDictationFailureHarness(page: Page) { - let resolveAudioChunk!: () => void; let resolveCreateRequest!: () => void; - const audioChunk = new Promise((resolve) => { - resolveAudioChunk = resolve; - }); const createRequest = new Promise((resolve) => { resolveCreateRequest = resolve; }); let failCreate: (() => void) | null = null; - await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { - const server = ws.connectToServer(); - - ws.onMessage((message) => { - const request = sessionMessage(message); - const type = request?.type; - const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null; - - if (type === "dictation_stream_start" && dictationId) { - sendSessionMessage(ws, { - type: "dictation_stream_ack", - payload: { dictationId, ackSeq: -1 }, - }); - return; + const dictation = await installDictationWsHarness(page, { + transcript: TRANSCRIPT, + interceptClientMessage: (request, ws) => { + if (request.type !== "workspace.create.request" || typeof request.requestId !== "string") { + return false; } - if (type === "dictation_stream_chunk" && dictationId) { - const seq = typeof request?.seq === "number" ? request.seq : 0; + const requestId = request.requestId; + failCreate = () => { sendSessionMessage(ws, { - type: "dictation_stream_ack", - payload: { dictationId, ackSeq: seq }, + type: "workspace.create.response", + payload: { + requestId, + workspace: null, + setupTerminalId: null, + error: CREATE_FAILURE, + }, }); - resolveAudioChunk(); - return; - } - if (type === "dictation_stream_finish" && dictationId) { - sendSessionMessage(ws, { - type: "dictation_stream_finish_accepted", - payload: { dictationId, timeoutMs: 5_000 }, - }); - sendSessionMessage(ws, { - type: "dictation_stream_final", - payload: { dictationId, text: TRANSCRIPT }, - }); - return; - } - if (request && type === "workspace.create.request" && typeof request.requestId === "string") { - const requestId = request.requestId; - failCreate = () => { - sendSessionMessage(ws, { - type: "workspace.create.response", - payload: { - requestId, - workspace: null, - setupTerminalId: null, - error: CREATE_FAILURE, - }, - }); - }; - resolveCreateRequest(); - return; - } - - server.send(message); - }); - - server.onMessage((message) => ws.send(enableDictationCapability(message))); + }; + resolveCreateRequest(); + return true; + }, }); + const audioChunk = dictation.nextChunk(); return { waitForAudio: () => audioChunk, diff --git a/packages/app/e2e/browser/terminal-dictation.spec.ts b/packages/app/e2e/browser/terminal-dictation.spec.ts new file mode 100644 index 0000000000..977e8bd827 --- /dev/null +++ b/packages/app/e2e/browser/terminal-dictation.spec.ts @@ -0,0 +1,598 @@ +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { Locator, Page } from "@playwright/test"; +import { expect, test } from "../support/fixtures"; +import { runWorkspaceActionFromCommandCenter } from "../support/helpers/command-center-workspace-actions"; +import { + holdMicrophone, + installDictationWsHarness, + installSyntheticMicrophone, + microphoneStats, + releaseMicrophone, + type DictationCapability, + type DictationWsHarness, +} from "../support/helpers/dictation-ws"; +import { TerminalE2EHarness, type TerminalInstance } from "../support/helpers/terminal-dsl"; +import { ensureExplorerSidebar } from "../support/helpers/workspace-tabs"; + +const MAC_USER_AGENT = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"; +const LINUX_USER_AGENT = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"; + +// Payloads are echoed as JSON with whitespace escaped, because the PTY capture +// below drops whitespace when it joins wrapped lines. +const ENCODE = String.raw` +const encode = (text) => + JSON.stringify(text).replace( + /\s/g, + (space) => "\\u" + space.charCodeAt(0).toString(16).padStart(4, "0"), + ); +`; + +// A TUI with bracketed paste on: echoes each paste body as INPUT:. +const BRACKETED_RECEIVER_SCRIPT = String.raw` +${ENCODE} +let buffer = ""; +process.stdin.setRawMode(true); +process.stdin.resume(); +process.stdout.write("\x1b[?2004hDICTATION_RECEIVER_READY\r\n"); +process.stdin.on("data", (chunk) => { + buffer += chunk.toString(); + for (;;) { + const paste = /\x1b\[200~([^]*?)\x1b\[201~/.exec(buffer); + if (!paste) break; + process.stdout.write("INPUT:" + encode(paste[1]) + "\r\n"); + buffer = buffer.slice(0, paste.index) + buffer.slice(paste.index + paste[0].length); + } +}); +`; + +// A program without bracketed paste: echoes every stdin byte, in raw mode so +// Ctrl+D arrives as \x04, as RAW_INPUT: after a quiet period. +const RAW_RECEIVER_SCRIPT = String.raw` +${ENCODE} +let pending = ""; +let timer = null; +process.stdin.setRawMode(true); +process.stdin.resume(); +process.stdout.write("RAW_RECEIVER_READY\r\n"); +process.stdin.on("data", (chunk) => { + pending += chunk.toString(); + clearTimeout(timer); + timer = setTimeout(() => { + process.stdout.write("RAW_INPUT:" + encode(pending) + "\r\n"); + pending = ""; + }, 200); +}); +`; + +type ReceiverKind = "bracketed" | "raw"; + +function visibleTestId(scope: Page | Locator, testId: string): Locator { + return scope.getByTestId(testId).filter({ visible: true }).first(); +} + +// Click the surface, not its helper textarea: the textarea sits at the left edge, +// under a split or Explorer resize handle. +function terminalSurface(scope: Page | Locator): Locator { + return scope.getByTestId("terminal-surface").filter({ visible: true }); +} + +function terminalInput(scope: Page | Locator): Locator { + return scope + .getByTestId("terminal-surface") + .filter({ visible: true }) + .locator(".xterm-helper-textarea"); +} + +function splitChild(page: Page, terminal: TerminalInstance): Locator { + return page + .getByTestId("split-group-child") + .filter({ has: page.getByTestId(`workspace-tab-terminal_${terminal.id}`) }); +} + +function explorerSidebar(page: Page): Locator { + return page.getByTestId("workspace-explorer-sidebar").filter({ visible: true }).first(); +} + +async function dragTabIntoExplorer(page: Page, terminal: TerminalInstance): Promise { + const source = page + .getByTestId("workspace-pane-main") + .getByTestId(`workspace-tab-terminal_${terminal.id}`); + const destination = explorerSidebar(page).getByTestId("workspace-tab-files"); + const sourceBox = await source.boundingBox(); + const destinationBox = await destination.boundingBox(); + if (!sourceBox || !destinationBox) { + throw new Error("Tab drag requires visible source and destination"); + } + const start = { x: sourceBox.x + 12, y: sourceBox.y + sourceBox.height / 2 }; + await page.mouse.move(start.x, start.y); + await page.mouse.down(); + await page.mouse.move(start.x + 12, start.y + 4); + await page.mouse.move(destinationBox.x + 12, destinationBox.y + destinationBox.height / 2, { + steps: 20, + }); + await page.mouse.up(); +} + +async function expectMicrophoneStat( + page: Page, + stat: "trackStops" | "getUserMediaCalls", + count: number, +): Promise { + await expect.poll(async () => (await microphoneStats(page))[stat]).toBe(count); +} + +test.describe("Terminal dictation", () => { + test.describe.configure({ timeout: 180_000 }); + + let harness: TerminalE2EHarness; + + test.beforeAll(async () => { + harness = await TerminalE2EHarness.create({ tempPrefix: "terminal-dictation-" }); + await writeFile( + path.join(harness.tempRepo.path, "bracketed-receiver.cjs"), + BRACKETED_RECEIVER_SCRIPT, + ); + await writeFile(path.join(harness.tempRepo.path, "raw-receiver.cjs"), RAW_RECEIVER_SCRIPT); + }); + + test.afterAll(async () => { + await harness?.cleanup(); + }); + + const terminals: TerminalInstance[] = []; + + test.afterEach(async () => { + for (const terminal of terminals.splice(0)) { + await harness.killTerminal(terminal.id); + } + }); + + async function createReceiver(kind: ReceiverKind, name: string): Promise { + const terminal = await harness.createTerminal({ + name, + command: process.execPath, + args: [kind === "bracketed" ? "bracketed-receiver.cjs" : "raw-receiver.cjs"], + }); + terminals.push(terminal); + await expectTerminalOutput( + terminal.id, + kind === "bracketed" ? "DICTATION_RECEIVER_READY" : "RAW_RECEIVER_READY", + ); + return terminal; + } + + async function setup( + page: Page, + options: { transcript?: string; capability?: DictationCapability } = {}, + ): Promise { + await installSyntheticMicrophone(page); + return await installDictationWsHarness(page, { + transcript: options.transcript ?? "first\nsecond", + capability: options.capability, + }); + } + + // Reads the PTY itself, so output from one pane cannot satisfy another's check. + async function terminalOutput(terminalId: string): Promise { + const capture = await harness.client.captureTerminal(terminalId); + return capture.lines.join("").replace(/\s/g, ""); + } + + async function expectTerminalOutput(terminalId: string, text: string): Promise { + await expect.poll(() => terminalOutput(terminalId), { timeout: 10_000 }).toContain(text); + } + + /** Everything the receiver got, in order and joined. */ + async function received(terminalId: string, kind: ReceiverKind): Promise { + const label = kind === "bracketed" ? "INPUT" : "RAW_INPUT"; + const output = await terminalOutput(terminalId); + return [ + ...output.matchAll( + new RegExp( + `${kind === "bracketed" ? "(? JSON.parse(json) as string) + .join(""); + } + + async function expectReceived( + terminalId: string, + kind: ReceiverKind, + text: string, + ): Promise { + await expect.poll(() => received(terminalId, kind), { timeout: 15_000 }).toBe(text); + } + + /** Gives a late transcript time to land, then checks that none did. */ + async function expectNothingReceived( + page: Page, + ...receivers: Array<[TerminalInstance, ReceiverKind]> + ): Promise { + await page.waitForTimeout(1_500); + for (const [terminal, kind] of receivers) { + expect(await received(terminal.id, kind)).toBe(""); + } + } + + async function startFromToolbar( + page: Page, + ws: DictationWsHarness, + scope: Page | Locator = page, + ) { + const chunk = ws.nextChunk(); + await visibleTestId(scope, "terminal-toolbar-dictation").click(); + await chunk; + } + + // The route helper needs the desktop tab row, so open there, then go compact. + async function openCompact(page: Page, terminal: TerminalInstance) { + await harness.openTerminal(page, { terminalId: terminal.id }); + await page.setViewportSize({ width: 390, height: 844 }); + await expect(visibleTestId(page, "terminal-dictation")).toBeVisible(); + } + + // The compact switcher labels terminals by their command, so pick by position. + async function selectCompactTab(page: Page, index: number) { + await page + .getByRole("button", { name: /Switch tabs/ }) + .filter({ visible: true }) + .click(); + await expect(page.getByText("Workspace tabs", { exact: true })).toBeVisible(); + const options = page + .getByText("bracketed-receiver.cjs", { exact: true }) + .filter({ visible: true }); + await expect(options).toHaveCount(2); + await options.nth(index).click(); + await expect(page.getByRole("button", { name: "Bottom sheet backdrop" })).toHaveCount(0); + } + + async function selectTerminalTab(scope: Page | Locator, terminal: TerminalInstance) { + await visibleTestId(scope, `workspace-tab-terminal_${terminal.id}`).click(); + } + + test("confirm pastes the transcript on one line with a trailing space and no Enter", async ({ + page, + }) => { + const ws = await setup(page); + const bracketed = await createReceiver("bracketed", "dictation-bracketed"); + const raw = await createReceiver("raw", "dictation-raw"); + + await harness.openTerminal(page, { terminalId: bracketed.id }); + await startFromToolbar(page, ws); + await visibleTestId(page, "terminal-dictation-confirm").click(); + await expectReceived(bracketed.id, "bracketed", "first second "); + await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible(); + + await selectTerminalTab(page, raw); + await startFromToolbar(page, ws); + await visibleTestId(page, "terminal-dictation-confirm").click(); + await expectReceived(raw.id, "raw", "first second "); + expect(await received(bracketed.id, "bracketed")).toBe("first second "); + }); + + test.describe("on macOS", () => { + test.use({ userAgent: MAC_USER_AGENT }); + + test("Cmd+D starts and then confirms dictation in the focused terminal", async ({ page }) => { + const ws = await setup(page, { transcript: "shortcut words" }); + const terminal = await createReceiver("bracketed", "dictation-shortcut"); + + await harness.openTerminal(page, { terminalId: terminal.id }); + const chunk = ws.nextChunk(); + await terminalInput(page).press("Meta+d"); + await expect(visibleTestId(page, "terminal-dictation-confirm")).toBeVisible(); + await chunk; + await terminalInput(page).press("Meta+d"); + + await expectReceived(terminal.id, "bracketed", "shortcut words "); + await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible(); + }); + + test("Cmd+D goes to the terminal with keyboard focus, main pane or Explorer", async ({ + page, + }) => { + const ws = await setup(page, { transcript: "main pane words" }); + const explorerTerminal = await createReceiver("bracketed", "dictation-explorer"); + const mainTerminal = await createReceiver("bracketed", "dictation-main"); + + await harness.openTerminal(page, { terminalId: explorerTerminal.id }); + await ensureExplorerSidebar(page); + await dragTabIntoExplorer(page, explorerTerminal); + const explorer = explorerSidebar(page); + await expect( + explorer.getByTestId(`workspace-tab-terminal_${explorerTerminal.id}`), + ).toBeVisible(); + const main = page.getByTestId("workspace-pane-main"); + await selectTerminalTab(main, mainTerminal); + await expect(explorer.getByTestId("terminal-surface")).toBeVisible(); + + let chunk = ws.nextChunk(); + await terminalSurface(main).click(); + await terminalInput(main).press("Meta+d"); + await expect(visibleTestId(main, "terminal-dictation-confirm")).toBeVisible(); + await expect(explorer.getByTestId("terminal-dictation-confirm")).toHaveCount(0); + await chunk; + await terminalInput(main).press("Meta+d"); + await expectReceived(mainTerminal.id, "bracketed", "main pane words "); + await expectNothingReceived(page, [explorerTerminal, "bracketed"]); + + ws.setTranscript("explorer words"); + chunk = ws.nextChunk(); + await terminalSurface(explorer).click(); + await terminalInput(explorer).press("Meta+d"); + await expect(visibleTestId(explorer, "terminal-dictation-confirm")).toBeVisible(); + await expect(main.getByTestId("terminal-dictation-confirm")).toHaveCount(0); + await chunk; + await terminalInput(explorer).press("Meta+d"); + await expectReceived(explorerTerminal.id, "bracketed", "explorer words "); + expect(await received(mainTerminal.id, "bracketed")).toBe("main pane words "); + }); + + test("Cmd+D records only in the focused split pane", async ({ page }) => { + const ws = await setup(page, { transcript: "right pane words" }); + const left = await createReceiver("bracketed", "dictation-split-left"); + const right = await createReceiver("bracketed", "dictation-split-right"); + + await harness.openTerminal(page, { terminalId: left.id }); + await runWorkspaceActionFromCommandCenter(page, "Split pane right"); + await visibleTestId(page, `workspace-tab-terminal_${right.id}`).click(); + await runWorkspaceActionFromCommandCenter(page, "Move tab right"); + const leftPane = splitChild(page, left); + const rightPane = splitChild(page, right); + await expect(terminalInput(leftPane)).toHaveCount(1); + await expect(terminalInput(rightPane)).toHaveCount(1); + + const chunk = ws.nextChunk(); + await terminalSurface(rightPane).click(); + await terminalInput(rightPane).press("Meta+d"); + await expect(visibleTestId(rightPane, "terminal-dictation-confirm")).toBeVisible(); + await expect(leftPane.getByTestId("terminal-dictation-confirm")).toHaveCount(0); + await chunk; + await terminalInput(rightPane).press("Meta+d"); + + await expectReceived(right.id, "bracketed", "right pane words "); + await expectNothingReceived(page, [left, "bracketed"]); + }); + }); + + test("Ctrl+D off macOS reaches the PTY as EOF and starts nothing", async ({ page }) => { + // Chromium keeps the host's navigator.platform unless the override sets it too. + const session = await page.context().newCDPSession(page); + await session.send("Emulation.setUserAgentOverride", { + userAgent: LINUX_USER_AGENT, + platform: "Linux x86_64", + }); + const ws = await setup(page); + const terminal = await createReceiver("raw", "dictation-ctrl-d"); + + await harness.openTerminal(page, { terminalId: terminal.id }); + await terminalInput(page).press("Control+d"); + + await expectReceived(terminal.id, "raw", "\u0004"); + expect(ws.streamStarts()).toBe(0); + expect((await microphoneStats(page)).getUserMediaCalls).toBe(0); + await expect(page.getByTestId("terminal-dictation-confirm")).toHaveCount(0); + }); + + test("switching the pane to another terminal while recording stops it and delivers nothing", async ({ + page, + }) => { + const ws = await setup(page); + const first = await createReceiver("bracketed", "dictation-switch-a"); + const second = await createReceiver("bracketed", "dictation-switch-b"); + + await harness.openTerminal(page, { terminalId: first.id }); + await startFromToolbar(page, ws); + await selectTerminalTab(page, second); + + await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible(); + await expect( + page.getByTestId("terminal-dictation-cancel").filter({ visible: true }), + ).toHaveCount(0); + await expectMicrophoneStat(page, "trackStops", 1); + const chunksAfterSwitch = ws.chunks(); + const startsAfterSwitch = ws.streamStarts(); + await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]); + expect(ws.chunks()).toBe(chunksAfterSwitch); + expect(ws.streamStarts()).toBe(startsAfterSwitch); + + await selectTerminalTab(page, first); + await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible(); + await expect( + page.getByTestId("terminal-dictation-cancel").filter({ visible: true }), + ).toHaveCount(0); + }); + + test("switching the pane to another terminal while processing drops the late transcript", async ({ + page, + }) => { + const ws = await setup(page); + const first = await createReceiver("bracketed", "dictation-processing-a"); + const second = await createReceiver("bracketed", "dictation-processing-b"); + + await harness.openTerminal(page, { terminalId: first.id }); + await startFromToolbar(page, ws); + ws.holdFinal(); + await visibleTestId(page, "terminal-dictation-confirm").click(); + await expect(visibleTestId(page, "terminal-dictation-confirm")).toHaveCount(0); + await expect(visibleTestId(page, "terminal-dictation-cancel")).toBeVisible(); + await selectTerminalTab(page, second); + await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible(); + ws.releaseFinal(); + + await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]); + }); + + test.describe("on compact web", () => { + // Keyboard shortcuts are off on compact form factors (keyboardShortcutsAvailable), so + // the key-bar mic is how compact web dictates. + test("the key-bar mic shows the overlay and confirm delivers", async ({ page }) => { + const ws = await setup(page, { transcript: "compact words" }); + const terminal = await createReceiver("bracketed", "dictation-compact-confirm"); + + await openCompact(page, terminal); + const chunk = ws.nextChunk(); + await visibleTestId(page, "terminal-dictation").click(); + const overlay = visibleTestId(page, "terminal-dictation-overlay"); + await expect(overlay).toBeVisible(); + await expect(visibleTestId(page, "terminal-dictation")).toHaveCount(0); + await expect( + overlay.getByRole("button", { name: "Insert transcription and send", exact: true }), + ).toHaveCount(0); + await chunk; + await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click(); + + await expectReceived(terminal.id, "bracketed", "compact words "); + await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0); + await expect(visibleTestId(page, "terminal-dictation")).toBeVisible(); + }); + + test("switching tabs while recording from the key bar stops it and delivers nothing", async ({ + page, + }) => { + const ws = await setup(page); + const first = await createReceiver("bracketed", "dictation-compact-a"); + const second = await createReceiver("bracketed", "dictation-compact-b"); + + await openCompact(page, first); + const chunk = ws.nextChunk(); + await visibleTestId(page, "terminal-dictation").click(); + await expect(visibleTestId(page, "terminal-dictation-overlay")).toBeVisible(); + await chunk; + + await selectCompactTab(page, 1); + await expectMicrophoneStat(page, "trackStops", 1); + await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]); + + await selectCompactTab(page, 0); + await expect(visibleTestId(page, "terminal-dictation")).toBeVisible(); + await expect( + page.getByTestId("terminal-dictation-overlay").filter({ visible: true }), + ).toHaveCount(0); + expect(await received(first.id, "bracketed")).toBe(""); + }); + + test("a failed dictation keeps the overlay with retry and discard", async ({ page }) => { + const ws = await setup(page, { transcript: "retried words" }); + const terminal = await createReceiver("bracketed", "dictation-compact-failure"); + + await openCompact(page, terminal); + const overlay = visibleTestId(page, "terminal-dictation-overlay"); + + ws.failNextFinish("Synthetic transcription failure"); + let chunk = ws.nextChunk(); + await visibleTestId(page, "terminal-dictation").click(); + await chunk; + await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click(); + await expect(overlay).toContainText("Dictation failed: Synthetic transcription failure"); + await overlay.getByRole("button", { name: "Retry dictation", exact: true }).click(); + await expectReceived(terminal.id, "bracketed", "retried words "); + await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0); + await expect(visibleTestId(page, "terminal-dictation")).toBeVisible(); + + ws.failNextFinish("Synthetic transcription failure"); + chunk = ws.nextChunk(); + await visibleTestId(page, "terminal-dictation").click(); + await chunk; + await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click(); + await expect(overlay).toContainText("Dictation failed: Synthetic transcription failure"); + await overlay.getByRole("button", { name: "Cancel dictation", exact: true }).click(); + await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0); + await expect(visibleTestId(page, "terminal-dictation")).toBeVisible(); + await page.waitForTimeout(1_500); + expect(await received(terminal.id, "bracketed")).toBe("retried words "); + }); + }); + + for (const scenario of [ + { + name: "a host without the dictation capability asks to update the host", + capability: "missing" as const, + message: "Update the host to use dictation in terminals.", + }, + { + name: "disabled dictation with an empty reason shows the fallback message", + capability: { enabled: false, reason: " " }, + message: "Dictation is not available on this host.", + }, + { + name: "disabled dictation shows the host's reason", + capability: { enabled: false, reason: "No speech model installed" }, + message: "No speech model installed", + }, + ]) { + test(scenario.name, async ({ page }) => { + const ws = await setup(page, { capability: scenario.capability }); + const terminal = await createReceiver("bracketed", "dictation-capability"); + + await harness.openTerminal(page, { terminalId: terminal.id }); + await visibleTestId(page, "terminal-toolbar-dictation").click(); + + await expect(page.getByTestId("app-toast-message")).toHaveText(scenario.message); + await expect(page.getByTestId("terminal-dictation-cancel")).toHaveCount(0); + expect((await microphoneStats(page)).getUserMediaCalls).toBe(0); + expect(ws.streamStarts()).toBe(0); + }); + } + + test("hiding a terminal while its microphone is still being acquired releases it", async ({ + page, + }) => { + const ws = await setup(page); + const first = await createReceiver("bracketed", "dictation-held-hide-a"); + const second = await createReceiver("bracketed", "dictation-held-hide-b"); + + await harness.openTerminal(page, { terminalId: first.id }); + await holdMicrophone(page); + await visibleTestId(page, "terminal-toolbar-dictation").click(); + await expectMicrophoneStat(page, "getUserMediaCalls", 1); + await expect(visibleTestId(page, "terminal-dictation-cancel")).toBeVisible(); + + await selectTerminalTab(page, second); + await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible(); + await releaseMicrophone(page); + + await expectMicrophoneStat(page, "trackStops", 1); + await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]); + expect(ws.streamStarts()).toBe(0); + expect(ws.chunks()).toBe(0); + + await selectTerminalTab(page, first); + await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible(); + await expect( + page.getByTestId("terminal-dictation-cancel").filter({ visible: true }), + ).toHaveCount(0); + await expect( + page.getByTestId("terminal-dictation-confirm").filter({ visible: true }), + ).toHaveCount(0); + }); + + test("closing a terminal while its microphone is still being acquired releases it", async ({ + page, + }) => { + const ws = await setup(page); + const first = await createReceiver("bracketed", "dictation-held-close-a"); + const second = await createReceiver("bracketed", "dictation-held-close-b"); + + await harness.openTerminal(page, { terminalId: first.id }); + await holdMicrophone(page); + await visibleTestId(page, "terminal-toolbar-dictation").click(); + await expectMicrophoneStat(page, "getUserMediaCalls", 1); + + // Closing the terminal closes its tab, so its pane unmounts with the start pending. + await harness.killTerminal(first.id); + await expect(page.getByTestId(`workspace-tab-terminal_${first.id}`)).toHaveCount(0); + await releaseMicrophone(page); + + await expectMicrophoneStat(page, "trackStops", 1); + await expectNothingReceived(page, [second, "bracketed"]); + expect(ws.streamStarts()).toBe(0); + expect(ws.chunks()).toBe(0); + }); +}); diff --git a/packages/app/e2e/browser/terminal-protocol-query.spec.ts b/packages/app/e2e/browser/terminal-protocol-query.spec.ts index 1ce18155ea..11d45dc21f 100644 --- a/packages/app/e2e/browser/terminal-protocol-query.spec.ts +++ b/packages/app/e2e/browser/terminal-protocol-query.spec.ts @@ -306,7 +306,11 @@ test.describe("Terminal protocol queries", () => { try { const upload = visibleTestId(page, "terminal-toolbar-upload"); await expect(upload).toHaveCount(1); + // The terminal slot sits right before "...": upload, then the mic. expect(await controlBefore(visibleTestId(page, "workspace-split-pane-menu"))).toBe( + "terminal-toolbar-dictation", + ); + expect(await controlBefore(visibleTestId(page, "terminal-toolbar-dictation"))).toBe( "terminal-toolbar-upload", ); diff --git a/packages/app/e2e/support/helpers/dictation-ws.ts b/packages/app/e2e/support/helpers/dictation-ws.ts new file mode 100644 index 0000000000..1496c13052 --- /dev/null +++ b/packages/app/e2e/support/helpers/dictation-ws.ts @@ -0,0 +1,278 @@ +import type { Page } from "@playwright/test"; +import { daemonWsRoutePattern } from "./daemon-port"; + +type WebSocketMessage = string | Buffer; + +interface SessionSocket { + send(message: string): void; +} + +/** What the rewritten `server_info` reports for dictation; `missing` drops the voice field. */ +export type DictationCapability = { enabled: boolean; reason: string } | "missing"; + +function parseEnvelope(message: WebSocketMessage): { + type?: unknown; + message?: Record; +} | null { + const raw = typeof message === "string" ? message : message.toString("utf8"); + try { + return JSON.parse(raw) as { type?: unknown; message?: Record }; + } catch { + return null; + } +} + +export function sessionMessage(message: WebSocketMessage): Record | null { + const envelope = parseEnvelope(message); + return envelope?.type === "session" && envelope.message ? envelope.message : null; +} + +export function sendSessionMessage(ws: SessionSocket, message: Record): void { + ws.send(JSON.stringify({ type: "session", message })); +} + +function rewriteDictationCapability( + message: WebSocketMessage, + capability: DictationCapability, +): WebSocketMessage { + const envelope = parseEnvelope(message); + const payload = envelope?.message?.payload; + if ( + envelope?.message?.type !== "status" || + !payload || + typeof payload !== "object" || + (payload as { status?: unknown }).status !== "server_info" + ) { + return message; + } + + const serverInfo = payload as Record; + if (capability === "missing") { + const capabilities = serverInfo.capabilities as Record | undefined; + if (capabilities) { + delete capabilities.voice; + } + } else { + serverInfo.capabilities = { + voice: { + dictation: capability, + voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, + }, + }; + } + return JSON.stringify(envelope); +} + +interface SyntheticMicrophoneState { + getUserMediaCalls: number; + trackStops: number; + hold: boolean; + pending: Array<() => void>; +} + +/** + * Replaces getUserMedia with an oscillator stream, so the app's real web audio + * source runs end to end. Counts calls and stopped tracks, and can hold an + * acquisition until the test releases it (a permission prompt left open). + */ +export async function installSyntheticMicrophone(page: Page): Promise { + await page.addInitScript(() => { + const state: SyntheticMicrophoneState = { + getUserMediaCalls: 0, + trackStops: 0, + hold: false, + pending: [], + }; + ( + window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState } + ).__paseoSyntheticMicrophone = state; + Object.defineProperty(navigator.mediaDevices, "getUserMedia", { + configurable: true, + value: async () => { + state.getUserMediaCalls += 1; + if (state.hold) { + await new Promise((resolve) => state.pending.push(resolve)); + } + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const destination = context.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + for (const track of destination.stream.getTracks()) { + const stop = track.stop.bind(track); + track.stop = () => { + state.trackStops += 1; + stop(); + }; + } + return destination.stream; + }, + }); + }); +} + +export async function microphoneStats( + page: Page, +): Promise<{ getUserMediaCalls: number; trackStops: number }> { + return await page.evaluate(() => { + const state = (window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState }) + .__paseoSyntheticMicrophone; + return { getUserMediaCalls: state.getUserMediaCalls, trackStops: state.trackStops }; + }); +} + +/** Makes the next getUserMedia calls wait until `releaseMicrophone`. */ +export async function holdMicrophone(page: Page): Promise { + await page.evaluate(() => { + ( + window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState } + ).__paseoSyntheticMicrophone.hold = true; + }); +} + +export async function releaseMicrophone(page: Page): Promise { + await page.evaluate(() => { + const state = (window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState }) + .__paseoSyntheticMicrophone; + state.hold = false; + for (const resolve of state.pending.splice(0)) { + resolve(); + } + }); +} + +export interface DictationWsHarness { + /** Every `dictation_stream_start` the app sent. */ + streamStarts(): number; + /** Audio chunks the app sent, across all dictation ids. */ + chunks(): number; + /** Resolves on the first chunk the app sends after this call. */ + nextChunk(): Promise; + /** The final transcript the next finish replies with. */ + setTranscript(text: string): void; + /** Keeps the next final reply until `releaseFinal`. */ + holdFinal(): void; + releaseFinal(): void; + /** Answers the next finish with a stream error instead of a final. */ + failNextFinish(error: string): void; +} + +/** + * Answers the app's dictation stream in place of the daemon: acknowledges starts + * and chunks, accepts finishes and replies with the scenario's transcript. Rewrites + * `server_info` to the given dictation capability and forwards everything else, so + * terminals, agents and workspaces keep using the real daemon. + * `interceptClientMessage` lets a spec answer other requests itself; return true + * when it handled one. + */ +export async function installDictationWsHarness( + page: Page, + options: { + transcript?: string; + capability?: DictationCapability; + interceptClientMessage?: (request: Record, ws: SessionSocket) => boolean; + } = {}, +): Promise { + const capability = options.capability ?? { enabled: true, reason: "" }; + let transcript = options.transcript ?? ""; + let starts = 0; + let chunkCount = 0; + let chunkWaiters: Array<() => void> = []; + let holdNextFinal = false; + let heldFinal: (() => void) | null = null; + let nextFinishError: string | null = null; + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + + ws.onMessage((message) => { + const request = sessionMessage(message); + const type = request?.type; + const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null; + + if (type === "dictation_stream_start" && dictationId) { + starts += 1; + sendSessionMessage(ws, { + type: "dictation_stream_ack", + payload: { dictationId, ackSeq: -1 }, + }); + return; + } + if (type === "dictation_stream_chunk" && dictationId) { + chunkCount += 1; + const seq = typeof request?.seq === "number" ? request.seq : 0; + sendSessionMessage(ws, { + type: "dictation_stream_ack", + payload: { dictationId, ackSeq: seq }, + }); + const waiters = chunkWaiters; + chunkWaiters = []; + for (const resolve of waiters) { + resolve(); + } + return; + } + if (type === "dictation_stream_finish" && dictationId) { + sendSessionMessage(ws, { + type: "dictation_stream_finish_accepted", + payload: { dictationId, timeoutMs: 30_000 }, + }); + if (nextFinishError !== null) { + const error = nextFinishError; + nextFinishError = null; + sendSessionMessage(ws, { + type: "dictation_stream_error", + payload: { dictationId, error, retryable: true }, + }); + return; + } + const text = transcript; + const reply = () => + sendSessionMessage(ws, { + type: "dictation_stream_final", + payload: { dictationId, text }, + }); + if (holdNextFinal) { + holdNextFinal = false; + heldFinal = reply; + } else { + reply(); + } + return; + } + if (type === "dictation_stream_cancel") { + return; + } + if (request && options.interceptClientMessage?.(request, ws)) { + return; + } + + server.send(message); + }); + + server.onMessage((message) => ws.send(rewriteDictationCapability(message, capability))); + }); + + return { + streamStarts: () => starts, + chunks: () => chunkCount, + nextChunk: () => new Promise((resolve) => chunkWaiters.push(resolve)), + setTranscript: (text) => { + transcript = text; + }, + holdFinal: () => { + holdNextFinal = true; + }, + releaseFinal: () => { + const reply = heldFinal; + heldFinal = null; + if (!reply) { + throw new Error("No final reply is being held"); + } + reply(); + }, + failNextFinish: (error) => { + nextFinishError = error; + }, + }; +} diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index 2191f0d480..033f9d40da 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -7,6 +7,7 @@ import { useTranslation } from "react-i18next"; import { VolumeMeter } from "./volume-meter"; import { FOOTER_HEIGHT } from "@/constants/layout"; import type { DictationStatus } from "@/hooks/use-dictation"; +import { formatDictationDuration } from "@/utils/format-dictation-duration"; interface DictationControlsProps { volume: number; @@ -24,12 +25,6 @@ interface DictationControlsProps { disabled?: boolean; } -function formatDuration(seconds: number): string { - const mins = Math.floor(seconds / 60); - const secs = seconds % 60; - return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; -} - export function DictationControls({ volume, duration, @@ -88,7 +83,7 @@ export function DictationControls({ - {status === "paused" ? t("message.dictation.paused") : formatDuration(duration)} + {status === "paused" ? t("message.dictation.paused") : formatDictationDuration(duration)} & { errorText?: string }) { + testID, +}: Omit & { + errorText?: string; + /** Omit to show insert as the only confirm action (terminals never send). */ + onAcceptAndSend?: () => void; + testID?: string; +}) { const { theme } = useUnistyles(); const { t } = useTranslation(); const isFailed = status === "failed"; @@ -190,7 +191,7 @@ export function DictationOverlay({ } return ( - + - {formatDuration(duration)} + {formatDictationDuration(duration)} {status === "paused" ? ( @@ -242,7 +243,7 @@ export function DictationOverlay({ ) : null} - {!actionsDisabled && !isFailed ? ( + {!actionsDisabled && !isFailed && onAcceptAndSend ? ( <> ) : null} + {!actionsDisabled && !isFailed && !onAcceptAndSend ? ( + + + + ) : null} ); diff --git a/packages/app/src/components/terminal-pane.tsx b/packages/app/src/components/terminal-pane.tsx index c87bcb4f07..b208ca58b6 100644 --- a/packages/app/src/components/terminal-pane.tsx +++ b/packages/app/src/components/terminal-pane.tsx @@ -2,7 +2,7 @@ import { TerminalFind, type TerminalPaneFindHandle } from "@/terminal/find"; import type { TerminalFindResult } from "@/terminal/runtime/terminal-emulator-runtime"; import { LoadingSpinner } from "@/components/ui/loading-spinner"; import { useRetainedPanelActive } from "@/components/retained-panel"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react"; import * as Clipboard from "expo-clipboard"; import { Pressable, @@ -63,7 +63,20 @@ import { type TerminalDropDelivery, } from "@/terminal/drop/terminal-file-drop"; import { preparePathsForTerminalInput } from "@/terminal/drop/terminal-path"; -import { useRegisterTerminalInputSink, type TerminalInputSink } from "@/terminal/input-sink"; +import { + useRegisterTerminalInputSink, + type TerminalDictationActions, + type TerminalInputSink, +} from "@/terminal/input-sink"; +import { + TerminalDictationOwner, + type TerminalDictationOwnerActions, +} from "@/terminal/dictation/terminal-dictation-owner"; +import { isTerminalKeyboardOwner } from "@/terminal/dictation/terminal-dictation-lifecycle"; +import { + TerminalKeyBarDictationButton, + TerminalKeyBarDictationGate, +} from "@/terminal/toolbar/terminal-key-bar-dictation"; import type { SelectedFile } from "@/attachments/selected-file"; import { TerminalUploadButton } from "@/terminal/toolbar/terminal-upload-button"; import { toErrorMessage } from "@/utils/error-messages"; @@ -91,7 +104,7 @@ import { type TerminalResizeRequest, } from "./terminal-resize-debouncer"; import { useIsCompactFormFactor } from "@/constants/layout"; -import { isNative } from "@/constants/platform"; +import { isNative, isWeb } from "@/constants/platform"; import { applyTerminalRendererReadyChange, resolveTerminalStreamTarget, @@ -950,13 +963,34 @@ export function TerminalPane({ ); }); + // The owner mounts only while connected and remounts per terminal; the sink keeps + // one stable forwarding object and reaches whichever owner is mounted now. + const dictationOwnerId = useId(); + const dictationActionsRef = useRef(null); + const dictationActions = useMemo( + () => ({ + ownerId: dictationOwnerId, + toggle: () => dictationActionsRef.current?.toggle(), + cancel: () => dictationActionsRef.current?.cancel(), + confirm: () => dictationActionsRef.current?.confirm(), + retry: () => dictationActionsRef.current?.retry(), + discard: () => dictationActionsRef.current?.discard(), + }), + [dictationOwnerId], + ); + const terminalSurfaceRef = useRef(null); + const isDictationKeyboardOwner = useStableEvent( + () => isWeb && isTerminalKeyboardOwner(terminalSurfaceRef.current, document.activeElement), + ); + const inputSink = useMemo( () => ({ pasteText: pasteTerminalText, deliverFiles: deliverPickedFiles, focus: requestTerminalFocus, + dictation: dictationActions, }), - [pasteTerminalText, deliverPickedFiles, requestTerminalFocus], + [pasteTerminalText, deliverPickedFiles, requestTerminalFocus, dictationActions], ); useRegisterTerminalInputSink(serverId, terminalId, inputSink); @@ -1203,6 +1237,8 @@ export function TerminalPane({ ) : null; case "upload": return ; + case "dictation": + return ; case "keyboardToggle": return ( + - + - {virtualKeyboardRows.map((row) => ( - - {row.map(renderVirtualKeyboardControl)} - - ))} + + {virtualKeyboardRows.map((row) => ( + + {row.map(renderVirtualKeyboardControl)} + + ))} + ) : null} diff --git a/packages/app/src/hooks/use-dictation-audio-source.web.ts b/packages/app/src/hooks/use-dictation-audio-source.web.ts index 6b71ca7181..cead422696 100644 --- a/packages/app/src/hooks/use-dictation-audio-source.web.ts +++ b/packages/app/src/hooks/use-dictation-audio-source.web.ts @@ -2,6 +2,7 @@ import { resampleToPcm16 } from "@/audio/pcm"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { parsePcm16Wav } from "@/utils/pcm16-wav"; import { isElectronRuntime } from "@/desktop/host"; +import { AttemptCancelledError } from "@/utils/attempt-guard"; import type { DictationAudioSource, @@ -154,6 +155,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic started: boolean; mode: "pcm" | "recorder"; recorder: RecorderRefs; + // Advanced by every stop(), so a start still waiting on getUserMedia can tell it + // was stopped and must not install its stream. + startGeneration: number; }>({ stream: null, context: null, @@ -170,6 +174,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic stoppedResolve: null, stoppedReject: null, }, + startGeneration: 0, }); const decodeAudioData = useCallback( @@ -233,7 +238,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic throw new Error("AudioContext unavailable"); } - const rawStream = await navigator.mediaDevices.getUserMedia({ + const startGeneration = refs.current.startGeneration; + const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, noiseSuppression: true, @@ -241,8 +247,11 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic autoGainControl: true, }, }); - - const stream = rawStream; + if (refs.current.startGeneration !== startGeneration) { + // Stopped while the permission prompt or device acquisition was pending. + disconnectDictationAudioGraph({ processor: null, source: null, gain: null, stream }); + throw new AttemptCancelledError("Dictation start was cancelled"); + } const context = new AudioContextCtor(); @@ -371,6 +380,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic const stop = useCallback(async () => { refs.current.started = false; + refs.current.startGeneration += 1; setVolume(0); const { processor, source, gain, context, stream, pending, mode, recorder } = refs.current; @@ -418,6 +428,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic stoppedResolve: null, stoppedReject: null, }, + startGeneration: refs.current.startGeneration, }; }, [decodeAudioData, emitPcmSegments]); @@ -436,7 +447,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic await start(); } catch (err) { const normalized = err instanceof Error ? err : new Error(String(err)); - onErrorRef.current?.(normalized); + if (!(normalized instanceof AttemptCancelledError)) { + onErrorRef.current?.(normalized); + } throw normalized; } }, diff --git a/packages/app/src/hooks/use-dictation.test.tsx b/packages/app/src/hooks/use-dictation.test.tsx new file mode 100644 index 0000000000..06c990a645 --- /dev/null +++ b/packages/app/src/hooks/use-dictation.test.tsx @@ -0,0 +1,203 @@ +/** + * @vitest-environment jsdom + */ +import { act, renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { AttemptCancelledError } from "@/utils/attempt-guard"; +import type { UseDictationOptions } from "./use-dictation.shared"; + +// jsdom has no microphone: the source's start() returns a promise the test settles, +// standing in for a permission prompt or a getUserMedia call that is still pending. +const audioSource = vi.hoisted(() => { + const state = { + settleStart: null as null | { resolve: () => void; reject: (error: unknown) => void }, + start: null as unknown as ReturnType, + stop: null as unknown as ReturnType, + onPcmSegment: null as null | ((audio: string) => void), + }; + return state; +}); + +vi.mock("@/hooks/use-dictation-audio-source", () => ({ + useDictationAudioSource: (config: { onPcmSegment: (audio: string) => void }) => { + audioSource.onPcmSegment = config.onPcmSegment; + return { + start: audioSource.start, + stop: audioSource.stop, + volume: 0, + isPaused: false, + }; + }, +})); + +const { useDictation } = await import("./use-dictation"); + +function createFakeClient() { + return { + isConnected: true, + startDictationStream: vi.fn(async () => {}), + sendDictationStreamChunk: vi.fn(), + finishDictationStream: vi.fn(async (dictationId: string) => ({ dictationId, text: "" })), + cancelDictationStream: vi.fn(), + subscribeRawMessages: vi.fn(() => () => {}), + subscribeConnectionStatus: vi.fn(() => () => {}), + on: vi.fn(() => () => {}), + }; +} + +function renderDictation(client: ReturnType) { + const onError = vi.fn(); + const onTranscript = vi.fn(); + const rendered = renderHook(() => + useDictation({ + client: client as unknown as UseDictationOptions["client"], + onTranscript, + onError, + }), + ); + return { ...rendered, onError, onTranscript }; +} + +async function flush(): Promise { + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); +} + +beforeEach(() => { + audioSource.settleStart = null; + audioSource.start = vi.fn( + () => + new Promise((resolve, reject) => { + audioSource.settleStart = { resolve, reject }; + }), + ); + audioSource.stop = vi.fn(async () => {}); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("useDictation pending startup", () => { + it("cancels a start that is still acquiring the microphone", async () => { + const client = createFakeClient(); + const { result, onError } = renderDictation(client); + + act(() => { + void result.current.startDictation(); + }); + expect(result.current.status).toBe("recording"); + expect(audioSource.settleStart).not.toBeNull(); + + await act(async () => { + await result.current.cancelDictation(); + }); + expect(result.current.status).toBe("idle"); + // The source is stopped while acquisition is pending, so it can release a late stream itself. + expect(audioSource.stop).toHaveBeenCalledTimes(1); + + await act(async () => { + audioSource.settleStart?.resolve(); + }); + await flush(); + + expect(audioSource.stop).toHaveBeenCalledTimes(2); + expect(result.current.isRecording).toBe(false); + expect(result.current.isRecordingActive()).toBe(false); + expect(result.current.status).toBe("idle"); + expect(client.startDictationStream).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + expect(result.current.error).toBeNull(); + }); + + it("treats the source's cancelled acquisition as a silent cancel", async () => { + const client = createFakeClient(); + const { result, onError } = renderDictation(client); + + act(() => { + void result.current.startDictation(); + }); + await act(async () => { + await result.current.cancelDictation(); + }); + await act(async () => { + audioSource.settleStart?.reject(new AttemptCancelledError()); + }); + await flush(); + + expect(result.current.status).toBe("idle"); + expect(result.current.isRecording).toBe(false); + expect(client.startDictationStream).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + expect(result.current.error).toBeNull(); + }); + + it("stops the late-acquired microphone after an unmount during acquisition", async () => { + const client = createFakeClient(); + const { result, unmount, onError } = renderDictation(client); + + act(() => { + void result.current.startDictation(); + }); + unmount(); + const stopsAtUnmount = audioSource.stop.mock.calls.length; + expect(stopsAtUnmount).toBeGreaterThanOrEqual(1); + + await act(async () => { + audioSource.settleStart?.resolve(); + }); + await flush(); + + expect(audioSource.stop.mock.calls.length).toBe(stopsAtUnmount + 1); + expect(client.startDictationStream).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + }); + + it("records and opens the stream when the start is not cancelled", async () => { + const client = createFakeClient(); + const { result, onError } = renderDictation(client); + + act(() => { + void result.current.startDictation(); + }); + await act(async () => { + audioSource.settleStart?.resolve(); + }); + await flush(); + + expect(result.current.isRecording).toBe(true); + expect(result.current.status).toBe("recording"); + expect(client.startDictationStream).toHaveBeenCalledTimes(1); + expect(audioSource.stop).not.toHaveBeenCalled(); + expect(onError).not.toHaveBeenCalled(); + }); + + it("drops the partial segment the source flushes while a recording is cancelled", async () => { + const client = createFakeClient(); + // Stopping a real source flushes the audio it still holds. + audioSource.stop = vi.fn(async () => { + audioSource.onPcmSegment?.("AAAA"); + }); + const { result } = renderDictation(client); + + act(() => { + void result.current.startDictation(); + }); + await act(async () => { + audioSource.settleStart?.resolve(); + }); + await flush(); + expect(client.startDictationStream).toHaveBeenCalledTimes(1); + + await act(async () => { + await result.current.cancelDictation(); + }); + await flush(); + + expect(result.current.status).toBe("idle"); + expect(client.startDictationStream).toHaveBeenCalledTimes(1); + expect(client.sendDictationStreamChunk).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index a2a3f46bcc..44dfc6d6b0 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -70,12 +70,19 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const durationIntervalRef = useRef | null>(null); const attemptGuardRef = useRef(new AttemptGuard()); + // Advanced only by cancel and unmount, never by confirm: a confirm while the stream + // is still starting must not make the start tear itself down. + const startGuardRef = useRef(new AttemptGuard()); const actionGateRef = useRef<{ starting: boolean; confirming: boolean; cancelling: boolean }>({ starting: false, confirming: false, cancelling: false, }); + // Audio is accepted from start until a cancel: stopping the source flushes its last + // partial segment, which after a cancel would open a new stream for audio nobody wants. + const acceptAudioRef = useRef(false); + const senderRef = useRef(null); if (!senderRef.current) { senderRef.current = new DictationStreamSender({ @@ -228,6 +235,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const audio = useDictationAudioSource({ onPcmSegment: (audioData) => { + if (!acceptAudioRef.current) { + return; + } senderRef.current?.enqueueSegment(audioData); }, onError: (err) => { @@ -264,15 +274,36 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { } actionGateRef.current.starting = true; + // A cancel or unmount while the microphone is still being acquired advances the + // start guard; the checks below then release whatever the start acquired late. + const startId = startGuardRef.current.next(); + const abandonStaleStart = async () => { + acceptAudioRef.current = false; + try { + senderRef.current?.cancel(); + } catch { + // no-op + } + await audio.stop().catch(() => undefined); + stopDurationTracking(); + isRecordingRef.current = false; + setIsRecording(false); + setStatus("idle"); + }; setError(null); setPartialTranscript(""); setDuration(0); setIsProcessing(false); setStatus("recording"); clearStreamingState(); + acceptAudioRef.current = true; try { await audio.start(); + if (!startGuardRef.current.isCurrent(startId)) { + await abandonStaleStart(); + return; + } isRecordingRef.current = true; setIsRecording(true); if (enableDuration) { @@ -280,8 +311,16 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { } if (client?.isConnected) { await startNewStream("start"); + if (!startGuardRef.current.isCurrent(startId)) { + await abandonStaleStart(); + return; + } } } catch (err) { + if (!startGuardRef.current.isCurrent(startId)) { + await abandonStaleStart(); + return; + } await audio.stop().catch(() => undefined); stopDurationTracking(); isRecordingRef.current = false; @@ -303,15 +342,35 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { stopDurationTracking, ]); + // The start is still waiting on the microphone. Show idle at once and stop the + // source now, so a stream acquired after this point is released by the source + // itself; the start continuation then sees the stale attempt and tears down the rest. + const cancelPendingStart = useCallback(async () => { + acceptAudioRef.current = false; + stopDurationTracking(); + setDuration(0); + setError(null); + setStatus("idle"); + clearStreamingState(); + await audio.stop().catch((err: unknown) => { + reportError(err, "Failed to stop pending dictation audio"); + }); + }, [audio, clearStreamingState, reportError, stopDurationTracking]); + const cancelDictation = useCallback(async () => { attemptGuardRef.current.cancel(); + startGuardRef.current.cancel(); if (actionGateRef.current.cancelling) { return; } if (!isRecordingRef.current && !isProcessingRef.current) { + if (actionGateRef.current.starting) { + await cancelPendingStart(); + } return; } actionGateRef.current.cancelling = true; + acceptAudioRef.current = false; stopDurationTracking(); setDuration(0); setError(null); @@ -334,7 +393,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { clearStreamingState(); actionGateRef.current.cancelling = false; } - }, [audio, clearStreamingState, reportError, stopDurationTracking]); + }, [audio, cancelPendingStart, clearStreamingState, reportError, stopDurationTracking]); const confirmDictation = useCallback(async () => { if (actionGateRef.current.confirming) { @@ -444,9 +503,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { useEffect(() => { const attemptGuard = attemptGuardRef.current; + const startGuard = startGuardRef.current; const audioStop = audioStopRef; return () => { attemptGuard.cancel(); + startGuard.cancel(); stopDurationTracking(); void audioStop.current().catch(() => undefined); senderRef.current?.dispose(); diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index e26ef022a0..b375df7044 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -622,6 +622,13 @@ export const ar: TranslationResources = { uploadingFile: "جارٍ رفع الملف…", actions: { uploadFile: "رفع ملف", + startDictation: "بدء الإملاء", + stopDictation: "إيقاف ولصق النص المفرغ", + }, + dictation: { + updateHost: "حدّث المضيف لاستخدام الإملاء في الطرفيات.", + unavailable: "الإملاء غير متاح على هذا المضيف.", + discard: "تجاهل التسجيل", }, }, sheet: { @@ -2607,6 +2614,7 @@ export const ar: TranslationResources = { cycleAgentMode: "تبديل وضع الوكيل", toggleVoiceMode: "تبديل الوضع الصوتي", startStopDictation: "بدء إملاء /stop", + startStopTerminalDictation: "بدء/إيقاف الإملاء في الطرفية", interruptAgent: "عامل المقاطعة", sendMessage: "أرسل رسالة", queueMessage: "رسالة قائمة الانتظار", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index eca04f7720..eb86899e63 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -619,6 +619,13 @@ export const en = { uploadingFile: "Uploading file…", actions: { uploadFile: "Upload file", + startDictation: "Start dictation", + stopDictation: "Stop and paste transcript", + }, + dictation: { + updateHost: "Update the host to use dictation in terminals.", + unavailable: "Dictation is not available on this host.", + discard: "Discard recording", }, }, sheet: { @@ -2694,6 +2701,7 @@ export const en = { cycleAgentMode: "Cycle agent mode", toggleVoiceMode: "Toggle voice mode", startStopDictation: "Start/stop dictation", + startStopTerminalDictation: "Start/stop dictation in terminal", interruptAgent: "Interrupt agent", sendMessage: "Send message", queueMessage: "Queue message", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 3d13c60e31..364ce81463 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -628,6 +628,13 @@ export const es: TranslationResources = { uploadingFile: "Subiendo archivo…", actions: { uploadFile: "Subir archivo", + startDictation: "Iniciar dictado", + stopDictation: "Detener y pegar la transcripción", + }, + dictation: { + updateHost: "Actualiza el host para usar el dictado en terminales.", + unavailable: "El dictado no está disponible en este host.", + discard: "Descartar grabación", }, }, sheet: { @@ -2625,6 +2632,7 @@ export const es: TranslationResources = { cycleAgentMode: "Alternar modo del agente", toggleVoiceMode: "Alternar modo de voz", startStopDictation: "Iniciar dictado/stop", + startStopTerminalDictation: "Iniciar/detener dictado en la terminal", interruptAgent: "agente de interrupción", sendMessage: "enviar mensaje", queueMessage: "mensaje de cola", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 0cd7d0163e..c11e1edac6 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -628,6 +628,13 @@ export const fr: TranslationResources = { uploadingFile: "Envoi du fichier…", actions: { uploadFile: "Envoyer un fichier", + startDictation: "Démarrer la dictée", + stopDictation: "Arrêter et coller la transcription", + }, + dictation: { + updateHost: "Mettez à jour l'hôte pour utiliser la dictée dans les terminaux.", + unavailable: "La dictée n'est pas disponible sur cet hôte.", + discard: "Supprimer l'enregistrement", }, }, sheet: { @@ -2626,6 +2633,7 @@ export const fr: TranslationResources = { cycleAgentMode: "Parcourir les modes de l’agent", toggleVoiceMode: "Activer/désactiver le mode vocal", startStopDictation: "Démarrer/arrêter la dictée", + startStopTerminalDictation: "Démarrer/arrêter la dictée dans le terminal", interruptAgent: "Interrompre l’agent", sendMessage: "Envoyer le message", queueMessage: "Mettre le message en file d’attente", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index 761f1428ea..4d7e992334 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -628,6 +628,13 @@ export const ja: TranslationResources = { uploadingFile: "ファイルをアップロード中…", actions: { uploadFile: "ファイルをアップロード", + startDictation: "音声入力を開始", + stopDictation: "停止して文字起こしを貼り付け", + }, + dictation: { + updateHost: "ターミナルで音声入力を使うにはホストを更新してください。", + unavailable: "このホストでは音声入力を利用できません。", + discard: "録音を破棄", }, }, sheet: { @@ -2592,6 +2599,7 @@ export const ja: TranslationResources = { cycleAgentMode: "エージェントモードを順に切り替え", toggleVoiceMode: "音声モードを切り替え", startStopDictation: "音声入力を開始/停止", + startStopTerminalDictation: "ターミナルで音声入力を開始/停止", interruptAgent: "エージェントを中断", sendMessage: "メッセージを送信", queueMessage: "メッセージをキューに追加", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index 5a01781a73..ae75977259 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -623,6 +623,13 @@ export const ko: TranslationResources = { uploadingFile: "파일 업로드 중…", actions: { uploadFile: "파일 업로드", + startDictation: "받아쓰기 시작", + stopDictation: "중지하고 받아쓴 내용 붙여넣기", + }, + dictation: { + updateHost: "터미널에서 받아쓰기를 사용하려면 호스트를 업데이트하세요.", + unavailable: "이 호스트에서는 받아쓰기를 사용할 수 없습니다.", + discard: "녹음 삭제", }, }, sheet: { @@ -2581,6 +2588,7 @@ export const ko: TranslationResources = { cycleAgentMode: "에이전트 모드 전환", toggleVoiceMode: "음성 모드 토글", startStopDictation: "받아쓰기 시작/중지", + startStopTerminalDictation: "터미널에서 받아쓰기 시작/중지", interruptAgent: "에이전트 중단", sendMessage: "메시지 보내기", queueMessage: "메시지 대기열에 추가", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index a97fb721f4..003ece7202 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -627,6 +627,13 @@ export const ptBR: TranslationResources = { uploadingFile: "Enviando arquivo…", actions: { uploadFile: "Enviar arquivo", + startDictation: "Iniciar ditado", + stopDictation: "Parar e colar a transcrição", + }, + dictation: { + updateHost: "Atualize o host para usar o ditado nos terminais.", + unavailable: "O ditado não está disponível neste host.", + discard: "Descartar gravação", }, }, sheet: { @@ -2606,6 +2613,7 @@ export const ptBR: TranslationResources = { cycleAgentMode: "Alternar modo do agente", toggleVoiceMode: "Alternar modo de voz", startStopDictation: "Iniciar/parar ditado", + startStopTerminalDictation: "Iniciar/parar ditado no terminal", interruptAgent: "Interromper agente", sendMessage: "Enviar mensagem", queueMessage: "Enfileirar mensagem", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index 7d8328ab17..de5837bb0d 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -629,6 +629,13 @@ export const ru: TranslationResources = { uploadingFile: "Загрузка файла…", actions: { uploadFile: "Загрузить файл", + startDictation: "Начать диктовку", + stopDictation: "Остановить и вставить расшифровку", + }, + dictation: { + updateHost: "Обновите хост, чтобы использовать диктовку в терминалах.", + unavailable: "Диктовка недоступна на этом хосте.", + discard: "Удалить запись", }, }, sheet: { @@ -2613,6 +2620,7 @@ export const ru: TranslationResources = { cycleAgentMode: "Переключить режим агента", toggleVoiceMode: "Переключить голосовой режим", startStopDictation: "Начать/остановить диктовку", + startStopTerminalDictation: "Начать/остановить диктовку в терминале", interruptAgent: "Прервать агента", sendMessage: "Отправить сообщение", queueMessage: "Поставить сообщение в очередь", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index cc2e885edb..4b310cc215 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -622,6 +622,13 @@ export const zhCN: TranslationResources = { uploadingFile: "正在上传文件…", actions: { uploadFile: "上传文件", + startDictation: "开始听写", + stopDictation: "停止并粘贴转写内容", + }, + dictation: { + updateHost: "请更新主机以在终端中使用听写。", + unavailable: "此主机不支持听写。", + discard: "丢弃录音", }, }, sheet: { @@ -2543,6 +2550,7 @@ export const zhCN: TranslationResources = { cycleAgentMode: "循环切换代理模式", toggleVoiceMode: "切换语音模式", startStopDictation: "开始/停止听写", + startStopTerminalDictation: "在终端中开始/停止听写", interruptAgent: "中断 Agent", sendMessage: "发送消息", queueMessage: "消息排队", diff --git a/packages/app/src/keyboard/actions.ts b/packages/app/src/keyboard/actions.ts index 1bf78d923f..0299850b0a 100644 --- a/packages/app/src/keyboard/actions.ts +++ b/packages/app/src/keyboard/actions.ts @@ -48,6 +48,7 @@ export type KeyboardActionId = | "command-center.files" | "shortcuts.dialog.toggle" | "workspace.terminal.new" + | "terminal.dictation-toggle" | "workspace.new" | "workspace.project.pick" | "worktree.new" diff --git a/packages/app/src/keyboard/keyboard-action-dispatcher.ts b/packages/app/src/keyboard/keyboard-action-dispatcher.ts index a3f2b4c515..f6234a1d09 100644 --- a/packages/app/src/keyboard/keyboard-action-dispatcher.ts +++ b/packages/app/src/keyboard/keyboard-action-dispatcher.ts @@ -44,6 +44,7 @@ export type KeyboardActionId = | "workspace.pane.close" | "workspace.focus.toggle" | "workspace.terminal.new" + | "terminal.dictation-toggle" | "workspace.browser.new" | "sidebar.toggle.right" | "sidebar.toggle.both" @@ -102,6 +103,7 @@ export type KeyboardActionDefinition = | { id: "workspace.pane.close"; scope: KeyboardActionScope } | { id: "workspace.focus.toggle"; scope: KeyboardActionScope } | { id: "workspace.terminal.new"; scope: KeyboardActionScope } + | { id: "terminal.dictation-toggle"; scope: KeyboardActionScope } | { id: "workspace.browser.new"; scope: KeyboardActionScope } | { id: "sidebar.toggle.right"; scope: KeyboardActionScope } | { id: "sidebar.toggle.both"; scope: KeyboardActionScope } diff --git a/packages/app/src/keyboard/keyboard-shortcuts.test.ts b/packages/app/src/keyboard/keyboard-shortcuts.test.ts index 770d7e710c..c7239ff626 100644 --- a/packages/app/src/keyboard/keyboard-shortcuts.test.ts +++ b/packages/app/src/keyboard/keyboard-shortcuts.test.ts @@ -323,6 +323,12 @@ describe("keyboard-shortcuts", () => { action: "message-input.action", payload: { kind: "dictation-toggle" }, }, + { + name: "routes Cmd+D to terminal dictation, not the message input, when terminal is focused on macOS", + event: { key: "d", code: "KeyD", metaKey: true }, + context: { isMac: true, focusScope: "terminal" }, + action: "terminal.dictation-toggle", + }, { name: "routes Shift+Tab to cycle agent mode from the message input", event: { key: "Tab", code: "Tab", shiftKey: true }, @@ -481,9 +487,14 @@ describe("keyboard-shortcuts", () => { context: { isMac: false, focusScope: "terminal" }, }, { - name: "does not route message-input actions when terminal is focused", + name: "leaves Ctrl+D to the PTY as EOF when terminal is focused on non-mac", + event: { key: "d", code: "KeyD", ctrlKey: true }, + context: { isMac: false, focusScope: "terminal" }, + }, + { + name: "does not bind Cmd+D to terminal dictation on non-mac", event: { key: "d", code: "KeyD", metaKey: true }, - context: { isMac: true, focusScope: "terminal" }, + context: { isMac: false, focusScope: "terminal" }, }, { name: "does not cycle agent mode outside the message input", diff --git a/packages/app/src/keyboard/keyboard-shortcuts.ts b/packages/app/src/keyboard/keyboard-shortcuts.ts index 9f6329c013..90b376c1e4 100644 --- a/packages/app/src/keyboard/keyboard-shortcuts.ts +++ b/packages/app/src/keyboard/keyboard-shortcuts.ts @@ -199,6 +199,7 @@ export const SHORTCUT_HELP_ROW_ORDER: Record = { "cycle-agent-mode": "settings.shortcuts.help.cycleAgentMode", "voice-toggle": "settings.shortcuts.help.toggleVoiceMode", "dictation-toggle": "settings.shortcuts.help.startStopDictation", + "terminal-dictation-toggle": "settings.shortcuts.help.startStopTerminalDictation", "agent-interrupt": "settings.shortcuts.help.interruptAgent", "voice-mute-toggle": "settings.shortcuts.help.muteUnmuteVoiceMode", }; @@ -1136,6 +1138,18 @@ const SHORTCUT_BINDINGS: readonly ShortcutBinding[] = [ label: "Start/stop dictation", }, }, + // Terminals take Cmd+D on macOS only: elsewhere Ctrl+D is the PTY's EOF and passes through. + { + id: "terminal-dictation-toggle-cmd-d-mac", + action: "terminal.dictation-toggle", + combo: "Cmd+D", + when: { mac: true, commandCenter: false, focusScope: "terminal" }, + help: { + id: "terminal-dictation-toggle", + section: "agent-input", + label: "Start/stop dictation in terminal", + }, + }, { id: "agent-interrupt", action: "agent.interrupt", diff --git a/packages/app/src/keyboard/route-shortcut.test.ts b/packages/app/src/keyboard/route-shortcut.test.ts index 5eddf3eeff..0a3d49eb32 100644 --- a/packages/app/src/keyboard/route-shortcut.test.ts +++ b/packages/app/src/keyboard/route-shortcut.test.ts @@ -39,6 +39,7 @@ describe("routeKeyboardShortcut — dispatch passthroughs", () => { ["workspace.pin", { id: "workspace.pin", scope: "sidebar" }], ["worktree.new", { id: "worktree.new", scope: "sidebar" }], ["workspace.terminal.new", { id: "workspace.terminal.new", scope: "workspace" }], + ["terminal.dictation-toggle", { id: "terminal.dictation-toggle", scope: "workspace" }], ["workspace.tab.close.current", { id: "workspace.tab.close-current", scope: "workspace" }], ["sidebar.toggle.right", { id: "sidebar.toggle.right", scope: "sidebar" }], ["workspace.pane.split.right", { id: "workspace.pane.split.right", scope: "workspace" }], diff --git a/packages/app/src/keyboard/route-shortcut.ts b/packages/app/src/keyboard/route-shortcut.ts index aa7679a7bf..a1fa5b41ba 100644 --- a/packages/app/src/keyboard/route-shortcut.ts +++ b/packages/app/src/keyboard/route-shortcut.ts @@ -51,6 +51,7 @@ const PASSTHROUGH_DISPATCH: Record = { "workspace.pin": { id: "workspace.pin", scope: "sidebar" }, "worktree.new": { id: "worktree.new", scope: "sidebar" }, "workspace.terminal.new": { id: "workspace.terminal.new", scope: "workspace" }, + "terminal.dictation-toggle": { id: "terminal.dictation-toggle", scope: "workspace" }, "workspace.tab.close.current": { id: "workspace.tab.close-current", scope: "workspace" }, "sidebar.toggle.right": { id: "sidebar.toggle.right", scope: "sidebar" }, "workspace.pane.split.right": { id: "workspace.pane.split.right", scope: "workspace" }, diff --git a/packages/app/src/screens/workspace/workspace-desktop-tabs-row.tsx b/packages/app/src/screens/workspace/workspace-desktop-tabs-row.tsx index 7fec8bdc2d..c5389fae16 100644 --- a/packages/app/src/screens/workspace/workspace-desktop-tabs-row.tsx +++ b/packages/app/src/screens/workspace/workspace-desktop-tabs-row.tsx @@ -105,8 +105,9 @@ const PANE_SPLIT_ACTIONS_RESERVED_WIDTH = PANE_SPLIT_ACTIONS_HORIZONTAL_PADDING * 2 + PANE_SPLIT_ACTIONS_OUTER_MARGIN; const PANE_MAXIMIZE_ACTION_RESERVED_WIDTH = smallIconButtonChromeFrameSize(false) + 1; -// The terminal slot holds one icon button (upload). -const TERMINAL_SLOT_RESERVED_WIDTH = smallIconButtonChromeFrameSize(false) + 1; +// The terminal slot holds two icon buttons (upload, mic). While the terminal dictates the +// mic grows into a pill; the row re-measures then instead of reserving that width always. +const TERMINAL_SLOT_RESERVED_WIDTH = 2 * (smallIconButtonChromeFrameSize(false) + 1); // Chip geometry. `layoutMetrics` measures tabs from these same numbers, so a chip that changes // shape without changing them mis-measures and drops the row into the overflow-scroll fallback at // the wrong width. Keep them together. diff --git a/packages/app/src/terminal/dictation/format-terminal-transcript.test.ts b/packages/app/src/terminal/dictation/format-terminal-transcript.test.ts new file mode 100644 index 0000000000..8a50c20a4c --- /dev/null +++ b/packages/app/src/terminal/dictation/format-terminal-transcript.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { formatTerminalTranscript } from "./format-terminal-transcript"; + +describe("formatTerminalTranscript", () => { + it.each([ + ["first\nsecond", "first second "], + ["first\r\nsecond", "first second "], + ["first\rsecond", "first second "], + ["\n hello \r\n", "hello "], + ["héllo 日本語", "héllo 日本語 "], + ])("formats %j as %j", (input, expected) => { + expect(formatTerminalTranscript(input)).toBe(expected); + }); + + it.each(["", " ", "\r\n"])("returns null for %j", (input) => { + expect(formatTerminalTranscript(input)).toBeNull(); + }); +}); diff --git a/packages/app/src/terminal/dictation/format-terminal-transcript.ts b/packages/app/src/terminal/dictation/format-terminal-transcript.ts new file mode 100644 index 0000000000..53147f6036 --- /dev/null +++ b/packages/app/src/terminal/dictation/format-terminal-transcript.ts @@ -0,0 +1,10 @@ +/** + * Turns a dictation transcript into terminal input: embedded line breaks become + * spaces (a newline would submit the line in most TUIs), the ends are trimmed, and + * one trailing space separates it from whatever is typed next. Never adds Enter. + * Returns null when nothing is left to paste. + */ +export function formatTerminalTranscript(text: string): string | null { + const singleLine = text.replace(/\r\n|\r|\n/g, " ").trim(); + return singleLine.length > 0 ? `${singleLine} ` : null; +} diff --git a/packages/app/src/terminal/dictation/terminal-dictation-availability.test.ts b/packages/app/src/terminal/dictation/terminal-dictation-availability.test.ts new file mode 100644 index 0000000000..9e44d4652d --- /dev/null +++ b/packages/app/src/terminal/dictation/terminal-dictation-availability.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import type { DaemonServerInfo } from "@/stores/session-store"; +import { resolveTerminalDictationAvailability } from "./terminal-dictation-availability"; + +function serverInfo(capabilities?: DaemonServerInfo["capabilities"]): DaemonServerInfo { + return { + serverId: "server-a", + hostname: null, + version: "0.13.2", + ...(capabilities ? { capabilities } : {}), + }; +} + +function voice(dictation: { enabled: boolean; reason: string }): DaemonServerInfo["capabilities"] { + return { voice: { dictation, voice: { enabled: false, reason: "" } } }; +} + +describe("resolveTerminalDictationAvailability", () => { + it("is disconnected while the host is not connected, whatever it reports", () => { + expect( + resolveTerminalDictationAvailability({ + serverInfo: serverInfo(voice({ enabled: true, reason: "" })), + isConnected: false, + }), + ).toEqual({ kind: "disconnected" }); + }); + + it("asks to update a host that reports no capabilities", () => { + expect( + resolveTerminalDictationAvailability({ serverInfo: serverInfo(), isConnected: true }), + ).toEqual({ kind: "unavailable", reason: "update-host" }); + expect(resolveTerminalDictationAvailability({ serverInfo: null, isConnected: true })).toEqual({ + kind: "unavailable", + reason: "update-host", + }); + }); + + it("asks to update a host whose capabilities have no voice field", () => { + expect( + resolveTerminalDictationAvailability({ serverInfo: serverInfo({}), isConnected: true }), + ).toEqual({ kind: "unavailable", reason: "update-host" }); + }); + + it("carries the host's trimmed reason when dictation is disabled", () => { + expect( + resolveTerminalDictationAvailability({ + serverInfo: serverInfo(voice({ enabled: false, reason: " No speech model installed " })), + isConnected: true, + }), + ).toEqual({ kind: "unavailable", reason: "disabled", message: "No speech model installed" }); + }); + + it.each(["", " "])("leaves the message empty for a disabled reason of %j", (reason) => { + expect( + resolveTerminalDictationAvailability({ + serverInfo: serverInfo(voice({ enabled: false, reason })), + isConnected: true, + }), + ).toEqual({ kind: "unavailable", reason: "disabled", message: null }); + }); + + it("is ready when the host enables dictation", () => { + expect( + resolveTerminalDictationAvailability({ + serverInfo: serverInfo(voice({ enabled: true, reason: "" })), + isConnected: true, + }), + ).toEqual({ kind: "ready" }); + }); +}); diff --git a/packages/app/src/terminal/dictation/terminal-dictation-availability.ts b/packages/app/src/terminal/dictation/terminal-dictation-availability.ts new file mode 100644 index 0000000000..647d8e90c4 --- /dev/null +++ b/packages/app/src/terminal/dictation/terminal-dictation-availability.ts @@ -0,0 +1,35 @@ +import type { DaemonServerInfo } from "@/stores/session-store"; +import { getVoiceReadinessState } from "@/utils/server-info-capabilities"; + +export type TerminalDictationAvailability = + | { kind: "ready" } + | { kind: "disconnected" } + | { kind: "unavailable"; reason: "update-host" } + | { kind: "unavailable"; reason: "disabled"; message: string | null }; + +/** + * Whether a terminal can dictate on this host. Gates on the host's explicit + * dictation capability only: a host too old to report it is told to update, so a + * terminal never falls back to the composer's agent-directory heuristic. + */ +export function resolveTerminalDictationAvailability(input: { + serverInfo: DaemonServerInfo | null | undefined; + isConnected: boolean; +}): TerminalDictationAvailability { + if (!input.isConnected) { + return { kind: "disconnected" }; + } + const dictation = getVoiceReadinessState({ serverInfo: input.serverInfo, mode: "dictation" }); + if (!dictation) { + return { kind: "unavailable", reason: "update-host" }; + } + if (!dictation.enabled) { + const message = dictation.reason.trim(); + return { + kind: "unavailable", + reason: "disabled", + message: message.length > 0 ? message : null, + }; + } + return { kind: "ready" }; +} diff --git a/packages/app/src/terminal/dictation/terminal-dictation-lifecycle.test.ts b/packages/app/src/terminal/dictation/terminal-dictation-lifecycle.test.ts new file mode 100644 index 0000000000..aedbb4adf4 --- /dev/null +++ b/packages/app/src/terminal/dictation/terminal-dictation-lifecycle.test.ts @@ -0,0 +1,89 @@ +/** + * @vitest-environment jsdom + */ +import { describe, expect, it } from "vitest"; +import type { DictationStatus } from "@/hooks/use-dictation"; +import { + isTerminalKeyboardOwner, + resolveTerminalDictationLifecycle, +} from "./terminal-dictation-lifecycle"; + +describe("resolveTerminalDictationLifecycle", () => { + const active: DictationStatus[] = ["recording", "paused", "uploading"]; + + it("keeps an idle terminal whatever happens to it", () => { + expect( + resolveTerminalDictationLifecycle({ + isPresented: false, + isRecordingOwner: false, + status: "idle", + }), + ).toBe("keep"); + }); + + it.each(active)("keeps a presented recording owner while %s", (status) => { + expect( + resolveTerminalDictationLifecycle({ isPresented: true, isRecordingOwner: true, status }), + ).toBe("keep"); + }); + + it.each(active)("cancels when the terminal stops being presented while %s", (status) => { + expect( + resolveTerminalDictationLifecycle({ isPresented: false, isRecordingOwner: true, status }), + ).toBe("cancel"); + }); + + it.each(active)("cancels when another terminal takes the recording while %s", (status) => { + expect( + resolveTerminalDictationLifecycle({ isPresented: true, isRecordingOwner: false, status }), + ).toBe("cancel"); + }); + + it("discards a failed attempt once hidden or superseded", () => { + expect( + resolveTerminalDictationLifecycle({ + isPresented: false, + isRecordingOwner: true, + status: "failed", + }), + ).toBe("discard"); + expect( + resolveTerminalDictationLifecycle({ + isPresented: true, + isRecordingOwner: false, + status: "failed", + }), + ).toBe("discard"); + }); +}); + +describe("isTerminalKeyboardOwner", () => { + function surfaceWithInput() { + const surface = document.createElement("div"); + const input = document.createElement("textarea"); + surface.append(input); + document.body.append(surface); + return { surface, input }; + } + + it("is true when focus is inside the surface", () => { + const { surface, input } = surfaceWithInput(); + expect(isTerminalKeyboardOwner(surface, input)).toBe(true); + surface.remove(); + }); + + it("is false when focus is in another surface", () => { + const first = surfaceWithInput(); + const second = surfaceWithInput(); + expect(isTerminalKeyboardOwner(first.surface, second.input)).toBe(false); + first.surface.remove(); + second.surface.remove(); + }); + + it("is false without a surface or a focused element", () => { + const { surface, input } = surfaceWithInput(); + expect(isTerminalKeyboardOwner(surface, null)).toBe(false); + expect(isTerminalKeyboardOwner(null, input)).toBe(false); + surface.remove(); + }); +}); diff --git a/packages/app/src/terminal/dictation/terminal-dictation-lifecycle.ts b/packages/app/src/terminal/dictation/terminal-dictation-lifecycle.ts new file mode 100644 index 0000000000..ec2a9b55ac --- /dev/null +++ b/packages/app/src/terminal/dictation/terminal-dictation-lifecycle.ts @@ -0,0 +1,39 @@ +import type { DictationStatus } from "@/hooks/use-dictation"; + +export type TerminalDictationLifecycleDecision = "keep" | "cancel" | "discard"; + +/** + * What a terminal's dictation does when its pane stops being presented (another + * tab or terminal selected, workspace left, explorer closed) or another terminal + * takes the recording. Recording, paused and processing are cancelled, a failed + * attempt is discarded. Status `recording` also covers a start still acquiring + * the microphone, because the hook sets it before awaiting the source. + */ +export function resolveTerminalDictationLifecycle(input: { + isPresented: boolean; + isRecordingOwner: boolean; + status: DictationStatus; +}): TerminalDictationLifecycleDecision { + if (input.status === "idle") { + return "keep"; + } + if (input.isPresented && input.isRecordingOwner) { + return "keep"; + } + return input.status === "failed" ? "discard" : "cancel"; +} + +/** + * True when DOM keyboard focus is inside this pane's terminal surface: the same + * focus that gives a shortcut the `terminal` scope, so exactly one pane owns it. + */ +export function isTerminalKeyboardOwner(surface: unknown, activeElement: unknown): boolean { + if (typeof Element === "undefined") { + return false; + } + return ( + surface instanceof Element && + activeElement instanceof Element && + surface.contains(activeElement) + ); +} diff --git a/packages/app/src/terminal/dictation/terminal-dictation-owner.tsx b/packages/app/src/terminal/dictation/terminal-dictation-owner.tsx new file mode 100644 index 0000000000..b21a38be27 --- /dev/null +++ b/packages/app/src/terminal/dictation/terminal-dictation-owner.tsx @@ -0,0 +1,198 @@ +import { useEffect, useRef, type MutableRefObject } from "react"; +import { useTranslation } from "react-i18next"; +import { useToast } from "@/contexts/toast-context"; +import { useDictation } from "@/hooks/use-dictation"; +import { useKeyboardActionHandler } from "@/hooks/use-keyboard-action-handler"; +import { useStableEvent } from "@/hooks/use-stable-event"; +import { useHostRuntimeClient, useHostRuntimeIsConnected } from "@/runtime/host-runtime"; +import { useSessionStore } from "@/stores/session-store"; +import { + claimTerminalRecording, + clearTerminalDictationState, + getTerminalRecordingOwnerId, + publishTerminalDictationState, + releaseTerminalRecording, + useTerminalRecordingOwnerId, + type TerminalDictationActions, +} from "@/terminal/input-sink"; +import { formatTerminalTranscript } from "./format-terminal-transcript"; +import { + resolveTerminalDictationAvailability, + type TerminalDictationAvailability, +} from "./terminal-dictation-availability"; +import { resolveTerminalDictationLifecycle } from "./terminal-dictation-lifecycle"; + +export type TerminalDictationOwnerActions = Omit; + +interface TerminalDictationOwnerProps { + ownerId: string; + serverId: string; + terminalId: string; + /** The pane's terminal is the one on screen: its tab is selected and its workspace focused. */ + isPresented: boolean; + pasteText: (text: string) => void; + focus: () => void; + /** Evaluated when the shortcut fires: true while DOM focus is inside this pane's terminal. */ + isKeyboardOwner: () => boolean; + /** Where the owner hands its actions to the pane's input sink. */ + actionsRef: MutableRefObject; +} + +/** + * Owns one terminal pane's dictation. Mounted inside the pane and keyed by its + * terminal, so a transcript can only reach the pane that recorded it; the tab-bar + * control and the key bar are views of the state it publishes. Renders nothing, + * and lives apart from the pane so volume ticks re-render only this component. + * + * The recording ends, microphone and pending result included, when the pane + * unmounts or changes terminal (remount through the key), when the terminal stops + * being presented, and when another terminal starts recording. + */ +export function TerminalDictationOwner({ + ownerId, + serverId, + terminalId, + isPresented, + pasteText, + focus, + isKeyboardOwner, + actionsRef, +}: TerminalDictationOwnerProps) { + const { t } = useTranslation(); + const toast = useToast(); + const client = useHostRuntimeClient(serverId); + const isConnected = useHostRuntimeIsConnected(serverId); + const serverInfo = useSessionStore((state) => state.sessions[serverId]?.serverInfo ?? null); + const recordingOwnerId = useTerminalRecordingOwnerId(); + const isPresentedRef = useRef(isPresented); + isPresentedRef.current = isPresented; + + const handleTranscript = useStableEvent((text: string) => { + // A result resolving in the same tick as a cancel: drop it unless this pane is + // still on screen and still the recording terminal. + if (!isPresentedRef.current || getTerminalRecordingOwnerId() !== ownerId) { + return; + } + const formatted = formatTerminalTranscript(text); + if (formatted !== null) { + pasteText(formatted); + } + focus(); + }); + const handleError = useStableEvent((error: Error) => { + toast.error(error.message); + }); + + const dictation = useDictation({ + client, + onTranscript: handleTranscript, + onError: handleError, + enableDuration: true, + }); + const statusRef = useRef(dictation.status); + statusRef.current = dictation.status; + + const unavailableMessage = ( + availability: Exclude, + ) => { + if (availability.kind === "disconnected") { + return t("workspace.terminal.hostDisconnected"); + } + if (availability.reason === "update-host") { + return t("workspace.terminal.dictation.updateHost"); + } + return availability.message ?? t("workspace.terminal.dictation.unavailable"); + }; + + const toggle = useStableEvent(() => { + if (!isPresentedRef.current) { + return; + } + const status = statusRef.current; + if (status === "recording" || status === "paused") { + void dictation.confirmDictation(); + return; + } + if (status !== "idle") { + return; + } + const availability = resolveTerminalDictationAvailability({ serverInfo, isConnected }); + if (availability.kind !== "ready") { + toast.error(unavailableMessage(availability)); + return; + } + claimTerminalRecording(ownerId); + void dictation.startDictation(); + }); + const cancel = useStableEvent(() => { + void dictation.cancelDictation(); + focus(); + }); + const confirm = useStableEvent(() => { + void dictation.confirmDictation(); + }); + const retry = useStableEvent(() => { + void dictation.retryFailedDictation(); + }); + const discard = useStableEvent(() => { + dictation.discardFailedDictation(); + focus(); + }); + + useEffect(() => { + const actions: TerminalDictationOwnerActions = { toggle, cancel, confirm, retry, discard }; + actionsRef.current = actions; + return () => { + if (actionsRef.current === actions) { + actionsRef.current = null; + } + }; + }, [actionsRef, cancel, confirm, discard, retry, toggle]); + + // Only a change of presentation or recording owner ends the recording, so the + // status is read at that moment instead of re-running on every status change. + const applyLifecycle = useStableEvent( + (input: { isPresented: boolean; isRecordingOwner: boolean }) => { + const decision = resolveTerminalDictationLifecycle({ ...input, status: statusRef.current }); + if (decision === "cancel") { + void dictation.cancelDictation(); + } else if (decision === "discard") { + dictation.discardFailedDictation(); + } + }, + ); + useEffect(() => { + applyLifecycle({ isPresented, isRecordingOwner: recordingOwnerId === ownerId }); + }, [applyLifecycle, isPresented, ownerId, recordingOwnerId]); + + useEffect(() => { + return () => releaseTerminalRecording(ownerId); + }, [ownerId]); + + const { status, isRecording, isProcessing, error, duration, volume } = dictation; + useEffect(() => { + publishTerminalDictationState(ownerId, { + status, + isRecording, + isProcessing, + error, + duration, + volume, + }); + }, [duration, error, isProcessing, isRecording, ownerId, status, volume]); + useEffect(() => () => clearTerminalDictationState(ownerId), [ownerId]); + + useKeyboardActionHandler({ + handlerId: `terminal-dictation:${serverId}:${terminalId}:${ownerId}`, + actions: ["terminal.dictation-toggle"], + enabled: isPresented, + priority: 0, + isActive: isKeyboardOwner, + handle: () => { + toggle(); + return true; + }, + }); + + return null; +} diff --git a/packages/app/src/terminal/input-sink/index.test.tsx b/packages/app/src/terminal/input-sink/index.test.tsx index d2a7bbcffd..22b3caca09 100644 --- a/packages/app/src/terminal/input-sink/index.test.tsx +++ b/packages/app/src/terminal/input-sink/index.test.tsx @@ -4,15 +4,45 @@ import { act, renderHook } from "@testing-library/react"; import { afterEach, describe, expect, it } from "vitest"; import { + claimTerminalRecording, + clearTerminalDictationState, getTerminalInputSink, + getTerminalRecordingOwnerId, + publishTerminalDictationState, registerTerminalInputSink, + releaseTerminalRecording, useRegisterTerminalInputSink, + useTerminalDictationState, useTerminalInputSink, + type TerminalDictationState, type TerminalInputSink, } from "./index"; function fakeSink(): TerminalInputSink { - return { pasteText: () => {}, deliverFiles: () => {}, focus: () => {} }; + return { + pasteText: () => {}, + deliverFiles: () => {}, + focus: () => {}, + dictation: { + ownerId: "owner", + toggle: () => {}, + cancel: () => {}, + confirm: () => {}, + retry: () => {}, + discard: () => {}, + }, + }; +} + +function dictationState(volume: number): TerminalDictationState { + return { + status: "recording", + isRecording: true, + isProcessing: false, + error: null, + duration: 0, + volume, + }; } const cleanups: Array<() => void> = []; @@ -95,3 +125,47 @@ describe("terminal input sinks", () => { expect(getTerminalInputSink("server-a", "term-1")).toBeNull(); }); }); + +describe("terminal dictation state", () => { + afterEach(() => { + act(() => { + clearTerminalDictationState("owner-a"); + clearTerminalDictationState("owner-b"); + releaseTerminalRecording("owner-a"); + releaseTerminalRecording("owner-b"); + }); + }); + + it("re-renders an owner's controls only for that owner's state", () => { + let renders = 0; + const { result } = renderHook(() => { + renders += 1; + return useTerminalDictationState("owner-a"); + }); + expect(result.current).toBeNull(); + const rendersBefore = renders; + + act(() => publishTerminalDictationState("owner-b", dictationState(0.5))); + expect(renders).toBe(rendersBefore); + + const published = dictationState(0.2); + act(() => publishTerminalDictationState("owner-a", published)); + expect(result.current).toBe(published); + + act(() => clearTerminalDictationState("owner-a")); + expect(result.current).toBeNull(); + }); + + it("keeps one recording owner and releases it only for that owner", () => { + act(() => claimTerminalRecording("owner-a")); + expect(getTerminalRecordingOwnerId()).toBe("owner-a"); + + act(() => claimTerminalRecording("owner-b")); + expect(getTerminalRecordingOwnerId()).toBe("owner-b"); + + act(() => releaseTerminalRecording("owner-a")); + expect(getTerminalRecordingOwnerId()).toBe("owner-b"); + act(() => releaseTerminalRecording("owner-b")); + expect(getTerminalRecordingOwnerId()).toBeNull(); + }); +}); diff --git a/packages/app/src/terminal/input-sink/index.ts b/packages/app/src/terminal/input-sink/index.ts index cc7a804304..750a857857 100644 --- a/packages/app/src/terminal/input-sink/index.ts +++ b/packages/app/src/terminal/input-sink/index.ts @@ -1,12 +1,15 @@ import { useEffect } from "react"; import { create } from "zustand"; import type { SelectedFile } from "@/attachments/selected-file"; +import type { DictationStatus } from "@/hooks/use-dictation"; /** - * What a mounted terminal pane accepts from controls rendered outside it: the - * pane's tab-bar slot and its key bar. Those controls are siblings of the pane - * under three different hosts (split pane, fallback row, explorer sidebar), so - * they find the pane here by terminal identity instead of through props. + * What a mounted terminal pane accepts from its controls. The tab-bar slot is a + * sibling of the pane under three different hosts (split pane, fallback row, + * explorer sidebar), so it finds the pane here by terminal identity instead of + * through props. The key bar renders inside the pane and receives the pane's own + * sink as a prop: when two panes show one terminal the registry holds only the + * newer one, and each key bar must still act on its own pane. */ export interface TerminalInputSink { /** Pastes text the way the terminal's TUI expects it (bracketed when enabled). */ @@ -15,13 +18,44 @@ export interface TerminalInputSink { deliverFiles(files: SelectedFile[]): void; /** Returns keyboard focus to the terminal. */ focus(): void; + /** Forwards to the pane's dictation owner; does nothing while no owner is mounted. */ + dictation: TerminalDictationActions; +} + +export interface TerminalDictationActions { + /** Identifies the pane's owner; its published state is under this id. */ + ownerId: string; + /** Starts dictation, or confirms it while recording. */ + toggle(): void; + cancel(): void; + confirm(): void; + retry(): void; + discard(): void; +} + +/** What a dictation owner publishes for its controls. */ +export interface TerminalDictationState { + status: DictationStatus; + isRecording: boolean; + isProcessing: boolean; + error: string | null; + duration: number; + volume: number; } interface TerminalInputSinkState { sinks: Record; + /** Keyed by owner id, apart from `sinks` so volume ticks never change a sink. */ + dictationStates: Record; + /** The owner whose terminal is recording; one terminal records at a time. */ + recordingOwnerId: string | null; } -const useTerminalInputSinkStore = create(() => ({ sinks: {} })); +const useTerminalInputSinkStore = create(() => ({ + sinks: {}, + dictationStates: {}, + recordingOwnerId: null, +})); export function terminalInputSinkKey(serverId: string, terminalId: string): string { return `${serverId}:${terminalId}`; @@ -79,3 +113,47 @@ export function useTerminalInputSink( const key = terminalInputSinkKey(serverId, terminalId); return useTerminalInputSinkStore((state) => state.sinks[key] ?? null); } + +export function publishTerminalDictationState( + ownerId: string, + dictationState: TerminalDictationState, +): void { + useTerminalInputSinkStore.setState((state) => ({ + dictationStates: { ...state.dictationStates, [ownerId]: dictationState }, + })); +} + +export function clearTerminalDictationState(ownerId: string): void { + useTerminalInputSinkStore.setState((state) => { + if (!(ownerId in state.dictationStates)) { + return state; + } + const { [ownerId]: _removed, ...dictationStates } = state.dictationStates; + return { dictationStates }; + }); +} + +/** The owner's published state, or null while no owner with this id is mounted. */ +export function useTerminalDictationState(ownerId: string): TerminalDictationState | null { + return useTerminalInputSinkStore((state) => state.dictationStates[ownerId] ?? null); +} + +/** Makes `ownerId` the recording terminal; the previous owner cancels itself. */ +export function claimTerminalRecording(ownerId: string): void { + useTerminalInputSinkStore.setState({ recordingOwnerId: ownerId }); +} + +/** Clears the recording owner only while it is still `ownerId`. */ +export function releaseTerminalRecording(ownerId: string): void { + useTerminalInputSinkStore.setState((state) => + state.recordingOwnerId === ownerId ? { recordingOwnerId: null } : state, + ); +} + +export function getTerminalRecordingOwnerId(): string | null { + return useTerminalInputSinkStore.getState().recordingOwnerId; +} + +export function useTerminalRecordingOwnerId(): string | null { + return useTerminalInputSinkStore((state) => state.recordingOwnerId); +} diff --git a/packages/app/src/terminal/runtime/terminal-virtual-keyboard.test.ts b/packages/app/src/terminal/runtime/terminal-virtual-keyboard.test.ts index 5702f6c815..fc06a35c01 100644 --- a/packages/app/src/terminal/runtime/terminal-virtual-keyboard.test.ts +++ b/packages/app/src/terminal/runtime/terminal-virtual-keyboard.test.ts @@ -82,12 +82,13 @@ describe("terminal virtual keyboard policy", () => { "terminal-key-esc", "terminal-key-tab", "terminal-key-ctrl", - "terminal-key-up", "terminal-key-shift", + "terminal-key-up", "terminal-key-alt", "terminal-keyboard-toggle", "terminal-paste", "terminal-upload", + "terminal-dictation", "terminal-key-left", "terminal-key-down", "terminal-key-right", @@ -104,14 +105,14 @@ describe("terminal virtual keyboard policy", () => { down: positions.get("terminal-key-down"), right: positions.get("terminal-key-right"), }).toEqual({ - up: { row: 0, col: 3 }, - left: { row: 1, col: 2 }, - down: { row: 1, col: 3 }, - right: { row: 1, col: 4 }, + up: { row: 0, col: 4 }, + left: { row: 1, col: 3 }, + down: { row: 1, col: 4 }, + right: { row: 1, col: 5 }, }); }); - it("keeps Paste and Upload next to each other in every arrangement", () => { + it("keeps Paste, Upload and Dictation next to each other in every arrangement", () => { for (const rows of [ TERMINAL_VIRTUAL_KEYBOARD_ROWS, TERMINAL_VIRTUAL_KEYBOARD_WIDE_ROWS, @@ -120,6 +121,7 @@ describe("terminal virtual keyboard policy", () => { const rowIds = rows.map((row) => row.map(getTerminalVirtualKeyboardControlId)); const ids = rowIds.find((row) => row.includes("terminal-paste")) ?? []; expect(ids.indexOf("terminal-upload")).toBe(ids.indexOf("terminal-paste") + 1); + expect(ids.indexOf("terminal-dictation")).toBe(ids.indexOf("terminal-paste") + 2); } }); diff --git a/packages/app/src/terminal/runtime/terminal-virtual-keyboard.ts b/packages/app/src/terminal/runtime/terminal-virtual-keyboard.ts index c7c673372a..3d5023089f 100644 --- a/packages/app/src/terminal/runtime/terminal-virtual-keyboard.ts +++ b/packages/app/src/terminal/runtime/terminal-virtual-keyboard.ts @@ -21,24 +21,28 @@ export type TerminalVirtualKeyboardControl = | { type: "upload"; } + | { + type: "dictation"; + } | { type: "keyboardToggle"; }; -// Paste and upload stay next to each other in every arrangement. +// Paste, upload and dictation stay next to each other in every arrangement. export const TERMINAL_VIRTUAL_KEYBOARD_ROWS = [ [ { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.esc }, { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.tab }, { type: "modifier", modifier: "ctrl" }, - { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up }, { type: "modifier", modifier: "shift" }, + { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up }, { type: "modifier", modifier: "alt" }, { type: "keyboardToggle" }, ], [ { type: "paste" }, { type: "upload" }, + { type: "dictation" }, { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.left }, { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.down }, { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.right }, @@ -59,6 +63,7 @@ export const TERMINAL_VIRTUAL_KEYBOARD_WIDE_ROWS = [ { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.right }, { type: "paste" }, { type: "upload" }, + { type: "dictation" }, { type: "keyboardToggle" }, { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.enter }, ], @@ -75,7 +80,7 @@ export const TERMINAL_VIRTUAL_KEYBOARD_THREE_COLUMN_ROWS = [ { type: "modifier", modifier: "shift" }, { type: "keyboardToggle" }, ], - [{ type: "paste" }, { type: "upload" }], + [{ type: "paste" }, { type: "upload" }, { type: "dictation" }], [ { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up }, { type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.enter }, @@ -121,6 +126,8 @@ export function getTerminalVirtualKeyboardControlId( return "terminal-paste"; case "upload": return "terminal-upload"; + case "dictation": + return "terminal-dictation"; case "keyboardToggle": return "terminal-keyboard-toggle"; } diff --git a/packages/app/src/terminal/toolbar/terminal-dictation-control.tsx b/packages/app/src/terminal/toolbar/terminal-dictation-control.tsx new file mode 100644 index 0000000000..41153163b9 --- /dev/null +++ b/packages/app/src/terminal/toolbar/terminal-dictation-control.tsx @@ -0,0 +1,160 @@ +import { Check, Mic, RefreshCcw, X } from "lucide-react-native"; +import { Text, View } from "react-native"; +import { useTranslation } from "react-i18next"; +import { StyleSheet, withUnistyles } from "react-native-unistyles"; +import { LoadingSpinner } from "@/components/ui/loading-spinner"; +import { ToolbarButton } from "@/components/ui/pane-content-toolbar"; +import { VolumeMeter } from "@/components/volume-meter"; +import { useShortcutKeys } from "@/hooks/use-shortcut-keys"; +import type { Theme } from "@/styles/theme"; +import { formatDictationDuration } from "@/utils/format-dictation-duration"; +import type { TerminalDictationState, TerminalInputSink } from "@/terminal/input-sink"; + +const ThemedMic = withUnistyles(Mic); +const ThemedX = withUnistyles(X); +const ThemedCheck = withUnistyles(Check); +const ThemedRefresh = withUnistyles(RefreshCcw); +const ThemedVolumeMeter = withUnistyles(VolumeMeter); +const ThemedSpinner = withUnistyles(LoadingSpinner); +const iconColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundExtraMuted }); +const activeIconColorMapping = (theme: Theme) => ({ color: theme.colors.foreground }); +const accentIconColorMapping = (theme: Theme) => ({ color: theme.colors.accent }); + +interface TerminalDictationControlProps { + sink: TerminalInputSink; + /** Null while the pane has no dictation owner mounted (host disconnected). */ + state: TerminalDictationState | null; +} + +/** + * The terminal's mic in its pane's tab bar. Idle it is one button; while the + * terminal records, processes or has failed, the same slot becomes a pill with + * cancel, meter and duration, and confirm (or retry and discard after a failure). + * A view of the pane's dictation owner: every press goes through the sink. + */ +export function TerminalDictationControl({ sink, state }: TerminalDictationControlProps) { + const { t } = useTranslation(); + const shortcut = useShortcutKeys("terminal-dictation-toggle"); + const status = state?.status ?? "idle"; + + if (!state || status === "idle") { + return ( + + + + ); + } + + const isFailed = status === "failed"; + const isProcessing = state.isProcessing; + + return ( + + {isFailed ? ( + + + + ) : ( + + + + )} + {isFailed ? ( + + {state.error + ? t("message.dictation.failed", { error: state.error }) + : t("message.dictation.failedRetry")} + + ) : ( + + + + {status === "paused" + ? t("message.dictation.paused") + : formatDictationDuration(state.duration)} + + + )} + {isProcessing ? ( + + + + ) : null} + {!isProcessing && isFailed ? ( + + + + ) : null} + {!isProcessing && !isFailed ? ( + + + + ) : null} + + ); +} + +const styles = StyleSheet.create((theme) => ({ + pill: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[1], + paddingHorizontal: theme.spacing[1], + borderRadius: theme.borderRadius.full, + borderWidth: 1, + borderColor: theme.colors.accent, + backgroundColor: theme.colors.surface1, + }, + meter: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[1], + }, + duration: { + color: theme.colors.foreground, + fontSize: theme.fontSize.sm, + fontVariant: ["tabular-nums"], + }, + errorText: { + maxWidth: 160, + color: theme.colors.destructive, + fontSize: theme.fontSize.sm, + }, + spinner: { + width: 20, + alignItems: "center", + justifyContent: "center", + }, +})); diff --git a/packages/app/src/terminal/toolbar/terminal-key-bar-dictation.tsx b/packages/app/src/terminal/toolbar/terminal-key-bar-dictation.tsx new file mode 100644 index 0000000000..50d08dad5b --- /dev/null +++ b/packages/app/src/terminal/toolbar/terminal-key-bar-dictation.tsx @@ -0,0 +1,66 @@ +import { useMemo, type ReactNode } from "react"; +import { Mic } from "lucide-react-native"; +import { useTranslation } from "react-i18next"; +import { withUnistyles } from "react-native-unistyles"; +import { DictationOverlay } from "@/components/dictation-controls"; +import { TerminalKeyBarIconAction } from "@/components/terminal-copy-paste-actions"; +import type { Theme } from "@/styles/theme"; +import { useTerminalDictationState, type TerminalInputSink } from "@/terminal/input-sink"; + +const ThemedMic = withUnistyles(Mic); +const keyBarIconColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundMuted }); + +interface TerminalKeyBarDictationProps { + /** The pane's own sink: a key bar acts on its pane, never on a newer pane of the same terminal. */ + sink: TerminalInputSink; +} + +/** The key-bar mic. Disabled while the pane has no dictation owner (host disconnected). */ +export function TerminalKeyBarDictationButton({ sink }: TerminalKeyBarDictationProps) { + const { t } = useTranslation(); + const state = useTerminalDictationState(sink.dictation.ownerId); + const icon = useMemo(() => , []); + return ( + + ); +} + +/** + * Shows the key bar's rows while the pane's terminal is not dictating, and the + * dictation overlay in their place through recording, paused, processing and + * failed. The rows come back once the attempt is delivered, cancelled or discarded. + * Lives apart from the pane so volume ticks re-render only this component. + */ +export function TerminalKeyBarDictationGate({ + sink, + children, +}: TerminalKeyBarDictationProps & { children: ReactNode }) { + const state = useTerminalDictationState(sink.dictation.ownerId); + if (!state || state.status === "idle") { + return children; + } + // Also recording while the microphone is still being acquired, so cancel is reachable then. + const isRecording = + state.isRecording || state.status === "recording" || state.status === "paused"; + return ( + + ); +} diff --git a/packages/app/src/terminal/toolbar/terminal-toolbar-slot.tsx b/packages/app/src/terminal/toolbar/terminal-toolbar-slot.tsx index 3ed169802c..356e917df8 100644 --- a/packages/app/src/terminal/toolbar/terminal-toolbar-slot.tsx +++ b/packages/app/src/terminal/toolbar/terminal-toolbar-slot.tsx @@ -1,4 +1,5 @@ -import { useTerminalInputSink } from "@/terminal/input-sink"; +import { useTerminalDictationState, useTerminalInputSink } from "@/terminal/input-sink"; +import { TerminalDictationControl } from "./terminal-dictation-control"; import { TerminalUploadButton } from "./terminal-upload-button"; interface TerminalToolbarSlotProps { @@ -8,13 +9,28 @@ interface TerminalToolbarSlotProps { /** * The terminal controls in a pane's tab bar, shown while the pane's active tab is - * a terminal. A view over the mounted pane's input sink: it owns no terminal - * state and renders nothing until the pane has registered. + * a terminal: upload, then the mic. A view over the mounted pane's input sink and + * its dictation state: it owns no terminal state and renders nothing until the + * pane has registered. */ export function TerminalToolbarSlot({ serverId, terminalId }: TerminalToolbarSlotProps) { const sink = useTerminalInputSink(serverId, terminalId); if (!sink) { return null; } - return ; + return ; +} + +function TerminalToolbarSlotControls({ + sink, +}: { + sink: NonNullable>; +}) { + const state = useTerminalDictationState(sink.dictation.ownerId); + return ( + <> + + + + ); } diff --git a/packages/app/src/utils/format-dictation-duration.ts b/packages/app/src/utils/format-dictation-duration.ts new file mode 100644 index 0000000000..4a0d20f2f2 --- /dev/null +++ b/packages/app/src/utils/format-dictation-duration.ts @@ -0,0 +1,6 @@ +/** Formats a dictation's elapsed seconds as `mm:ss`. */ +export function formatDictationDuration(seconds: number): string { + const mins = Math.floor(seconds / 60); + const secs = seconds % 60; + return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`; +}