M2: terminal dictation: mic in tab-bar slot and key bar, recording pill and overlay, per-terminal recording lifetime incl. pending mic acquisition, Cmd+D on macOS by keyboard focus, i18n

This commit is contained in:
Sebastian Arena
2026-10-26 21:59:03 -04:00
parent 1eea1381eb
commit 113dd83d95
40 changed files with 2264 additions and 174 deletions
@@ -1,6 +1,10 @@
import { expect, test, type Page } from "../support/fixtures";
import { gotoAppShell } from "../support/helpers/app";
import { daemonWsRoutePattern } from "../support/helpers/daemon-port";
import {
installDictationWsHarness,
installSyntheticMicrophone,
sendSessionMessage,
} from "../support/helpers/dictation-ws";
import {
openNewWorkspaceComposer,
selectWorkspaceIsolation,
@@ -17,141 +21,39 @@ import {
} from "../support/helpers/composer";
import { waitForSidebarHydration } from "../support/helpers/workspace-ui";
type WebSocketMessage = string | Buffer;
const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace";
const CREATE_FAILURE = "Synthetic workspace creation failure";
function parseEnvelope(message: WebSocketMessage): {
type?: unknown;
message?: Record<string, unknown>;
} | null {
const raw = typeof message === "string" ? message : message.toString("utf8");
try {
return JSON.parse(raw) as { type?: unknown; message?: Record<string, unknown> };
} catch {
return null;
}
}
function sessionMessage(message: WebSocketMessage): Record<string, unknown> | null {
const envelope = parseEnvelope(message);
return envelope?.type === "session" && envelope.message ? envelope.message : null;
}
function sendSessionMessage(
ws: { send(message: string): void },
message: Record<string, unknown>,
): void {
ws.send(JSON.stringify({ type: "session", message }));
}
function enableDictationCapability(message: WebSocketMessage): WebSocketMessage {
const envelope = parseEnvelope(message);
const payload = envelope?.message?.payload;
if (
envelope?.message?.type !== "status" ||
!payload ||
typeof payload !== "object" ||
(payload as { status?: unknown }).status !== "server_info"
) {
return message;
}
(payload as Record<string, unknown>).capabilities = {
voice: {
dictation: { enabled: true, reason: "" },
voice: { enabled: false, reason: "Realtime voice is disabled in this test." },
},
};
return JSON.stringify(envelope);
}
async function installSyntheticMicrophone(page: Page): Promise<void> {
await page.addInitScript(() => {
const mediaDevices = navigator.mediaDevices;
Object.defineProperty(mediaDevices, "getUserMedia", {
configurable: true,
value: async () => {
const context = new AudioContext();
const oscillator = context.createOscillator();
const destination = context.createMediaStreamDestination();
oscillator.connect(destination);
oscillator.start();
return destination.stream;
},
});
});
}
async function installDictationFailureHarness(page: Page) {
let resolveAudioChunk!: () => void;
let resolveCreateRequest!: () => void;
const audioChunk = new Promise<void>((resolve) => {
resolveAudioChunk = resolve;
});
const createRequest = new Promise<void>((resolve) => {
resolveCreateRequest = resolve;
});
let failCreate: (() => void) | null = null;
await page.routeWebSocket(daemonWsRoutePattern(), (ws) => {
const server = ws.connectToServer();
ws.onMessage((message) => {
const request = sessionMessage(message);
const type = request?.type;
const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null;
if (type === "dictation_stream_start" && dictationId) {
sendSessionMessage(ws, {
type: "dictation_stream_ack",
payload: { dictationId, ackSeq: -1 },
});
return;
const dictation = await installDictationWsHarness(page, {
transcript: TRANSCRIPT,
interceptClientMessage: (request, ws) => {
if (request.type !== "workspace.create.request" || typeof request.requestId !== "string") {
return false;
}
if (type === "dictation_stream_chunk" && dictationId) {
const seq = typeof request?.seq === "number" ? request.seq : 0;
const requestId = request.requestId;
failCreate = () => {
sendSessionMessage(ws, {
type: "dictation_stream_ack",
payload: { dictationId, ackSeq: seq },
type: "workspace.create.response",
payload: {
requestId,
workspace: null,
setupTerminalId: null,
error: CREATE_FAILURE,
},
});
resolveAudioChunk();
return;
}
if (type === "dictation_stream_finish" && dictationId) {
sendSessionMessage(ws, {
type: "dictation_stream_finish_accepted",
payload: { dictationId, timeoutMs: 5_000 },
});
sendSessionMessage(ws, {
type: "dictation_stream_final",
payload: { dictationId, text: TRANSCRIPT },
});
return;
}
if (request && type === "workspace.create.request" && typeof request.requestId === "string") {
const requestId = request.requestId;
failCreate = () => {
sendSessionMessage(ws, {
type: "workspace.create.response",
payload: {
requestId,
workspace: null,
setupTerminalId: null,
error: CREATE_FAILURE,
},
});
};
resolveCreateRequest();
return;
}
server.send(message);
});
server.onMessage((message) => ws.send(enableDictationCapability(message)));
};
resolveCreateRequest();
return true;
},
});
const audioChunk = dictation.nextChunk();
return {
waitForAudio: () => audioChunk,
@@ -0,0 +1,598 @@
import { writeFile } from "node:fs/promises";
import path from "node:path";
import type { Locator, Page } from "@playwright/test";
import { expect, test } from "../support/fixtures";
import { runWorkspaceActionFromCommandCenter } from "../support/helpers/command-center-workspace-actions";
import {
holdMicrophone,
installDictationWsHarness,
installSyntheticMicrophone,
microphoneStats,
releaseMicrophone,
type DictationCapability,
type DictationWsHarness,
} from "../support/helpers/dictation-ws";
import { TerminalE2EHarness, type TerminalInstance } from "../support/helpers/terminal-dsl";
import { ensureExplorerSidebar } from "../support/helpers/workspace-tabs";
const MAC_USER_AGENT =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
const LINUX_USER_AGENT =
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
// Payloads are echoed as JSON with whitespace escaped, because the PTY capture
// below drops whitespace when it joins wrapped lines.
const ENCODE = String.raw`
const encode = (text) =>
JSON.stringify(text).replace(
/\s/g,
(space) => "\\u" + space.charCodeAt(0).toString(16).padStart(4, "0"),
);
`;
// A TUI with bracketed paste on: echoes each paste body as INPUT:<json>.
const BRACKETED_RECEIVER_SCRIPT = String.raw`
${ENCODE}
let buffer = "";
process.stdin.setRawMode(true);
process.stdin.resume();
process.stdout.write("\x1b[?2004hDICTATION_RECEIVER_READY\r\n");
process.stdin.on("data", (chunk) => {
buffer += chunk.toString();
for (;;) {
const paste = /\x1b\[200~([^]*?)\x1b\[201~/.exec(buffer);
if (!paste) break;
process.stdout.write("INPUT:" + encode(paste[1]) + "\r\n");
buffer = buffer.slice(0, paste.index) + buffer.slice(paste.index + paste[0].length);
}
});
`;
// A program without bracketed paste: echoes every stdin byte, in raw mode so
// Ctrl+D arrives as \x04, as RAW_INPUT:<json> after a quiet period.
const RAW_RECEIVER_SCRIPT = String.raw`
${ENCODE}
let pending = "";
let timer = null;
process.stdin.setRawMode(true);
process.stdin.resume();
process.stdout.write("RAW_RECEIVER_READY\r\n");
process.stdin.on("data", (chunk) => {
pending += chunk.toString();
clearTimeout(timer);
timer = setTimeout(() => {
process.stdout.write("RAW_INPUT:" + encode(pending) + "\r\n");
pending = "";
}, 200);
});
`;
type ReceiverKind = "bracketed" | "raw";
function visibleTestId(scope: Page | Locator, testId: string): Locator {
return scope.getByTestId(testId).filter({ visible: true }).first();
}
// Click the surface, not its helper textarea: the textarea sits at the left edge,
// under a split or Explorer resize handle.
function terminalSurface(scope: Page | Locator): Locator {
return scope.getByTestId("terminal-surface").filter({ visible: true });
}
function terminalInput(scope: Page | Locator): Locator {
return scope
.getByTestId("terminal-surface")
.filter({ visible: true })
.locator(".xterm-helper-textarea");
}
function splitChild(page: Page, terminal: TerminalInstance): Locator {
return page
.getByTestId("split-group-child")
.filter({ has: page.getByTestId(`workspace-tab-terminal_${terminal.id}`) });
}
function explorerSidebar(page: Page): Locator {
return page.getByTestId("workspace-explorer-sidebar").filter({ visible: true }).first();
}
async function dragTabIntoExplorer(page: Page, terminal: TerminalInstance): Promise<void> {
const source = page
.getByTestId("workspace-pane-main")
.getByTestId(`workspace-tab-terminal_${terminal.id}`);
const destination = explorerSidebar(page).getByTestId("workspace-tab-files");
const sourceBox = await source.boundingBox();
const destinationBox = await destination.boundingBox();
if (!sourceBox || !destinationBox) {
throw new Error("Tab drag requires visible source and destination");
}
const start = { x: sourceBox.x + 12, y: sourceBox.y + sourceBox.height / 2 };
await page.mouse.move(start.x, start.y);
await page.mouse.down();
await page.mouse.move(start.x + 12, start.y + 4);
await page.mouse.move(destinationBox.x + 12, destinationBox.y + destinationBox.height / 2, {
steps: 20,
});
await page.mouse.up();
}
async function expectMicrophoneStat(
page: Page,
stat: "trackStops" | "getUserMediaCalls",
count: number,
): Promise<void> {
await expect.poll(async () => (await microphoneStats(page))[stat]).toBe(count);
}
test.describe("Terminal dictation", () => {
test.describe.configure({ timeout: 180_000 });
let harness: TerminalE2EHarness;
test.beforeAll(async () => {
harness = await TerminalE2EHarness.create({ tempPrefix: "terminal-dictation-" });
await writeFile(
path.join(harness.tempRepo.path, "bracketed-receiver.cjs"),
BRACKETED_RECEIVER_SCRIPT,
);
await writeFile(path.join(harness.tempRepo.path, "raw-receiver.cjs"), RAW_RECEIVER_SCRIPT);
});
test.afterAll(async () => {
await harness?.cleanup();
});
const terminals: TerminalInstance[] = [];
test.afterEach(async () => {
for (const terminal of terminals.splice(0)) {
await harness.killTerminal(terminal.id);
}
});
async function createReceiver(kind: ReceiverKind, name: string): Promise<TerminalInstance> {
const terminal = await harness.createTerminal({
name,
command: process.execPath,
args: [kind === "bracketed" ? "bracketed-receiver.cjs" : "raw-receiver.cjs"],
});
terminals.push(terminal);
await expectTerminalOutput(
terminal.id,
kind === "bracketed" ? "DICTATION_RECEIVER_READY" : "RAW_RECEIVER_READY",
);
return terminal;
}
async function setup(
page: Page,
options: { transcript?: string; capability?: DictationCapability } = {},
): Promise<DictationWsHarness> {
await installSyntheticMicrophone(page);
return await installDictationWsHarness(page, {
transcript: options.transcript ?? "first\nsecond",
capability: options.capability,
});
}
// Reads the PTY itself, so output from one pane cannot satisfy another's check.
async function terminalOutput(terminalId: string): Promise<string> {
const capture = await harness.client.captureTerminal(terminalId);
return capture.lines.join("").replace(/\s/g, "");
}
async function expectTerminalOutput(terminalId: string, text: string): Promise<void> {
await expect.poll(() => terminalOutput(terminalId), { timeout: 10_000 }).toContain(text);
}
/** Everything the receiver got, in order and joined. */
async function received(terminalId: string, kind: ReceiverKind): Promise<string> {
const label = kind === "bracketed" ? "INPUT" : "RAW_INPUT";
const output = await terminalOutput(terminalId);
return [
...output.matchAll(
new RegExp(
`${kind === "bracketed" ? "(?<!RAW_)" : ""}${label}:("(?:[^"\\\\]|\\\\.)*")`,
"g",
),
),
]
.map(([, json]) => JSON.parse(json) as string)
.join("");
}
async function expectReceived(
terminalId: string,
kind: ReceiverKind,
text: string,
): Promise<void> {
await expect.poll(() => received(terminalId, kind), { timeout: 15_000 }).toBe(text);
}
/** Gives a late transcript time to land, then checks that none did. */
async function expectNothingReceived(
page: Page,
...receivers: Array<[TerminalInstance, ReceiverKind]>
): Promise<void> {
await page.waitForTimeout(1_500);
for (const [terminal, kind] of receivers) {
expect(await received(terminal.id, kind)).toBe("");
}
}
async function startFromToolbar(
page: Page,
ws: DictationWsHarness,
scope: Page | Locator = page,
) {
const chunk = ws.nextChunk();
await visibleTestId(scope, "terminal-toolbar-dictation").click();
await chunk;
}
// The route helper needs the desktop tab row, so open there, then go compact.
async function openCompact(page: Page, terminal: TerminalInstance) {
await harness.openTerminal(page, { terminalId: terminal.id });
await page.setViewportSize({ width: 390, height: 844 });
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
}
// The compact switcher labels terminals by their command, so pick by position.
async function selectCompactTab(page: Page, index: number) {
await page
.getByRole("button", { name: /Switch tabs/ })
.filter({ visible: true })
.click();
await expect(page.getByText("Workspace tabs", { exact: true })).toBeVisible();
const options = page
.getByText("bracketed-receiver.cjs", { exact: true })
.filter({ visible: true });
await expect(options).toHaveCount(2);
await options.nth(index).click();
await expect(page.getByRole("button", { name: "Bottom sheet backdrop" })).toHaveCount(0);
}
async function selectTerminalTab(scope: Page | Locator, terminal: TerminalInstance) {
await visibleTestId(scope, `workspace-tab-terminal_${terminal.id}`).click();
}
test("confirm pastes the transcript on one line with a trailing space and no Enter", async ({
page,
}) => {
const ws = await setup(page);
const bracketed = await createReceiver("bracketed", "dictation-bracketed");
const raw = await createReceiver("raw", "dictation-raw");
await harness.openTerminal(page, { terminalId: bracketed.id });
await startFromToolbar(page, ws);
await visibleTestId(page, "terminal-dictation-confirm").click();
await expectReceived(bracketed.id, "bracketed", "first second ");
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
await selectTerminalTab(page, raw);
await startFromToolbar(page, ws);
await visibleTestId(page, "terminal-dictation-confirm").click();
await expectReceived(raw.id, "raw", "first second ");
expect(await received(bracketed.id, "bracketed")).toBe("first second ");
});
test.describe("on macOS", () => {
test.use({ userAgent: MAC_USER_AGENT });
test("Cmd+D starts and then confirms dictation in the focused terminal", async ({ page }) => {
const ws = await setup(page, { transcript: "shortcut words" });
const terminal = await createReceiver("bracketed", "dictation-shortcut");
await harness.openTerminal(page, { terminalId: terminal.id });
const chunk = ws.nextChunk();
await terminalInput(page).press("Meta+d");
await expect(visibleTestId(page, "terminal-dictation-confirm")).toBeVisible();
await chunk;
await terminalInput(page).press("Meta+d");
await expectReceived(terminal.id, "bracketed", "shortcut words ");
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
});
test("Cmd+D goes to the terminal with keyboard focus, main pane or Explorer", async ({
page,
}) => {
const ws = await setup(page, { transcript: "main pane words" });
const explorerTerminal = await createReceiver("bracketed", "dictation-explorer");
const mainTerminal = await createReceiver("bracketed", "dictation-main");
await harness.openTerminal(page, { terminalId: explorerTerminal.id });
await ensureExplorerSidebar(page);
await dragTabIntoExplorer(page, explorerTerminal);
const explorer = explorerSidebar(page);
await expect(
explorer.getByTestId(`workspace-tab-terminal_${explorerTerminal.id}`),
).toBeVisible();
const main = page.getByTestId("workspace-pane-main");
await selectTerminalTab(main, mainTerminal);
await expect(explorer.getByTestId("terminal-surface")).toBeVisible();
let chunk = ws.nextChunk();
await terminalSurface(main).click();
await terminalInput(main).press("Meta+d");
await expect(visibleTestId(main, "terminal-dictation-confirm")).toBeVisible();
await expect(explorer.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
await chunk;
await terminalInput(main).press("Meta+d");
await expectReceived(mainTerminal.id, "bracketed", "main pane words ");
await expectNothingReceived(page, [explorerTerminal, "bracketed"]);
ws.setTranscript("explorer words");
chunk = ws.nextChunk();
await terminalSurface(explorer).click();
await terminalInput(explorer).press("Meta+d");
await expect(visibleTestId(explorer, "terminal-dictation-confirm")).toBeVisible();
await expect(main.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
await chunk;
await terminalInput(explorer).press("Meta+d");
await expectReceived(explorerTerminal.id, "bracketed", "explorer words ");
expect(await received(mainTerminal.id, "bracketed")).toBe("main pane words ");
});
test("Cmd+D records only in the focused split pane", async ({ page }) => {
const ws = await setup(page, { transcript: "right pane words" });
const left = await createReceiver("bracketed", "dictation-split-left");
const right = await createReceiver("bracketed", "dictation-split-right");
await harness.openTerminal(page, { terminalId: left.id });
await runWorkspaceActionFromCommandCenter(page, "Split pane right");
await visibleTestId(page, `workspace-tab-terminal_${right.id}`).click();
await runWorkspaceActionFromCommandCenter(page, "Move tab right");
const leftPane = splitChild(page, left);
const rightPane = splitChild(page, right);
await expect(terminalInput(leftPane)).toHaveCount(1);
await expect(terminalInput(rightPane)).toHaveCount(1);
const chunk = ws.nextChunk();
await terminalSurface(rightPane).click();
await terminalInput(rightPane).press("Meta+d");
await expect(visibleTestId(rightPane, "terminal-dictation-confirm")).toBeVisible();
await expect(leftPane.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
await chunk;
await terminalInput(rightPane).press("Meta+d");
await expectReceived(right.id, "bracketed", "right pane words ");
await expectNothingReceived(page, [left, "bracketed"]);
});
});
test("Ctrl+D off macOS reaches the PTY as EOF and starts nothing", async ({ page }) => {
// Chromium keeps the host's navigator.platform unless the override sets it too.
const session = await page.context().newCDPSession(page);
await session.send("Emulation.setUserAgentOverride", {
userAgent: LINUX_USER_AGENT,
platform: "Linux x86_64",
});
const ws = await setup(page);
const terminal = await createReceiver("raw", "dictation-ctrl-d");
await harness.openTerminal(page, { terminalId: terminal.id });
await terminalInput(page).press("Control+d");
await expectReceived(terminal.id, "raw", "\u0004");
expect(ws.streamStarts()).toBe(0);
expect((await microphoneStats(page)).getUserMediaCalls).toBe(0);
await expect(page.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
});
test("switching the pane to another terminal while recording stops it and delivers nothing", async ({
page,
}) => {
const ws = await setup(page);
const first = await createReceiver("bracketed", "dictation-switch-a");
const second = await createReceiver("bracketed", "dictation-switch-b");
await harness.openTerminal(page, { terminalId: first.id });
await startFromToolbar(page, ws);
await selectTerminalTab(page, second);
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
await expect(
page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
).toHaveCount(0);
await expectMicrophoneStat(page, "trackStops", 1);
const chunksAfterSwitch = ws.chunks();
const startsAfterSwitch = ws.streamStarts();
await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
expect(ws.chunks()).toBe(chunksAfterSwitch);
expect(ws.streamStarts()).toBe(startsAfterSwitch);
await selectTerminalTab(page, first);
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
await expect(
page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
).toHaveCount(0);
});
test("switching the pane to another terminal while processing drops the late transcript", async ({
page,
}) => {
const ws = await setup(page);
const first = await createReceiver("bracketed", "dictation-processing-a");
const second = await createReceiver("bracketed", "dictation-processing-b");
await harness.openTerminal(page, { terminalId: first.id });
await startFromToolbar(page, ws);
ws.holdFinal();
await visibleTestId(page, "terminal-dictation-confirm").click();
await expect(visibleTestId(page, "terminal-dictation-confirm")).toHaveCount(0);
await expect(visibleTestId(page, "terminal-dictation-cancel")).toBeVisible();
await selectTerminalTab(page, second);
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
ws.releaseFinal();
await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
});
test.describe("on compact web", () => {
// Keyboard shortcuts are off on compact form factors (keyboardShortcutsAvailable), so
// the key-bar mic is how compact web dictates.
test("the key-bar mic shows the overlay and confirm delivers", async ({ page }) => {
const ws = await setup(page, { transcript: "compact words" });
const terminal = await createReceiver("bracketed", "dictation-compact-confirm");
await openCompact(page, terminal);
const chunk = ws.nextChunk();
await visibleTestId(page, "terminal-dictation").click();
const overlay = visibleTestId(page, "terminal-dictation-overlay");
await expect(overlay).toBeVisible();
await expect(visibleTestId(page, "terminal-dictation")).toHaveCount(0);
await expect(
overlay.getByRole("button", { name: "Insert transcription and send", exact: true }),
).toHaveCount(0);
await chunk;
await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
await expectReceived(terminal.id, "bracketed", "compact words ");
await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
});
test("switching tabs while recording from the key bar stops it and delivers nothing", async ({
page,
}) => {
const ws = await setup(page);
const first = await createReceiver("bracketed", "dictation-compact-a");
const second = await createReceiver("bracketed", "dictation-compact-b");
await openCompact(page, first);
const chunk = ws.nextChunk();
await visibleTestId(page, "terminal-dictation").click();
await expect(visibleTestId(page, "terminal-dictation-overlay")).toBeVisible();
await chunk;
await selectCompactTab(page, 1);
await expectMicrophoneStat(page, "trackStops", 1);
await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
await selectCompactTab(page, 0);
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
await expect(
page.getByTestId("terminal-dictation-overlay").filter({ visible: true }),
).toHaveCount(0);
expect(await received(first.id, "bracketed")).toBe("");
});
test("a failed dictation keeps the overlay with retry and discard", async ({ page }) => {
const ws = await setup(page, { transcript: "retried words" });
const terminal = await createReceiver("bracketed", "dictation-compact-failure");
await openCompact(page, terminal);
const overlay = visibleTestId(page, "terminal-dictation-overlay");
ws.failNextFinish("Synthetic transcription failure");
let chunk = ws.nextChunk();
await visibleTestId(page, "terminal-dictation").click();
await chunk;
await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
await expect(overlay).toContainText("Dictation failed: Synthetic transcription failure");
await overlay.getByRole("button", { name: "Retry dictation", exact: true }).click();
await expectReceived(terminal.id, "bracketed", "retried words ");
await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
ws.failNextFinish("Synthetic transcription failure");
chunk = ws.nextChunk();
await visibleTestId(page, "terminal-dictation").click();
await chunk;
await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
await expect(overlay).toContainText("Dictation failed: Synthetic transcription failure");
await overlay.getByRole("button", { name: "Cancel dictation", exact: true }).click();
await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
await page.waitForTimeout(1_500);
expect(await received(terminal.id, "bracketed")).toBe("retried words ");
});
});
for (const scenario of [
{
name: "a host without the dictation capability asks to update the host",
capability: "missing" as const,
message: "Update the host to use dictation in terminals.",
},
{
name: "disabled dictation with an empty reason shows the fallback message",
capability: { enabled: false, reason: " " },
message: "Dictation is not available on this host.",
},
{
name: "disabled dictation shows the host's reason",
capability: { enabled: false, reason: "No speech model installed" },
message: "No speech model installed",
},
]) {
test(scenario.name, async ({ page }) => {
const ws = await setup(page, { capability: scenario.capability });
const terminal = await createReceiver("bracketed", "dictation-capability");
await harness.openTerminal(page, { terminalId: terminal.id });
await visibleTestId(page, "terminal-toolbar-dictation").click();
await expect(page.getByTestId("app-toast-message")).toHaveText(scenario.message);
await expect(page.getByTestId("terminal-dictation-cancel")).toHaveCount(0);
expect((await microphoneStats(page)).getUserMediaCalls).toBe(0);
expect(ws.streamStarts()).toBe(0);
});
}
test("hiding a terminal while its microphone is still being acquired releases it", async ({
page,
}) => {
const ws = await setup(page);
const first = await createReceiver("bracketed", "dictation-held-hide-a");
const second = await createReceiver("bracketed", "dictation-held-hide-b");
await harness.openTerminal(page, { terminalId: first.id });
await holdMicrophone(page);
await visibleTestId(page, "terminal-toolbar-dictation").click();
await expectMicrophoneStat(page, "getUserMediaCalls", 1);
await expect(visibleTestId(page, "terminal-dictation-cancel")).toBeVisible();
await selectTerminalTab(page, second);
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
await releaseMicrophone(page);
await expectMicrophoneStat(page, "trackStops", 1);
await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
expect(ws.streamStarts()).toBe(0);
expect(ws.chunks()).toBe(0);
await selectTerminalTab(page, first);
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
await expect(
page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
).toHaveCount(0);
await expect(
page.getByTestId("terminal-dictation-confirm").filter({ visible: true }),
).toHaveCount(0);
});
test("closing a terminal while its microphone is still being acquired releases it", async ({
page,
}) => {
const ws = await setup(page);
const first = await createReceiver("bracketed", "dictation-held-close-a");
const second = await createReceiver("bracketed", "dictation-held-close-b");
await harness.openTerminal(page, { terminalId: first.id });
await holdMicrophone(page);
await visibleTestId(page, "terminal-toolbar-dictation").click();
await expectMicrophoneStat(page, "getUserMediaCalls", 1);
// Closing the terminal closes its tab, so its pane unmounts with the start pending.
await harness.killTerminal(first.id);
await expect(page.getByTestId(`workspace-tab-terminal_${first.id}`)).toHaveCount(0);
await releaseMicrophone(page);
await expectMicrophoneStat(page, "trackStops", 1);
await expectNothingReceived(page, [second, "bracketed"]);
expect(ws.streamStarts()).toBe(0);
expect(ws.chunks()).toBe(0);
});
});
@@ -306,7 +306,11 @@ test.describe("Terminal protocol queries", () => {
try {
const upload = visibleTestId(page, "terminal-toolbar-upload");
await expect(upload).toHaveCount(1);
// The terminal slot sits right before "...": upload, then the mic.
expect(await controlBefore(visibleTestId(page, "workspace-split-pane-menu"))).toBe(
"terminal-toolbar-dictation",
);
expect(await controlBefore(visibleTestId(page, "terminal-toolbar-dictation"))).toBe(
"terminal-toolbar-upload",
);
@@ -0,0 +1,278 @@
import type { Page } from "@playwright/test";
import { daemonWsRoutePattern } from "./daemon-port";
type WebSocketMessage = string | Buffer;
interface SessionSocket {
send(message: string): void;
}
/** What the rewritten `server_info` reports for dictation; `missing` drops the voice field. */
export type DictationCapability = { enabled: boolean; reason: string } | "missing";
function parseEnvelope(message: WebSocketMessage): {
type?: unknown;
message?: Record<string, unknown>;
} | null {
const raw = typeof message === "string" ? message : message.toString("utf8");
try {
return JSON.parse(raw) as { type?: unknown; message?: Record<string, unknown> };
} catch {
return null;
}
}
export function sessionMessage(message: WebSocketMessage): Record<string, unknown> | null {
const envelope = parseEnvelope(message);
return envelope?.type === "session" && envelope.message ? envelope.message : null;
}
export function sendSessionMessage(ws: SessionSocket, message: Record<string, unknown>): void {
ws.send(JSON.stringify({ type: "session", message }));
}
function rewriteDictationCapability(
message: WebSocketMessage,
capability: DictationCapability,
): WebSocketMessage {
const envelope = parseEnvelope(message);
const payload = envelope?.message?.payload;
if (
envelope?.message?.type !== "status" ||
!payload ||
typeof payload !== "object" ||
(payload as { status?: unknown }).status !== "server_info"
) {
return message;
}
const serverInfo = payload as Record<string, unknown>;
if (capability === "missing") {
const capabilities = serverInfo.capabilities as Record<string, unknown> | undefined;
if (capabilities) {
delete capabilities.voice;
}
} else {
serverInfo.capabilities = {
voice: {
dictation: capability,
voice: { enabled: false, reason: "Realtime voice is disabled in this test." },
},
};
}
return JSON.stringify(envelope);
}
interface SyntheticMicrophoneState {
getUserMediaCalls: number;
trackStops: number;
hold: boolean;
pending: Array<() => void>;
}
/**
* Replaces getUserMedia with an oscillator stream, so the app's real web audio
* source runs end to end. Counts calls and stopped tracks, and can hold an
* acquisition until the test releases it (a permission prompt left open).
*/
export async function installSyntheticMicrophone(page: Page): Promise<void> {
await page.addInitScript(() => {
const state: SyntheticMicrophoneState = {
getUserMediaCalls: 0,
trackStops: 0,
hold: false,
pending: [],
};
(
window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState }
).__paseoSyntheticMicrophone = state;
Object.defineProperty(navigator.mediaDevices, "getUserMedia", {
configurable: true,
value: async () => {
state.getUserMediaCalls += 1;
if (state.hold) {
await new Promise<void>((resolve) => state.pending.push(resolve));
}
const context = new AudioContext();
const oscillator = context.createOscillator();
const destination = context.createMediaStreamDestination();
oscillator.connect(destination);
oscillator.start();
for (const track of destination.stream.getTracks()) {
const stop = track.stop.bind(track);
track.stop = () => {
state.trackStops += 1;
stop();
};
}
return destination.stream;
},
});
});
}
export async function microphoneStats(
page: Page,
): Promise<{ getUserMediaCalls: number; trackStops: number }> {
return await page.evaluate(() => {
const state = (window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState })
.__paseoSyntheticMicrophone;
return { getUserMediaCalls: state.getUserMediaCalls, trackStops: state.trackStops };
});
}
/** Makes the next getUserMedia calls wait until `releaseMicrophone`. */
export async function holdMicrophone(page: Page): Promise<void> {
await page.evaluate(() => {
(
window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState }
).__paseoSyntheticMicrophone.hold = true;
});
}
export async function releaseMicrophone(page: Page): Promise<void> {
await page.evaluate(() => {
const state = (window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState })
.__paseoSyntheticMicrophone;
state.hold = false;
for (const resolve of state.pending.splice(0)) {
resolve();
}
});
}
export interface DictationWsHarness {
/** Every `dictation_stream_start` the app sent. */
streamStarts(): number;
/** Audio chunks the app sent, across all dictation ids. */
chunks(): number;
/** Resolves on the first chunk the app sends after this call. */
nextChunk(): Promise<void>;
/** The final transcript the next finish replies with. */
setTranscript(text: string): void;
/** Keeps the next final reply until `releaseFinal`. */
holdFinal(): void;
releaseFinal(): void;
/** Answers the next finish with a stream error instead of a final. */
failNextFinish(error: string): void;
}
/**
* Answers the app's dictation stream in place of the daemon: acknowledges starts
* and chunks, accepts finishes and replies with the scenario's transcript. Rewrites
* `server_info` to the given dictation capability and forwards everything else, so
* terminals, agents and workspaces keep using the real daemon.
* `interceptClientMessage` lets a spec answer other requests itself; return true
* when it handled one.
*/
export async function installDictationWsHarness(
page: Page,
options: {
transcript?: string;
capability?: DictationCapability;
interceptClientMessage?: (request: Record<string, unknown>, ws: SessionSocket) => boolean;
} = {},
): Promise<DictationWsHarness> {
const capability = options.capability ?? { enabled: true, reason: "" };
let transcript = options.transcript ?? "";
let starts = 0;
let chunkCount = 0;
let chunkWaiters: Array<() => void> = [];
let holdNextFinal = false;
let heldFinal: (() => void) | null = null;
let nextFinishError: string | null = null;
await page.routeWebSocket(daemonWsRoutePattern(), (ws) => {
const server = ws.connectToServer();
ws.onMessage((message) => {
const request = sessionMessage(message);
const type = request?.type;
const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null;
if (type === "dictation_stream_start" && dictationId) {
starts += 1;
sendSessionMessage(ws, {
type: "dictation_stream_ack",
payload: { dictationId, ackSeq: -1 },
});
return;
}
if (type === "dictation_stream_chunk" && dictationId) {
chunkCount += 1;
const seq = typeof request?.seq === "number" ? request.seq : 0;
sendSessionMessage(ws, {
type: "dictation_stream_ack",
payload: { dictationId, ackSeq: seq },
});
const waiters = chunkWaiters;
chunkWaiters = [];
for (const resolve of waiters) {
resolve();
}
return;
}
if (type === "dictation_stream_finish" && dictationId) {
sendSessionMessage(ws, {
type: "dictation_stream_finish_accepted",
payload: { dictationId, timeoutMs: 30_000 },
});
if (nextFinishError !== null) {
const error = nextFinishError;
nextFinishError = null;
sendSessionMessage(ws, {
type: "dictation_stream_error",
payload: { dictationId, error, retryable: true },
});
return;
}
const text = transcript;
const reply = () =>
sendSessionMessage(ws, {
type: "dictation_stream_final",
payload: { dictationId, text },
});
if (holdNextFinal) {
holdNextFinal = false;
heldFinal = reply;
} else {
reply();
}
return;
}
if (type === "dictation_stream_cancel") {
return;
}
if (request && options.interceptClientMessage?.(request, ws)) {
return;
}
server.send(message);
});
server.onMessage((message) => ws.send(rewriteDictationCapability(message, capability)));
});
return {
streamStarts: () => starts,
chunks: () => chunkCount,
nextChunk: () => new Promise<void>((resolve) => chunkWaiters.push(resolve)),
setTranscript: (text) => {
transcript = text;
},
holdFinal: () => {
holdNextFinal = true;
},
releaseFinal: () => {
const reply = heldFinal;
heldFinal = null;
if (!reply) {
throw new Error("No final reply is being held");
}
reply();
},
failNextFinish: (error) => {
nextFinishError = error;
},
};
}
@@ -7,6 +7,7 @@ import { useTranslation } from "react-i18next";
import { VolumeMeter } from "./volume-meter";
import { FOOTER_HEIGHT } from "@/constants/layout";
import type { DictationStatus } from "@/hooks/use-dictation";
import { formatDictationDuration } from "@/utils/format-dictation-duration";
interface DictationControlsProps {
volume: number;
@@ -24,12 +25,6 @@ interface DictationControlsProps {
disabled?: boolean;
}
function formatDuration(seconds: number): string {
const mins = Math.floor(seconds / 60);
const secs = seconds % 60;
return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
}
export function DictationControls({
volume,
duration,
@@ -88,7 +83,7 @@ export function DictationControls({
<VolumeMeter volume={volume} isMuted={false} isSpeaking={false} orientation="horizontal" />
</View>
<Text style={timerTextStyle}>
{status === "paused" ? t("message.dictation.paused") : formatDuration(duration)}
{status === "paused" ? t("message.dictation.paused") : formatDictationDuration(duration)}
</Text>
<View style={styles.actionGroup}>
<Pressable
@@ -152,7 +147,13 @@ export function DictationOverlay({
onAcceptAndSend,
onRetry,
onDiscard,
}: Omit<DictationControlsProps, "onStart" | "disabled" | "transcript"> & { errorText?: string }) {
testID,
}: Omit<DictationControlsProps, "onStart" | "disabled" | "transcript" | "onAcceptAndSend"> & {
errorText?: string;
/** Omit to show insert as the only confirm action (terminals never send). */
onAcceptAndSend?: () => void;
testID?: string;
}) {
const { theme } = useUnistyles();
const { t } = useTranslation();
const isFailed = status === "failed";
@@ -190,7 +191,7 @@ export function DictationOverlay({
}
return (
<View style={containerStyle}>
<View style={containerStyle} testID={testID}>
<Pressable
onPress={handleCancel}
disabled={actionsDisabled && !isFailed}
@@ -210,7 +211,7 @@ export function DictationOverlay({
orientation="horizontal"
color={theme.colors.accentForeground}
/>
<Text style={overlayTimerTextStyle}>{formatDuration(duration)}</Text>
<Text style={overlayTimerTextStyle}>{formatDictationDuration(duration)}</Text>
</View>
{status === "paused" ? (
<Text numberOfLines={2} style={overlayTranscriptTextStyle}>
@@ -242,7 +243,7 @@ export function DictationOverlay({
<RefreshCcw size={theme.iconSize.lg} color={theme.colors.accent} strokeWidth={2.5} />
</Pressable>
) : null}
{!actionsDisabled && !isFailed ? (
{!actionsDisabled && !isFailed && onAcceptAndSend ? (
<>
<Pressable
onPress={onAccept}
@@ -266,6 +267,16 @@ export function DictationOverlay({
</Pressable>
</>
) : null}
{!actionsDisabled && !isFailed && !onAcceptAndSend ? (
<Pressable
onPress={onAccept}
accessibilityRole="button"
accessibilityLabel={t("message.dictation.insert")}
style={overlayConfirmButtonStyle}
>
<Check size={theme.iconSize.lg} color={theme.colors.accent} strokeWidth={2.5} />
</Pressable>
) : null}
</View>
</View>
);
+62 -13
View File
@@ -2,7 +2,7 @@ import { TerminalFind, type TerminalPaneFindHandle } from "@/terminal/find";
import type { TerminalFindResult } from "@/terminal/runtime/terminal-emulator-runtime";
import { LoadingSpinner } from "@/components/ui/loading-spinner";
import { useRetainedPanelActive } from "@/components/retained-panel";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react";
import * as Clipboard from "expo-clipboard";
import {
Pressable,
@@ -63,7 +63,20 @@ import {
type TerminalDropDelivery,
} from "@/terminal/drop/terminal-file-drop";
import { preparePathsForTerminalInput } from "@/terminal/drop/terminal-path";
import { useRegisterTerminalInputSink, type TerminalInputSink } from "@/terminal/input-sink";
import {
useRegisterTerminalInputSink,
type TerminalDictationActions,
type TerminalInputSink,
} from "@/terminal/input-sink";
import {
TerminalDictationOwner,
type TerminalDictationOwnerActions,
} from "@/terminal/dictation/terminal-dictation-owner";
import { isTerminalKeyboardOwner } from "@/terminal/dictation/terminal-dictation-lifecycle";
import {
TerminalKeyBarDictationButton,
TerminalKeyBarDictationGate,
} from "@/terminal/toolbar/terminal-key-bar-dictation";
import type { SelectedFile } from "@/attachments/selected-file";
import { TerminalUploadButton } from "@/terminal/toolbar/terminal-upload-button";
import { toErrorMessage } from "@/utils/error-messages";
@@ -91,7 +104,7 @@ import {
type TerminalResizeRequest,
} from "./terminal-resize-debouncer";
import { useIsCompactFormFactor } from "@/constants/layout";
import { isNative } from "@/constants/platform";
import { isNative, isWeb } from "@/constants/platform";
import {
applyTerminalRendererReadyChange,
resolveTerminalStreamTarget,
@@ -950,13 +963,34 @@ export function TerminalPane({
);
});
// The owner mounts only while connected and remounts per terminal; the sink keeps
// one stable forwarding object and reaches whichever owner is mounted now.
const dictationOwnerId = useId();
const dictationActionsRef = useRef<TerminalDictationOwnerActions | null>(null);
const dictationActions = useMemo<TerminalDictationActions>(
() => ({
ownerId: dictationOwnerId,
toggle: () => dictationActionsRef.current?.toggle(),
cancel: () => dictationActionsRef.current?.cancel(),
confirm: () => dictationActionsRef.current?.confirm(),
retry: () => dictationActionsRef.current?.retry(),
discard: () => dictationActionsRef.current?.discard(),
}),
[dictationOwnerId],
);
const terminalSurfaceRef = useRef<View>(null);
const isDictationKeyboardOwner = useStableEvent(
() => isWeb && isTerminalKeyboardOwner(terminalSurfaceRef.current, document.activeElement),
);
const inputSink = useMemo<TerminalInputSink>(
() => ({
pasteText: pasteTerminalText,
deliverFiles: deliverPickedFiles,
focus: requestTerminalFocus,
dictation: dictationActions,
}),
[pasteTerminalText, deliverPickedFiles, requestTerminalFocus],
[pasteTerminalText, deliverPickedFiles, requestTerminalFocus, dictationActions],
);
useRegisterTerminalInputSink(serverId, terminalId, inputSink);
@@ -1203,6 +1237,8 @@ export function TerminalPane({
) : null;
case "upload":
return <TerminalUploadButton key={controlId} sink={inputSink} placement="keybar" />;
case "dictation":
return <TerminalKeyBarDictationButton key={controlId} sink={inputSink} />;
case "keyboardToggle":
return (
<KeyboardToggleButton
@@ -1232,9 +1268,20 @@ export function TerminalPane({
return (
<Animated.View style={containerStyle}>
<TerminalDictationOwner
key={`${serverId}:${terminalId}`}
ownerId={dictationOwnerId}
serverId={serverId}
terminalId={terminalId}
isPresented={isTerminalPresented}
pasteText={pasteTerminalText}
focus={requestTerminalFocus}
isKeyboardOwner={isDictationKeyboardOwner}
actionsRef={dictationActionsRef}
/>
<FileDropZone style={styles.outputContainer}>
<TerminalFileDropSink onFiles={handleDroppedFiles} />
<View style={styles.terminalGestureContainer}>
<View ref={terminalSurfaceRef} style={styles.terminalGestureContainer}>
<TerminalEmulator
ref={emulatorRef}
dom={TERMINAL_EMULATOR_DOM_PROPS}
@@ -1308,14 +1355,16 @@ export function TerminalPane({
onLayout={handleKeyBarLayout}
testID={isNative && !isMobile ? "terminal-virtual-keyboard-native-wide" : undefined}
>
{virtualKeyboardRows.map((row) => (
<View
key={row.map(getTerminalVirtualKeyboardControlId).join(":")}
style={styles.keyboardRow}
>
{row.map(renderVirtualKeyboardControl)}
</View>
))}
<TerminalKeyBarDictationGate sink={inputSink}>
{virtualKeyboardRows.map((row) => (
<View
key={row.map(getTerminalVirtualKeyboardControlId).join(":")}
style={styles.keyboardRow}
>
{row.map(renderVirtualKeyboardControl)}
</View>
))}
</TerminalKeyBarDictationGate>
</View>
</View>
) : null}
@@ -2,6 +2,7 @@ import { resampleToPcm16 } from "@/audio/pcm";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { parsePcm16Wav } from "@/utils/pcm16-wav";
import { isElectronRuntime } from "@/desktop/host";
import { AttemptCancelledError } from "@/utils/attempt-guard";
import type {
DictationAudioSource,
@@ -154,6 +155,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
started: boolean;
mode: "pcm" | "recorder";
recorder: RecorderRefs;
// Advanced by every stop(), so a start still waiting on getUserMedia can tell it
// was stopped and must not install its stream.
startGeneration: number;
}>({
stream: null,
context: null,
@@ -170,6 +174,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
stoppedResolve: null,
stoppedReject: null,
},
startGeneration: 0,
});
const decodeAudioData = useCallback(
@@ -233,7 +238,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
throw new Error("AudioContext unavailable");
}
const rawStream = await navigator.mediaDevices.getUserMedia({
const startGeneration = refs.current.startGeneration;
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
noiseSuppression: true,
@@ -241,8 +247,11 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
autoGainControl: true,
},
});
const stream = rawStream;
if (refs.current.startGeneration !== startGeneration) {
// Stopped while the permission prompt or device acquisition was pending.
disconnectDictationAudioGraph({ processor: null, source: null, gain: null, stream });
throw new AttemptCancelledError("Dictation start was cancelled");
}
const context = new AudioContextCtor();
@@ -371,6 +380,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
const stop = useCallback(async () => {
refs.current.started = false;
refs.current.startGeneration += 1;
setVolume(0);
const { processor, source, gain, context, stream, pending, mode, recorder } = refs.current;
@@ -418,6 +428,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
stoppedResolve: null,
stoppedReject: null,
},
startGeneration: refs.current.startGeneration,
};
}, [decodeAudioData, emitPcmSegments]);
@@ -436,7 +447,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
await start();
} catch (err) {
const normalized = err instanceof Error ? err : new Error(String(err));
onErrorRef.current?.(normalized);
if (!(normalized instanceof AttemptCancelledError)) {
onErrorRef.current?.(normalized);
}
throw normalized;
}
},
@@ -0,0 +1,203 @@
/**
* @vitest-environment jsdom
*/
import { act, renderHook } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { AttemptCancelledError } from "@/utils/attempt-guard";
import type { UseDictationOptions } from "./use-dictation.shared";
// jsdom has no microphone: the source's start() returns a promise the test settles,
// standing in for a permission prompt or a getUserMedia call that is still pending.
const audioSource = vi.hoisted(() => {
const state = {
settleStart: null as null | { resolve: () => void; reject: (error: unknown) => void },
start: null as unknown as ReturnType<typeof import("vitest").vi.fn>,
stop: null as unknown as ReturnType<typeof import("vitest").vi.fn>,
onPcmSegment: null as null | ((audio: string) => void),
};
return state;
});
vi.mock("@/hooks/use-dictation-audio-source", () => ({
useDictationAudioSource: (config: { onPcmSegment: (audio: string) => void }) => {
audioSource.onPcmSegment = config.onPcmSegment;
return {
start: audioSource.start,
stop: audioSource.stop,
volume: 0,
isPaused: false,
};
},
}));
const { useDictation } = await import("./use-dictation");
function createFakeClient() {
return {
isConnected: true,
startDictationStream: vi.fn(async () => {}),
sendDictationStreamChunk: vi.fn(),
finishDictationStream: vi.fn(async (dictationId: string) => ({ dictationId, text: "" })),
cancelDictationStream: vi.fn(),
subscribeRawMessages: vi.fn(() => () => {}),
subscribeConnectionStatus: vi.fn(() => () => {}),
on: vi.fn(() => () => {}),
};
}
function renderDictation(client: ReturnType<typeof createFakeClient>) {
const onError = vi.fn();
const onTranscript = vi.fn();
const rendered = renderHook(() =>
useDictation({
client: client as unknown as UseDictationOptions["client"],
onTranscript,
onError,
}),
);
return { ...rendered, onError, onTranscript };
}
async function flush(): Promise<void> {
await act(async () => {
await Promise.resolve();
await Promise.resolve();
});
}
beforeEach(() => {
audioSource.settleStart = null;
audioSource.start = vi.fn(
() =>
new Promise<void>((resolve, reject) => {
audioSource.settleStart = { resolve, reject };
}),
);
audioSource.stop = vi.fn(async () => {});
});
afterEach(() => {
vi.restoreAllMocks();
});
describe("useDictation pending startup", () => {
it("cancels a start that is still acquiring the microphone", async () => {
const client = createFakeClient();
const { result, onError } = renderDictation(client);
act(() => {
void result.current.startDictation();
});
expect(result.current.status).toBe("recording");
expect(audioSource.settleStart).not.toBeNull();
await act(async () => {
await result.current.cancelDictation();
});
expect(result.current.status).toBe("idle");
// The source is stopped while acquisition is pending, so it can release a late stream itself.
expect(audioSource.stop).toHaveBeenCalledTimes(1);
await act(async () => {
audioSource.settleStart?.resolve();
});
await flush();
expect(audioSource.stop).toHaveBeenCalledTimes(2);
expect(result.current.isRecording).toBe(false);
expect(result.current.isRecordingActive()).toBe(false);
expect(result.current.status).toBe("idle");
expect(client.startDictationStream).not.toHaveBeenCalled();
expect(onError).not.toHaveBeenCalled();
expect(result.current.error).toBeNull();
});
it("treats the source's cancelled acquisition as a silent cancel", async () => {
const client = createFakeClient();
const { result, onError } = renderDictation(client);
act(() => {
void result.current.startDictation();
});
await act(async () => {
await result.current.cancelDictation();
});
await act(async () => {
audioSource.settleStart?.reject(new AttemptCancelledError());
});
await flush();
expect(result.current.status).toBe("idle");
expect(result.current.isRecording).toBe(false);
expect(client.startDictationStream).not.toHaveBeenCalled();
expect(onError).not.toHaveBeenCalled();
expect(result.current.error).toBeNull();
});
it("stops the late-acquired microphone after an unmount during acquisition", async () => {
const client = createFakeClient();
const { result, unmount, onError } = renderDictation(client);
act(() => {
void result.current.startDictation();
});
unmount();
const stopsAtUnmount = audioSource.stop.mock.calls.length;
expect(stopsAtUnmount).toBeGreaterThanOrEqual(1);
await act(async () => {
audioSource.settleStart?.resolve();
});
await flush();
expect(audioSource.stop.mock.calls.length).toBe(stopsAtUnmount + 1);
expect(client.startDictationStream).not.toHaveBeenCalled();
expect(onError).not.toHaveBeenCalled();
});
it("records and opens the stream when the start is not cancelled", async () => {
const client = createFakeClient();
const { result, onError } = renderDictation(client);
act(() => {
void result.current.startDictation();
});
await act(async () => {
audioSource.settleStart?.resolve();
});
await flush();
expect(result.current.isRecording).toBe(true);
expect(result.current.status).toBe("recording");
expect(client.startDictationStream).toHaveBeenCalledTimes(1);
expect(audioSource.stop).not.toHaveBeenCalled();
expect(onError).not.toHaveBeenCalled();
});
it("drops the partial segment the source flushes while a recording is cancelled", async () => {
const client = createFakeClient();
// Stopping a real source flushes the audio it still holds.
audioSource.stop = vi.fn(async () => {
audioSource.onPcmSegment?.("AAAA");
});
const { result } = renderDictation(client);
act(() => {
void result.current.startDictation();
});
await act(async () => {
audioSource.settleStart?.resolve();
});
await flush();
expect(client.startDictationStream).toHaveBeenCalledTimes(1);
await act(async () => {
await result.current.cancelDictation();
});
await flush();
expect(result.current.status).toBe("idle");
expect(client.startDictationStream).toHaveBeenCalledTimes(1);
expect(client.sendDictationStreamChunk).not.toHaveBeenCalled();
});
});
+62 -1
View File
@@ -70,12 +70,19 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const attemptGuardRef = useRef(new AttemptGuard());
// Advanced only by cancel and unmount, never by confirm: a confirm while the stream
// is still starting must not make the start tear itself down.
const startGuardRef = useRef(new AttemptGuard());
const actionGateRef = useRef<{ starting: boolean; confirming: boolean; cancelling: boolean }>({
starting: false,
confirming: false,
cancelling: false,
});
// Audio is accepted from start until a cancel: stopping the source flushes its last
// partial segment, which after a cancel would open a new stream for audio nobody wants.
const acceptAudioRef = useRef(false);
const senderRef = useRef<DictationStreamSender | null>(null);
if (!senderRef.current) {
senderRef.current = new DictationStreamSender({
@@ -228,6 +235,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const audio = useDictationAudioSource({
onPcmSegment: (audioData) => {
if (!acceptAudioRef.current) {
return;
}
senderRef.current?.enqueueSegment(audioData);
},
onError: (err) => {
@@ -264,15 +274,36 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
}
actionGateRef.current.starting = true;
// A cancel or unmount while the microphone is still being acquired advances the
// start guard; the checks below then release whatever the start acquired late.
const startId = startGuardRef.current.next();
const abandonStaleStart = async () => {
acceptAudioRef.current = false;
try {
senderRef.current?.cancel();
} catch {
// no-op
}
await audio.stop().catch(() => undefined);
stopDurationTracking();
isRecordingRef.current = false;
setIsRecording(false);
setStatus("idle");
};
setError(null);
setPartialTranscript("");
setDuration(0);
setIsProcessing(false);
setStatus("recording");
clearStreamingState();
acceptAudioRef.current = true;
try {
await audio.start();
if (!startGuardRef.current.isCurrent(startId)) {
await abandonStaleStart();
return;
}
isRecordingRef.current = true;
setIsRecording(true);
if (enableDuration) {
@@ -280,8 +311,16 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
}
if (client?.isConnected) {
await startNewStream("start");
if (!startGuardRef.current.isCurrent(startId)) {
await abandonStaleStart();
return;
}
}
} catch (err) {
if (!startGuardRef.current.isCurrent(startId)) {
await abandonStaleStart();
return;
}
await audio.stop().catch(() => undefined);
stopDurationTracking();
isRecordingRef.current = false;
@@ -303,15 +342,35 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
stopDurationTracking,
]);
// The start is still waiting on the microphone. Show idle at once and stop the
// source now, so a stream acquired after this point is released by the source
// itself; the start continuation then sees the stale attempt and tears down the rest.
const cancelPendingStart = useCallback(async () => {
acceptAudioRef.current = false;
stopDurationTracking();
setDuration(0);
setError(null);
setStatus("idle");
clearStreamingState();
await audio.stop().catch((err: unknown) => {
reportError(err, "Failed to stop pending dictation audio");
});
}, [audio, clearStreamingState, reportError, stopDurationTracking]);
const cancelDictation = useCallback(async () => {
attemptGuardRef.current.cancel();
startGuardRef.current.cancel();
if (actionGateRef.current.cancelling) {
return;
}
if (!isRecordingRef.current && !isProcessingRef.current) {
if (actionGateRef.current.starting) {
await cancelPendingStart();
}
return;
}
actionGateRef.current.cancelling = true;
acceptAudioRef.current = false;
stopDurationTracking();
setDuration(0);
setError(null);
@@ -334,7 +393,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
clearStreamingState();
actionGateRef.current.cancelling = false;
}
}, [audio, clearStreamingState, reportError, stopDurationTracking]);
}, [audio, cancelPendingStart, clearStreamingState, reportError, stopDurationTracking]);
const confirmDictation = useCallback(async () => {
if (actionGateRef.current.confirming) {
@@ -444,9 +503,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
useEffect(() => {
const attemptGuard = attemptGuardRef.current;
const startGuard = startGuardRef.current;
const audioStop = audioStopRef;
return () => {
attemptGuard.cancel();
startGuard.cancel();
stopDurationTracking();
void audioStop.current().catch(() => undefined);
senderRef.current?.dispose();
+8
View File
@@ -622,6 +622,13 @@ export const ar: TranslationResources = {
uploadingFile: "جارٍ رفع الملف…",
actions: {
uploadFile: "رفع ملف",
startDictation: "بدء الإملاء",
stopDictation: "إيقاف ولصق النص المفرغ",
},
dictation: {
updateHost: "حدّث المضيف لاستخدام الإملاء في الطرفيات.",
unavailable: "الإملاء غير متاح على هذا المضيف.",
discard: "تجاهل التسجيل",
},
},
sheet: {
@@ -2607,6 +2614,7 @@ export const ar: TranslationResources = {
cycleAgentMode: "تبديل وضع الوكيل",
toggleVoiceMode: "تبديل الوضع الصوتي",
startStopDictation: "بدء إملاء /stop",
startStopTerminalDictation: "بدء/إيقاف الإملاء في الطرفية",
interruptAgent: "عامل المقاطعة",
sendMessage: "أرسل رسالة",
queueMessage: "رسالة قائمة الانتظار",
+8
View File
@@ -619,6 +619,13 @@ export const en = {
uploadingFile: "Uploading file…",
actions: {
uploadFile: "Upload file",
startDictation: "Start dictation",
stopDictation: "Stop and paste transcript",
},
dictation: {
updateHost: "Update the host to use dictation in terminals.",
unavailable: "Dictation is not available on this host.",
discard: "Discard recording",
},
},
sheet: {
@@ -2694,6 +2701,7 @@ export const en = {
cycleAgentMode: "Cycle agent mode",
toggleVoiceMode: "Toggle voice mode",
startStopDictation: "Start/stop dictation",
startStopTerminalDictation: "Start/stop dictation in terminal",
interruptAgent: "Interrupt agent",
sendMessage: "Send message",
queueMessage: "Queue message",
+8
View File
@@ -628,6 +628,13 @@ export const es: TranslationResources = {
uploadingFile: "Subiendo archivo…",
actions: {
uploadFile: "Subir archivo",
startDictation: "Iniciar dictado",
stopDictation: "Detener y pegar la transcripción",
},
dictation: {
updateHost: "Actualiza el host para usar el dictado en terminales.",
unavailable: "El dictado no está disponible en este host.",
discard: "Descartar grabación",
},
},
sheet: {
@@ -2625,6 +2632,7 @@ export const es: TranslationResources = {
cycleAgentMode: "Alternar modo del agente",
toggleVoiceMode: "Alternar modo de voz",
startStopDictation: "Iniciar dictado/stop",
startStopTerminalDictation: "Iniciar/detener dictado en la terminal",
interruptAgent: "agente de interrupción",
sendMessage: "enviar mensaje",
queueMessage: "mensaje de cola",
+8
View File
@@ -628,6 +628,13 @@ export const fr: TranslationResources = {
uploadingFile: "Envoi du fichier…",
actions: {
uploadFile: "Envoyer un fichier",
startDictation: "Démarrer la dictée",
stopDictation: "Arrêter et coller la transcription",
},
dictation: {
updateHost: "Mettez à jour l'hôte pour utiliser la dictée dans les terminaux.",
unavailable: "La dictée n'est pas disponible sur cet hôte.",
discard: "Supprimer l'enregistrement",
},
},
sheet: {
@@ -2626,6 +2633,7 @@ export const fr: TranslationResources = {
cycleAgentMode: "Parcourir les modes de l’agent",
toggleVoiceMode: "Activer/désactiver le mode vocal",
startStopDictation: "Démarrer/arrêter la dictée",
startStopTerminalDictation: "Démarrer/arrêter la dictée dans le terminal",
interruptAgent: "Interrompre l’agent",
sendMessage: "Envoyer le message",
queueMessage: "Mettre le message en file d’attente",
+8
View File
@@ -628,6 +628,13 @@ export const ja: TranslationResources = {
uploadingFile: "ファイルをアップロード中…",
actions: {
uploadFile: "ファイルをアップロード",
startDictation: "音声入力を開始",
stopDictation: "停止して文字起こしを貼り付け",
},
dictation: {
updateHost: "ターミナルで音声入力を使うにはホストを更新してください。",
unavailable: "このホストでは音声入力を利用できません。",
discard: "録音を破棄",
},
},
sheet: {
@@ -2592,6 +2599,7 @@ export const ja: TranslationResources = {
cycleAgentMode: "エージェントモードを順に切り替え",
toggleVoiceMode: "音声モードを切り替え",
startStopDictation: "音声入力を開始/停止",
startStopTerminalDictation: "ターミナルで音声入力を開始/停止",
interruptAgent: "エージェントを中断",
sendMessage: "メッセージを送信",
queueMessage: "メッセージをキューに追加",
+8
View File
@@ -623,6 +623,13 @@ export const ko: TranslationResources = {
uploadingFile: "파일 업로드 중…",
actions: {
uploadFile: "파일 업로드",
startDictation: "받아쓰기 시작",
stopDictation: "중지하고 받아쓴 내용 붙여넣기",
},
dictation: {
updateHost: "터미널에서 받아쓰기를 사용하려면 호스트를 업데이트하세요.",
unavailable: "이 호스트에서는 받아쓰기를 사용할 수 없습니다.",
discard: "녹음 삭제",
},
},
sheet: {
@@ -2581,6 +2588,7 @@ export const ko: TranslationResources = {
cycleAgentMode: "에이전트 모드 전환",
toggleVoiceMode: "음성 모드 토글",
startStopDictation: "받아쓰기 시작/중지",
startStopTerminalDictation: "터미널에서 받아쓰기 시작/중지",
interruptAgent: "에이전트 중단",
sendMessage: "메시지 보내기",
queueMessage: "메시지 대기열에 추가",
+8
View File
@@ -627,6 +627,13 @@ export const ptBR: TranslationResources = {
uploadingFile: "Enviando arquivo…",
actions: {
uploadFile: "Enviar arquivo",
startDictation: "Iniciar ditado",
stopDictation: "Parar e colar a transcrição",
},
dictation: {
updateHost: "Atualize o host para usar o ditado nos terminais.",
unavailable: "O ditado não está disponível neste host.",
discard: "Descartar gravação",
},
},
sheet: {
@@ -2606,6 +2613,7 @@ export const ptBR: TranslationResources = {
cycleAgentMode: "Alternar modo do agente",
toggleVoiceMode: "Alternar modo de voz",
startStopDictation: "Iniciar/parar ditado",
startStopTerminalDictation: "Iniciar/parar ditado no terminal",
interruptAgent: "Interromper agente",
sendMessage: "Enviar mensagem",
queueMessage: "Enfileirar mensagem",
+8
View File
@@ -629,6 +629,13 @@ export const ru: TranslationResources = {
uploadingFile: "Загрузка файла…",
actions: {
uploadFile: "Загрузить файл",
startDictation: "Начать диктовку",
stopDictation: "Остановить и вставить расшифровку",
},
dictation: {
updateHost: "Обновите хост, чтобы использовать диктовку в терминалах.",
unavailable: "Диктовка недоступна на этом хосте.",
discard: "Удалить запись",
},
},
sheet: {
@@ -2613,6 +2620,7 @@ export const ru: TranslationResources = {
cycleAgentMode: "Переключить режим агента",
toggleVoiceMode: "Переключить голосовой режим",
startStopDictation: "Начать/остановить диктовку",
startStopTerminalDictation: "Начать/остановить диктовку в терминале",
interruptAgent: "Прервать агента",
sendMessage: "Отправить сообщение",
queueMessage: "Поставить сообщение в очередь",
+8
View File
@@ -622,6 +622,13 @@ export const zhCN: TranslationResources = {
uploadingFile: "正在上传文件…",
actions: {
uploadFile: "上传文件",
startDictation: "开始听写",
stopDictation: "停止并粘贴转写内容",
},
dictation: {
updateHost: "请更新主机以在终端中使用听写。",
unavailable: "此主机不支持听写。",
discard: "丢弃录音",
},
},
sheet: {
@@ -2543,6 +2550,7 @@ export const zhCN: TranslationResources = {
cycleAgentMode: "循环切换代理模式",
toggleVoiceMode: "切换语音模式",
startStopDictation: "开始/停止听写",
startStopTerminalDictation: "在终端中开始/停止听写",
interruptAgent: "中断 Agent",
sendMessage: "发送消息",
queueMessage: "消息排队",
+1
View File
@@ -48,6 +48,7 @@ export type KeyboardActionId =
| "command-center.files"
| "shortcuts.dialog.toggle"
| "workspace.terminal.new"
| "terminal.dictation-toggle"
| "workspace.new"
| "workspace.project.pick"
| "worktree.new"
@@ -44,6 +44,7 @@ export type KeyboardActionId =
| "workspace.pane.close"
| "workspace.focus.toggle"
| "workspace.terminal.new"
| "terminal.dictation-toggle"
| "workspace.browser.new"
| "sidebar.toggle.right"
| "sidebar.toggle.both"
@@ -102,6 +103,7 @@ export type KeyboardActionDefinition =
| { id: "workspace.pane.close"; scope: KeyboardActionScope }
| { id: "workspace.focus.toggle"; scope: KeyboardActionScope }
| { id: "workspace.terminal.new"; scope: KeyboardActionScope }
| { id: "terminal.dictation-toggle"; scope: KeyboardActionScope }
| { id: "workspace.browser.new"; scope: KeyboardActionScope }
| { id: "sidebar.toggle.right"; scope: KeyboardActionScope }
| { id: "sidebar.toggle.both"; scope: KeyboardActionScope }
@@ -323,6 +323,12 @@ describe("keyboard-shortcuts", () => {
action: "message-input.action",
payload: { kind: "dictation-toggle" },
},
{
name: "routes Cmd+D to terminal dictation, not the message input, when terminal is focused on macOS",
event: { key: "d", code: "KeyD", metaKey: true },
context: { isMac: true, focusScope: "terminal" },
action: "terminal.dictation-toggle",
},
{
name: "routes Shift+Tab to cycle agent mode from the message input",
event: { key: "Tab", code: "Tab", shiftKey: true },
@@ -481,9 +487,14 @@ describe("keyboard-shortcuts", () => {
context: { isMac: false, focusScope: "terminal" },
},
{
name: "does not route message-input actions when terminal is focused",
name: "leaves Ctrl+D to the PTY as EOF when terminal is focused on non-mac",
event: { key: "d", code: "KeyD", ctrlKey: true },
context: { isMac: false, focusScope: "terminal" },
},
{
name: "does not bind Cmd+D to terminal dictation on non-mac",
event: { key: "d", code: "KeyD", metaKey: true },
context: { isMac: true, focusScope: "terminal" },
context: { isMac: false, focusScope: "terminal" },
},
{
name: "does not cycle agent mode outside the message input",
@@ -199,6 +199,7 @@ export const SHORTCUT_HELP_ROW_ORDER: Record<ShortcutSectionId, readonly string[
"cycle-agent-mode",
"voice-toggle",
"dictation-toggle",
"terminal-dictation-toggle",
"agent-interrupt",
"voice-mute-toggle",
],
@@ -246,6 +247,7 @@ const SHORTCUT_HELP_LABEL_KEYS: Record<string, string> = {
"cycle-agent-mode": "settings.shortcuts.help.cycleAgentMode",
"voice-toggle": "settings.shortcuts.help.toggleVoiceMode",
"dictation-toggle": "settings.shortcuts.help.startStopDictation",
"terminal-dictation-toggle": "settings.shortcuts.help.startStopTerminalDictation",
"agent-interrupt": "settings.shortcuts.help.interruptAgent",
"voice-mute-toggle": "settings.shortcuts.help.muteUnmuteVoiceMode",
};
@@ -1136,6 +1138,18 @@ const SHORTCUT_BINDINGS: readonly ShortcutBinding[] = [
label: "Start/stop dictation",
},
},
// Terminals take Cmd+D on macOS only: elsewhere Ctrl+D is the PTY's EOF and passes through.
{
id: "terminal-dictation-toggle-cmd-d-mac",
action: "terminal.dictation-toggle",
combo: "Cmd+D",
when: { mac: true, commandCenter: false, focusScope: "terminal" },
help: {
id: "terminal-dictation-toggle",
section: "agent-input",
label: "Start/stop dictation in terminal",
},
},
{
id: "agent-interrupt",
action: "agent.interrupt",
@@ -39,6 +39,7 @@ describe("routeKeyboardShortcut — dispatch passthroughs", () => {
["workspace.pin", { id: "workspace.pin", scope: "sidebar" }],
["worktree.new", { id: "worktree.new", scope: "sidebar" }],
["workspace.terminal.new", { id: "workspace.terminal.new", scope: "workspace" }],
["terminal.dictation-toggle", { id: "terminal.dictation-toggle", scope: "workspace" }],
["workspace.tab.close.current", { id: "workspace.tab.close-current", scope: "workspace" }],
["sidebar.toggle.right", { id: "sidebar.toggle.right", scope: "sidebar" }],
["workspace.pane.split.right", { id: "workspace.pane.split.right", scope: "workspace" }],
@@ -51,6 +51,7 @@ const PASSTHROUGH_DISPATCH: Record<string, KeyboardActionDefinition> = {
"workspace.pin": { id: "workspace.pin", scope: "sidebar" },
"worktree.new": { id: "worktree.new", scope: "sidebar" },
"workspace.terminal.new": { id: "workspace.terminal.new", scope: "workspace" },
"terminal.dictation-toggle": { id: "terminal.dictation-toggle", scope: "workspace" },
"workspace.tab.close.current": { id: "workspace.tab.close-current", scope: "workspace" },
"sidebar.toggle.right": { id: "sidebar.toggle.right", scope: "sidebar" },
"workspace.pane.split.right": { id: "workspace.pane.split.right", scope: "workspace" },
@@ -105,8 +105,9 @@ const PANE_SPLIT_ACTIONS_RESERVED_WIDTH =
PANE_SPLIT_ACTIONS_HORIZONTAL_PADDING * 2 +
PANE_SPLIT_ACTIONS_OUTER_MARGIN;
const PANE_MAXIMIZE_ACTION_RESERVED_WIDTH = smallIconButtonChromeFrameSize(false) + 1;
// The terminal slot holds one icon button (upload).
const TERMINAL_SLOT_RESERVED_WIDTH = smallIconButtonChromeFrameSize(false) + 1;
// The terminal slot holds two icon buttons (upload, mic). While the terminal dictates the
// mic grows into a pill; the row re-measures then instead of reserving that width always.
const TERMINAL_SLOT_RESERVED_WIDTH = 2 * (smallIconButtonChromeFrameSize(false) + 1);
// Chip geometry. `layoutMetrics` measures tabs from these same numbers, so a chip that changes
// shape without changing them mis-measures and drops the row into the overflow-scroll fallback at
// the wrong width. Keep them together.
@@ -0,0 +1,18 @@
import { describe, expect, it } from "vitest";
import { formatTerminalTranscript } from "./format-terminal-transcript";
describe("formatTerminalTranscript", () => {
it.each([
["first\nsecond", "first second "],
["first\r\nsecond", "first second "],
["first\rsecond", "first second "],
["\n hello \r\n", "hello "],
["héllo 日本語", "héllo 日本語 "],
])("formats %j as %j", (input, expected) => {
expect(formatTerminalTranscript(input)).toBe(expected);
});
it.each(["", " ", "\r\n"])("returns null for %j", (input) => {
expect(formatTerminalTranscript(input)).toBeNull();
});
});
@@ -0,0 +1,10 @@
/**
* Turns a dictation transcript into terminal input: embedded line breaks become
* spaces (a newline would submit the line in most TUIs), the ends are trimmed, and
* one trailing space separates it from whatever is typed next. Never adds Enter.
* Returns null when nothing is left to paste.
*/
export function formatTerminalTranscript(text: string): string | null {
const singleLine = text.replace(/\r\n|\r|\n/g, " ").trim();
return singleLine.length > 0 ? `${singleLine} ` : null;
}
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import type { DaemonServerInfo } from "@/stores/session-store";
import { resolveTerminalDictationAvailability } from "./terminal-dictation-availability";
function serverInfo(capabilities?: DaemonServerInfo["capabilities"]): DaemonServerInfo {
return {
serverId: "server-a",
hostname: null,
version: "0.13.2",
...(capabilities ? { capabilities } : {}),
};
}
function voice(dictation: { enabled: boolean; reason: string }): DaemonServerInfo["capabilities"] {
return { voice: { dictation, voice: { enabled: false, reason: "" } } };
}
describe("resolveTerminalDictationAvailability", () => {
it("is disconnected while the host is not connected, whatever it reports", () => {
expect(
resolveTerminalDictationAvailability({
serverInfo: serverInfo(voice({ enabled: true, reason: "" })),
isConnected: false,
}),
).toEqual({ kind: "disconnected" });
});
it("asks to update a host that reports no capabilities", () => {
expect(
resolveTerminalDictationAvailability({ serverInfo: serverInfo(), isConnected: true }),
).toEqual({ kind: "unavailable", reason: "update-host" });
expect(resolveTerminalDictationAvailability({ serverInfo: null, isConnected: true })).toEqual({
kind: "unavailable",
reason: "update-host",
});
});
it("asks to update a host whose capabilities have no voice field", () => {
expect(
resolveTerminalDictationAvailability({ serverInfo: serverInfo({}), isConnected: true }),
).toEqual({ kind: "unavailable", reason: "update-host" });
});
it("carries the host's trimmed reason when dictation is disabled", () => {
expect(
resolveTerminalDictationAvailability({
serverInfo: serverInfo(voice({ enabled: false, reason: " No speech model installed " })),
isConnected: true,
}),
).toEqual({ kind: "unavailable", reason: "disabled", message: "No speech model installed" });
});
it.each(["", " "])("leaves the message empty for a disabled reason of %j", (reason) => {
expect(
resolveTerminalDictationAvailability({
serverInfo: serverInfo(voice({ enabled: false, reason })),
isConnected: true,
}),
).toEqual({ kind: "unavailable", reason: "disabled", message: null });
});
it("is ready when the host enables dictation", () => {
expect(
resolveTerminalDictationAvailability({
serverInfo: serverInfo(voice({ enabled: true, reason: "" })),
isConnected: true,
}),
).toEqual({ kind: "ready" });
});
});
@@ -0,0 +1,35 @@
import type { DaemonServerInfo } from "@/stores/session-store";
import { getVoiceReadinessState } from "@/utils/server-info-capabilities";
export type TerminalDictationAvailability =
| { kind: "ready" }
| { kind: "disconnected" }
| { kind: "unavailable"; reason: "update-host" }
| { kind: "unavailable"; reason: "disabled"; message: string | null };
/**
* Whether a terminal can dictate on this host. Gates on the host's explicit
* dictation capability only: a host too old to report it is told to update, so a
* terminal never falls back to the composer's agent-directory heuristic.
*/
export function resolveTerminalDictationAvailability(input: {
serverInfo: DaemonServerInfo | null | undefined;
isConnected: boolean;
}): TerminalDictationAvailability {
if (!input.isConnected) {
return { kind: "disconnected" };
}
const dictation = getVoiceReadinessState({ serverInfo: input.serverInfo, mode: "dictation" });
if (!dictation) {
return { kind: "unavailable", reason: "update-host" };
}
if (!dictation.enabled) {
const message = dictation.reason.trim();
return {
kind: "unavailable",
reason: "disabled",
message: message.length > 0 ? message : null,
};
}
return { kind: "ready" };
}
@@ -0,0 +1,89 @@
/**
* @vitest-environment jsdom
*/
import { describe, expect, it } from "vitest";
import type { DictationStatus } from "@/hooks/use-dictation";
import {
isTerminalKeyboardOwner,
resolveTerminalDictationLifecycle,
} from "./terminal-dictation-lifecycle";
describe("resolveTerminalDictationLifecycle", () => {
const active: DictationStatus[] = ["recording", "paused", "uploading"];
it("keeps an idle terminal whatever happens to it", () => {
expect(
resolveTerminalDictationLifecycle({
isPresented: false,
isRecordingOwner: false,
status: "idle",
}),
).toBe("keep");
});
it.each(active)("keeps a presented recording owner while %s", (status) => {
expect(
resolveTerminalDictationLifecycle({ isPresented: true, isRecordingOwner: true, status }),
).toBe("keep");
});
it.each(active)("cancels when the terminal stops being presented while %s", (status) => {
expect(
resolveTerminalDictationLifecycle({ isPresented: false, isRecordingOwner: true, status }),
).toBe("cancel");
});
it.each(active)("cancels when another terminal takes the recording while %s", (status) => {
expect(
resolveTerminalDictationLifecycle({ isPresented: true, isRecordingOwner: false, status }),
).toBe("cancel");
});
it("discards a failed attempt once hidden or superseded", () => {
expect(
resolveTerminalDictationLifecycle({
isPresented: false,
isRecordingOwner: true,
status: "failed",
}),
).toBe("discard");
expect(
resolveTerminalDictationLifecycle({
isPresented: true,
isRecordingOwner: false,
status: "failed",
}),
).toBe("discard");
});
});
describe("isTerminalKeyboardOwner", () => {
function surfaceWithInput() {
const surface = document.createElement("div");
const input = document.createElement("textarea");
surface.append(input);
document.body.append(surface);
return { surface, input };
}
it("is true when focus is inside the surface", () => {
const { surface, input } = surfaceWithInput();
expect(isTerminalKeyboardOwner(surface, input)).toBe(true);
surface.remove();
});
it("is false when focus is in another surface", () => {
const first = surfaceWithInput();
const second = surfaceWithInput();
expect(isTerminalKeyboardOwner(first.surface, second.input)).toBe(false);
first.surface.remove();
second.surface.remove();
});
it("is false without a surface or a focused element", () => {
const { surface, input } = surfaceWithInput();
expect(isTerminalKeyboardOwner(surface, null)).toBe(false);
expect(isTerminalKeyboardOwner(null, input)).toBe(false);
surface.remove();
});
});
@@ -0,0 +1,39 @@
import type { DictationStatus } from "@/hooks/use-dictation";
export type TerminalDictationLifecycleDecision = "keep" | "cancel" | "discard";
/**
* What a terminal's dictation does when its pane stops being presented (another
* tab or terminal selected, workspace left, explorer closed) or another terminal
* takes the recording. Recording, paused and processing are cancelled, a failed
* attempt is discarded. Status `recording` also covers a start still acquiring
* the microphone, because the hook sets it before awaiting the source.
*/
export function resolveTerminalDictationLifecycle(input: {
isPresented: boolean;
isRecordingOwner: boolean;
status: DictationStatus;
}): TerminalDictationLifecycleDecision {
if (input.status === "idle") {
return "keep";
}
if (input.isPresented && input.isRecordingOwner) {
return "keep";
}
return input.status === "failed" ? "discard" : "cancel";
}
/**
* True when DOM keyboard focus is inside this pane's terminal surface: the same
* focus that gives a shortcut the `terminal` scope, so exactly one pane owns it.
*/
export function isTerminalKeyboardOwner(surface: unknown, activeElement: unknown): boolean {
if (typeof Element === "undefined") {
return false;
}
return (
surface instanceof Element &&
activeElement instanceof Element &&
surface.contains(activeElement)
);
}
@@ -0,0 +1,198 @@
import { useEffect, useRef, type MutableRefObject } from "react";
import { useTranslation } from "react-i18next";
import { useToast } from "@/contexts/toast-context";
import { useDictation } from "@/hooks/use-dictation";
import { useKeyboardActionHandler } from "@/hooks/use-keyboard-action-handler";
import { useStableEvent } from "@/hooks/use-stable-event";
import { useHostRuntimeClient, useHostRuntimeIsConnected } from "@/runtime/host-runtime";
import { useSessionStore } from "@/stores/session-store";
import {
claimTerminalRecording,
clearTerminalDictationState,
getTerminalRecordingOwnerId,
publishTerminalDictationState,
releaseTerminalRecording,
useTerminalRecordingOwnerId,
type TerminalDictationActions,
} from "@/terminal/input-sink";
import { formatTerminalTranscript } from "./format-terminal-transcript";
import {
resolveTerminalDictationAvailability,
type TerminalDictationAvailability,
} from "./terminal-dictation-availability";
import { resolveTerminalDictationLifecycle } from "./terminal-dictation-lifecycle";
export type TerminalDictationOwnerActions = Omit<TerminalDictationActions, "ownerId">;
interface TerminalDictationOwnerProps {
ownerId: string;
serverId: string;
terminalId: string;
/** The pane's terminal is the one on screen: its tab is selected and its workspace focused. */
isPresented: boolean;
pasteText: (text: string) => void;
focus: () => void;
/** Evaluated when the shortcut fires: true while DOM focus is inside this pane's terminal. */
isKeyboardOwner: () => boolean;
/** Where the owner hands its actions to the pane's input sink. */
actionsRef: MutableRefObject<TerminalDictationOwnerActions | null>;
}
/**
* Owns one terminal pane's dictation. Mounted inside the pane and keyed by its
* terminal, so a transcript can only reach the pane that recorded it; the tab-bar
* control and the key bar are views of the state it publishes. Renders nothing,
* and lives apart from the pane so volume ticks re-render only this component.
*
* The recording ends, microphone and pending result included, when the pane
* unmounts or changes terminal (remount through the key), when the terminal stops
* being presented, and when another terminal starts recording.
*/
export function TerminalDictationOwner({
ownerId,
serverId,
terminalId,
isPresented,
pasteText,
focus,
isKeyboardOwner,
actionsRef,
}: TerminalDictationOwnerProps) {
const { t } = useTranslation();
const toast = useToast();
const client = useHostRuntimeClient(serverId);
const isConnected = useHostRuntimeIsConnected(serverId);
const serverInfo = useSessionStore((state) => state.sessions[serverId]?.serverInfo ?? null);
const recordingOwnerId = useTerminalRecordingOwnerId();
const isPresentedRef = useRef(isPresented);
isPresentedRef.current = isPresented;
const handleTranscript = useStableEvent((text: string) => {
// A result resolving in the same tick as a cancel: drop it unless this pane is
// still on screen and still the recording terminal.
if (!isPresentedRef.current || getTerminalRecordingOwnerId() !== ownerId) {
return;
}
const formatted = formatTerminalTranscript(text);
if (formatted !== null) {
pasteText(formatted);
}
focus();
});
const handleError = useStableEvent((error: Error) => {
toast.error(error.message);
});
const dictation = useDictation({
client,
onTranscript: handleTranscript,
onError: handleError,
enableDuration: true,
});
const statusRef = useRef(dictation.status);
statusRef.current = dictation.status;
const unavailableMessage = (
availability: Exclude<TerminalDictationAvailability, { kind: "ready" }>,
) => {
if (availability.kind === "disconnected") {
return t("workspace.terminal.hostDisconnected");
}
if (availability.reason === "update-host") {
return t("workspace.terminal.dictation.updateHost");
}
return availability.message ?? t("workspace.terminal.dictation.unavailable");
};
const toggle = useStableEvent(() => {
if (!isPresentedRef.current) {
return;
}
const status = statusRef.current;
if (status === "recording" || status === "paused") {
void dictation.confirmDictation();
return;
}
if (status !== "idle") {
return;
}
const availability = resolveTerminalDictationAvailability({ serverInfo, isConnected });
if (availability.kind !== "ready") {
toast.error(unavailableMessage(availability));
return;
}
claimTerminalRecording(ownerId);
void dictation.startDictation();
});
const cancel = useStableEvent(() => {
void dictation.cancelDictation();
focus();
});
const confirm = useStableEvent(() => {
void dictation.confirmDictation();
});
const retry = useStableEvent(() => {
void dictation.retryFailedDictation();
});
const discard = useStableEvent(() => {
dictation.discardFailedDictation();
focus();
});
useEffect(() => {
const actions: TerminalDictationOwnerActions = { toggle, cancel, confirm, retry, discard };
actionsRef.current = actions;
return () => {
if (actionsRef.current === actions) {
actionsRef.current = null;
}
};
}, [actionsRef, cancel, confirm, discard, retry, toggle]);
// Only a change of presentation or recording owner ends the recording, so the
// status is read at that moment instead of re-running on every status change.
const applyLifecycle = useStableEvent(
(input: { isPresented: boolean; isRecordingOwner: boolean }) => {
const decision = resolveTerminalDictationLifecycle({ ...input, status: statusRef.current });
if (decision === "cancel") {
void dictation.cancelDictation();
} else if (decision === "discard") {
dictation.discardFailedDictation();
}
},
);
useEffect(() => {
applyLifecycle({ isPresented, isRecordingOwner: recordingOwnerId === ownerId });
}, [applyLifecycle, isPresented, ownerId, recordingOwnerId]);
useEffect(() => {
return () => releaseTerminalRecording(ownerId);
}, [ownerId]);
const { status, isRecording, isProcessing, error, duration, volume } = dictation;
useEffect(() => {
publishTerminalDictationState(ownerId, {
status,
isRecording,
isProcessing,
error,
duration,
volume,
});
}, [duration, error, isProcessing, isRecording, ownerId, status, volume]);
useEffect(() => () => clearTerminalDictationState(ownerId), [ownerId]);
useKeyboardActionHandler({
handlerId: `terminal-dictation:${serverId}:${terminalId}:${ownerId}`,
actions: ["terminal.dictation-toggle"],
enabled: isPresented,
priority: 0,
isActive: isKeyboardOwner,
handle: () => {
toggle();
return true;
},
});
return null;
}
@@ -4,15 +4,45 @@
import { act, renderHook } from "@testing-library/react";
import { afterEach, describe, expect, it } from "vitest";
import {
claimTerminalRecording,
clearTerminalDictationState,
getTerminalInputSink,
getTerminalRecordingOwnerId,
publishTerminalDictationState,
registerTerminalInputSink,
releaseTerminalRecording,
useRegisterTerminalInputSink,
useTerminalDictationState,
useTerminalInputSink,
type TerminalDictationState,
type TerminalInputSink,
} from "./index";
function fakeSink(): TerminalInputSink {
return { pasteText: () => {}, deliverFiles: () => {}, focus: () => {} };
return {
pasteText: () => {},
deliverFiles: () => {},
focus: () => {},
dictation: {
ownerId: "owner",
toggle: () => {},
cancel: () => {},
confirm: () => {},
retry: () => {},
discard: () => {},
},
};
}
function dictationState(volume: number): TerminalDictationState {
return {
status: "recording",
isRecording: true,
isProcessing: false,
error: null,
duration: 0,
volume,
};
}
const cleanups: Array<() => void> = [];
@@ -95,3 +125,47 @@ describe("terminal input sinks", () => {
expect(getTerminalInputSink("server-a", "term-1")).toBeNull();
});
});
describe("terminal dictation state", () => {
afterEach(() => {
act(() => {
clearTerminalDictationState("owner-a");
clearTerminalDictationState("owner-b");
releaseTerminalRecording("owner-a");
releaseTerminalRecording("owner-b");
});
});
it("re-renders an owner's controls only for that owner's state", () => {
let renders = 0;
const { result } = renderHook(() => {
renders += 1;
return useTerminalDictationState("owner-a");
});
expect(result.current).toBeNull();
const rendersBefore = renders;
act(() => publishTerminalDictationState("owner-b", dictationState(0.5)));
expect(renders).toBe(rendersBefore);
const published = dictationState(0.2);
act(() => publishTerminalDictationState("owner-a", published));
expect(result.current).toBe(published);
act(() => clearTerminalDictationState("owner-a"));
expect(result.current).toBeNull();
});
it("keeps one recording owner and releases it only for that owner", () => {
act(() => claimTerminalRecording("owner-a"));
expect(getTerminalRecordingOwnerId()).toBe("owner-a");
act(() => claimTerminalRecording("owner-b"));
expect(getTerminalRecordingOwnerId()).toBe("owner-b");
act(() => releaseTerminalRecording("owner-a"));
expect(getTerminalRecordingOwnerId()).toBe("owner-b");
act(() => releaseTerminalRecording("owner-b"));
expect(getTerminalRecordingOwnerId()).toBeNull();
});
});
+83 -5
View File
@@ -1,12 +1,15 @@
import { useEffect } from "react";
import { create } from "zustand";
import type { SelectedFile } from "@/attachments/selected-file";
import type { DictationStatus } from "@/hooks/use-dictation";
/**
* What a mounted terminal pane accepts from controls rendered outside it: the
* pane's tab-bar slot and its key bar. Those controls are siblings of the pane
* under three different hosts (split pane, fallback row, explorer sidebar), so
* they find the pane here by terminal identity instead of through props.
* What a mounted terminal pane accepts from its controls. The tab-bar slot is a
* sibling of the pane under three different hosts (split pane, fallback row,
* explorer sidebar), so it finds the pane here by terminal identity instead of
* through props. The key bar renders inside the pane and receives the pane's own
* sink as a prop: when two panes show one terminal the registry holds only the
* newer one, and each key bar must still act on its own pane.
*/
export interface TerminalInputSink {
/** Pastes text the way the terminal's TUI expects it (bracketed when enabled). */
@@ -15,13 +18,44 @@ export interface TerminalInputSink {
deliverFiles(files: SelectedFile[]): void;
/** Returns keyboard focus to the terminal. */
focus(): void;
/** Forwards to the pane's dictation owner; does nothing while no owner is mounted. */
dictation: TerminalDictationActions;
}
export interface TerminalDictationActions {
/** Identifies the pane's owner; its published state is under this id. */
ownerId: string;
/** Starts dictation, or confirms it while recording. */
toggle(): void;
cancel(): void;
confirm(): void;
retry(): void;
discard(): void;
}
/** What a dictation owner publishes for its controls. */
export interface TerminalDictationState {
status: DictationStatus;
isRecording: boolean;
isProcessing: boolean;
error: string | null;
duration: number;
volume: number;
}
interface TerminalInputSinkState {
sinks: Record<string, TerminalInputSink>;
/** Keyed by owner id, apart from `sinks` so volume ticks never change a sink. */
dictationStates: Record<string, TerminalDictationState>;
/** The owner whose terminal is recording; one terminal records at a time. */
recordingOwnerId: string | null;
}
const useTerminalInputSinkStore = create<TerminalInputSinkState>(() => ({ sinks: {} }));
const useTerminalInputSinkStore = create<TerminalInputSinkState>(() => ({
sinks: {},
dictationStates: {},
recordingOwnerId: null,
}));
export function terminalInputSinkKey(serverId: string, terminalId: string): string {
return `${serverId}:${terminalId}`;
@@ -79,3 +113,47 @@ export function useTerminalInputSink(
const key = terminalInputSinkKey(serverId, terminalId);
return useTerminalInputSinkStore((state) => state.sinks[key] ?? null);
}
export function publishTerminalDictationState(
ownerId: string,
dictationState: TerminalDictationState,
): void {
useTerminalInputSinkStore.setState((state) => ({
dictationStates: { ...state.dictationStates, [ownerId]: dictationState },
}));
}
export function clearTerminalDictationState(ownerId: string): void {
useTerminalInputSinkStore.setState((state) => {
if (!(ownerId in state.dictationStates)) {
return state;
}
const { [ownerId]: _removed, ...dictationStates } = state.dictationStates;
return { dictationStates };
});
}
/** The owner's published state, or null while no owner with this id is mounted. */
export function useTerminalDictationState(ownerId: string): TerminalDictationState | null {
return useTerminalInputSinkStore((state) => state.dictationStates[ownerId] ?? null);
}
/** Makes `ownerId` the recording terminal; the previous owner cancels itself. */
export function claimTerminalRecording(ownerId: string): void {
useTerminalInputSinkStore.setState({ recordingOwnerId: ownerId });
}
/** Clears the recording owner only while it is still `ownerId`. */
export function releaseTerminalRecording(ownerId: string): void {
useTerminalInputSinkStore.setState((state) =>
state.recordingOwnerId === ownerId ? { recordingOwnerId: null } : state,
);
}
export function getTerminalRecordingOwnerId(): string | null {
return useTerminalInputSinkStore.getState().recordingOwnerId;
}
export function useTerminalRecordingOwnerId(): string | null {
return useTerminalInputSinkStore((state) => state.recordingOwnerId);
}
@@ -82,12 +82,13 @@ describe("terminal virtual keyboard policy", () => {
"terminal-key-esc",
"terminal-key-tab",
"terminal-key-ctrl",
"terminal-key-up",
"terminal-key-shift",
"terminal-key-up",
"terminal-key-alt",
"terminal-keyboard-toggle",
"terminal-paste",
"terminal-upload",
"terminal-dictation",
"terminal-key-left",
"terminal-key-down",
"terminal-key-right",
@@ -104,14 +105,14 @@ describe("terminal virtual keyboard policy", () => {
down: positions.get("terminal-key-down"),
right: positions.get("terminal-key-right"),
}).toEqual({
up: { row: 0, col: 3 },
left: { row: 1, col: 2 },
down: { row: 1, col: 3 },
right: { row: 1, col: 4 },
up: { row: 0, col: 4 },
left: { row: 1, col: 3 },
down: { row: 1, col: 4 },
right: { row: 1, col: 5 },
});
});
it("keeps Paste and Upload next to each other in every arrangement", () => {
it("keeps Paste, Upload and Dictation next to each other in every arrangement", () => {
for (const rows of [
TERMINAL_VIRTUAL_KEYBOARD_ROWS,
TERMINAL_VIRTUAL_KEYBOARD_WIDE_ROWS,
@@ -120,6 +121,7 @@ describe("terminal virtual keyboard policy", () => {
const rowIds = rows.map((row) => row.map(getTerminalVirtualKeyboardControlId));
const ids = rowIds.find((row) => row.includes("terminal-paste")) ?? [];
expect(ids.indexOf("terminal-upload")).toBe(ids.indexOf("terminal-paste") + 1);
expect(ids.indexOf("terminal-dictation")).toBe(ids.indexOf("terminal-paste") + 2);
}
});
@@ -21,24 +21,28 @@ export type TerminalVirtualKeyboardControl =
| {
type: "upload";
}
| {
type: "dictation";
}
| {
type: "keyboardToggle";
};
// Paste and upload stay next to each other in every arrangement.
// Paste, upload and dictation stay next to each other in every arrangement.
export const TERMINAL_VIRTUAL_KEYBOARD_ROWS = [
[
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.esc },
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.tab },
{ type: "modifier", modifier: "ctrl" },
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up },
{ type: "modifier", modifier: "shift" },
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up },
{ type: "modifier", modifier: "alt" },
{ type: "keyboardToggle" },
],
[
{ type: "paste" },
{ type: "upload" },
{ type: "dictation" },
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.left },
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.down },
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.right },
@@ -59,6 +63,7 @@ export const TERMINAL_VIRTUAL_KEYBOARD_WIDE_ROWS = [
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.right },
{ type: "paste" },
{ type: "upload" },
{ type: "dictation" },
{ type: "keyboardToggle" },
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.enter },
],
@@ -75,7 +80,7 @@ export const TERMINAL_VIRTUAL_KEYBOARD_THREE_COLUMN_ROWS = [
{ type: "modifier", modifier: "shift" },
{ type: "keyboardToggle" },
],
[{ type: "paste" }, { type: "upload" }],
[{ type: "paste" }, { type: "upload" }, { type: "dictation" }],
[
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up },
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.enter },
@@ -121,6 +126,8 @@ export function getTerminalVirtualKeyboardControlId(
return "terminal-paste";
case "upload":
return "terminal-upload";
case "dictation":
return "terminal-dictation";
case "keyboardToggle":
return "terminal-keyboard-toggle";
}
@@ -0,0 +1,160 @@
import { Check, Mic, RefreshCcw, X } from "lucide-react-native";
import { Text, View } from "react-native";
import { useTranslation } from "react-i18next";
import { StyleSheet, withUnistyles } from "react-native-unistyles";
import { LoadingSpinner } from "@/components/ui/loading-spinner";
import { ToolbarButton } from "@/components/ui/pane-content-toolbar";
import { VolumeMeter } from "@/components/volume-meter";
import { useShortcutKeys } from "@/hooks/use-shortcut-keys";
import type { Theme } from "@/styles/theme";
import { formatDictationDuration } from "@/utils/format-dictation-duration";
import type { TerminalDictationState, TerminalInputSink } from "@/terminal/input-sink";
const ThemedMic = withUnistyles(Mic);
const ThemedX = withUnistyles(X);
const ThemedCheck = withUnistyles(Check);
const ThemedRefresh = withUnistyles(RefreshCcw);
const ThemedVolumeMeter = withUnistyles(VolumeMeter);
const ThemedSpinner = withUnistyles(LoadingSpinner);
const iconColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundExtraMuted });
const activeIconColorMapping = (theme: Theme) => ({ color: theme.colors.foreground });
const accentIconColorMapping = (theme: Theme) => ({ color: theme.colors.accent });
interface TerminalDictationControlProps {
sink: TerminalInputSink;
/** Null while the pane has no dictation owner mounted (host disconnected). */
state: TerminalDictationState | null;
}
/**
* The terminal's mic in its pane's tab bar. Idle it is one button; while the
* terminal records, processes or has failed, the same slot becomes a pill with
* cancel, meter and duration, and confirm (or retry and discard after a failure).
* A view of the pane's dictation owner: every press goes through the sink.
*/
export function TerminalDictationControl({ sink, state }: TerminalDictationControlProps) {
const { t } = useTranslation();
const shortcut = useShortcutKeys("terminal-dictation-toggle");
const status = state?.status ?? "idle";
if (!state || status === "idle") {
return (
<ToolbarButton
label={t("workspace.terminal.actions.startDictation")}
shortcut={shortcut}
testID="terminal-toolbar-dictation"
disabled={!state}
onPress={sink.dictation.toggle}
>
<ThemedMic size={14} uniProps={iconColorMapping} />
</ToolbarButton>
);
}
const isFailed = status === "failed";
const isProcessing = state.isProcessing;
return (
<View style={styles.pill} testID="terminal-dictation-pill">
{isFailed ? (
<ToolbarButton
label={t("workspace.terminal.dictation.discard")}
testID="terminal-dictation-discard"
round
onPress={sink.dictation.discard}
>
<ThemedX size={14} uniProps={activeIconColorMapping} />
</ToolbarButton>
) : (
<ToolbarButton
label={t("message.dictation.cancel")}
testID="terminal-dictation-cancel"
round
onPress={sink.dictation.cancel}
>
<ThemedX size={14} uniProps={activeIconColorMapping} />
</ToolbarButton>
)}
{isFailed ? (
<Text style={styles.errorText} numberOfLines={1} testID="terminal-dictation-error">
{state.error
? t("message.dictation.failed", { error: state.error })
: t("message.dictation.failedRetry")}
</Text>
) : (
<View style={styles.meter}>
<ThemedVolumeMeter
volume={state.volume}
orientation="horizontal"
variant="compact"
uniProps={accentIconColorMapping}
/>
<Text style={styles.duration}>
{status === "paused"
? t("message.dictation.paused")
: formatDictationDuration(state.duration)}
</Text>
</View>
)}
{isProcessing ? (
<View style={styles.spinner}>
<ThemedSpinner size="small" uniProps={activeIconColorMapping} />
</View>
) : null}
{!isProcessing && isFailed ? (
<ToolbarButton
label={t("message.dictation.retry")}
testID="terminal-dictation-retry"
round
onPress={sink.dictation.retry}
>
<ThemedRefresh size={14} uniProps={accentIconColorMapping} />
</ToolbarButton>
) : null}
{!isProcessing && !isFailed ? (
<ToolbarButton
label={t("workspace.terminal.actions.stopDictation")}
shortcut={shortcut}
testID="terminal-dictation-confirm"
round
onPress={sink.dictation.confirm}
>
<ThemedCheck size={14} uniProps={accentIconColorMapping} />
</ToolbarButton>
) : null}
</View>
);
}
const styles = StyleSheet.create((theme) => ({
pill: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[1],
paddingHorizontal: theme.spacing[1],
borderRadius: theme.borderRadius.full,
borderWidth: 1,
borderColor: theme.colors.accent,
backgroundColor: theme.colors.surface1,
},
meter: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[1],
},
duration: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
fontVariant: ["tabular-nums"],
},
errorText: {
maxWidth: 160,
color: theme.colors.destructive,
fontSize: theme.fontSize.sm,
},
spinner: {
width: 20,
alignItems: "center",
justifyContent: "center",
},
}));
@@ -0,0 +1,66 @@
import { useMemo, type ReactNode } from "react";
import { Mic } from "lucide-react-native";
import { useTranslation } from "react-i18next";
import { withUnistyles } from "react-native-unistyles";
import { DictationOverlay } from "@/components/dictation-controls";
import { TerminalKeyBarIconAction } from "@/components/terminal-copy-paste-actions";
import type { Theme } from "@/styles/theme";
import { useTerminalDictationState, type TerminalInputSink } from "@/terminal/input-sink";
const ThemedMic = withUnistyles(Mic);
const keyBarIconColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundMuted });
interface TerminalKeyBarDictationProps {
/** The pane's own sink: a key bar acts on its pane, never on a newer pane of the same terminal. */
sink: TerminalInputSink;
}
/** The key-bar mic. Disabled while the pane has no dictation owner (host disconnected). */
export function TerminalKeyBarDictationButton({ sink }: TerminalKeyBarDictationProps) {
const { t } = useTranslation();
const state = useTerminalDictationState(sink.dictation.ownerId);
const icon = useMemo(() => <ThemedMic size={16} uniProps={keyBarIconColorMapping} />, []);
return (
<TerminalKeyBarIconAction
accessibilityLabel={t("workspace.terminal.actions.startDictation")}
testID="terminal-dictation"
icon={icon}
disabled={!state}
onPress={sink.dictation.toggle}
/>
);
}
/**
* Shows the key bar's rows while the pane's terminal is not dictating, and the
* dictation overlay in their place through recording, paused, processing and
* failed. The rows come back once the attempt is delivered, cancelled or discarded.
* Lives apart from the pane so volume ticks re-render only this component.
*/
export function TerminalKeyBarDictationGate({
sink,
children,
}: TerminalKeyBarDictationProps & { children: ReactNode }) {
const state = useTerminalDictationState(sink.dictation.ownerId);
if (!state || state.status === "idle") {
return children;
}
// Also recording while the microphone is still being acquired, so cancel is reachable then.
const isRecording =
state.isRecording || state.status === "recording" || state.status === "paused";
return (
<DictationOverlay
testID="terminal-dictation-overlay"
volume={state.volume}
duration={state.duration}
isRecording={isRecording}
isProcessing={state.isProcessing}
status={state.status}
errorText={state.status === "failed" ? (state.error ?? undefined) : undefined}
onCancel={sink.dictation.cancel}
onAccept={sink.dictation.confirm}
onRetry={sink.dictation.retry}
onDiscard={sink.dictation.discard}
/>
);
}
@@ -1,4 +1,5 @@
import { useTerminalInputSink } from "@/terminal/input-sink";
import { useTerminalDictationState, useTerminalInputSink } from "@/terminal/input-sink";
import { TerminalDictationControl } from "./terminal-dictation-control";
import { TerminalUploadButton } from "./terminal-upload-button";
interface TerminalToolbarSlotProps {
@@ -8,13 +9,28 @@ interface TerminalToolbarSlotProps {
/**
* The terminal controls in a pane's tab bar, shown while the pane's active tab is
* a terminal. A view over the mounted pane's input sink: it owns no terminal
* state and renders nothing until the pane has registered.
* a terminal: upload, then the mic. A view over the mounted pane's input sink and
* its dictation state: it owns no terminal state and renders nothing until the
* pane has registered.
*/
export function TerminalToolbarSlot({ serverId, terminalId }: TerminalToolbarSlotProps) {
const sink = useTerminalInputSink(serverId, terminalId);
if (!sink) {
return null;
}
return <TerminalUploadButton sink={sink} placement="toolbar" />;
return <TerminalToolbarSlotControls sink={sink} />;
}
function TerminalToolbarSlotControls({
sink,
}: {
sink: NonNullable<ReturnType<typeof useTerminalInputSink>>;
}) {
const state = useTerminalDictationState(sink.dictation.ownerId);
return (
<>
<TerminalUploadButton sink={sink} placement="toolbar" />
<TerminalDictationControl sink={sink} state={state} />
</>
);
}
@@ -0,0 +1,6 @@
/** Formats a dictation's elapsed seconds as `mm:ss`. */
export function formatDictationDuration(seconds: number): string {
const mins = Math.floor(seconds / 60);
const secs = seconds % 60;
return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
}