mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-06 16:34:36 +00:00
M2: terminal dictation: mic in tab-bar slot and key bar, recording pill and overlay, per-terminal recording lifetime incl. pending mic acquisition, Cmd+D on macOS by keyboard focus, i18n
This commit is contained in:
@@ -1,6 +1,10 @@
|
||||
import { expect, test, type Page } from "../support/fixtures";
|
||||
import { gotoAppShell } from "../support/helpers/app";
|
||||
import { daemonWsRoutePattern } from "../support/helpers/daemon-port";
|
||||
import {
|
||||
installDictationWsHarness,
|
||||
installSyntheticMicrophone,
|
||||
sendSessionMessage,
|
||||
} from "../support/helpers/dictation-ws";
|
||||
import {
|
||||
openNewWorkspaceComposer,
|
||||
selectWorkspaceIsolation,
|
||||
@@ -17,141 +21,39 @@ import {
|
||||
} from "../support/helpers/composer";
|
||||
import { waitForSidebarHydration } from "../support/helpers/workspace-ui";
|
||||
|
||||
type WebSocketMessage = string | Buffer;
|
||||
|
||||
const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace";
|
||||
const CREATE_FAILURE = "Synthetic workspace creation failure";
|
||||
|
||||
function parseEnvelope(message: WebSocketMessage): {
|
||||
type?: unknown;
|
||||
message?: Record<string, unknown>;
|
||||
} | null {
|
||||
const raw = typeof message === "string" ? message : message.toString("utf8");
|
||||
try {
|
||||
return JSON.parse(raw) as { type?: unknown; message?: Record<string, unknown> };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sessionMessage(message: WebSocketMessage): Record<string, unknown> | null {
|
||||
const envelope = parseEnvelope(message);
|
||||
return envelope?.type === "session" && envelope.message ? envelope.message : null;
|
||||
}
|
||||
|
||||
function sendSessionMessage(
|
||||
ws: { send(message: string): void },
|
||||
message: Record<string, unknown>,
|
||||
): void {
|
||||
ws.send(JSON.stringify({ type: "session", message }));
|
||||
}
|
||||
|
||||
function enableDictationCapability(message: WebSocketMessage): WebSocketMessage {
|
||||
const envelope = parseEnvelope(message);
|
||||
const payload = envelope?.message?.payload;
|
||||
if (
|
||||
envelope?.message?.type !== "status" ||
|
||||
!payload ||
|
||||
typeof payload !== "object" ||
|
||||
(payload as { status?: unknown }).status !== "server_info"
|
||||
) {
|
||||
return message;
|
||||
}
|
||||
|
||||
(payload as Record<string, unknown>).capabilities = {
|
||||
voice: {
|
||||
dictation: { enabled: true, reason: "" },
|
||||
voice: { enabled: false, reason: "Realtime voice is disabled in this test." },
|
||||
},
|
||||
};
|
||||
return JSON.stringify(envelope);
|
||||
}
|
||||
|
||||
async function installSyntheticMicrophone(page: Page): Promise<void> {
|
||||
await page.addInitScript(() => {
|
||||
const mediaDevices = navigator.mediaDevices;
|
||||
Object.defineProperty(mediaDevices, "getUserMedia", {
|
||||
configurable: true,
|
||||
value: async () => {
|
||||
const context = new AudioContext();
|
||||
const oscillator = context.createOscillator();
|
||||
const destination = context.createMediaStreamDestination();
|
||||
oscillator.connect(destination);
|
||||
oscillator.start();
|
||||
return destination.stream;
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function installDictationFailureHarness(page: Page) {
|
||||
let resolveAudioChunk!: () => void;
|
||||
let resolveCreateRequest!: () => void;
|
||||
const audioChunk = new Promise<void>((resolve) => {
|
||||
resolveAudioChunk = resolve;
|
||||
});
|
||||
const createRequest = new Promise<void>((resolve) => {
|
||||
resolveCreateRequest = resolve;
|
||||
});
|
||||
let failCreate: (() => void) | null = null;
|
||||
|
||||
await page.routeWebSocket(daemonWsRoutePattern(), (ws) => {
|
||||
const server = ws.connectToServer();
|
||||
|
||||
ws.onMessage((message) => {
|
||||
const request = sessionMessage(message);
|
||||
const type = request?.type;
|
||||
const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null;
|
||||
|
||||
if (type === "dictation_stream_start" && dictationId) {
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_ack",
|
||||
payload: { dictationId, ackSeq: -1 },
|
||||
});
|
||||
return;
|
||||
const dictation = await installDictationWsHarness(page, {
|
||||
transcript: TRANSCRIPT,
|
||||
interceptClientMessage: (request, ws) => {
|
||||
if (request.type !== "workspace.create.request" || typeof request.requestId !== "string") {
|
||||
return false;
|
||||
}
|
||||
if (type === "dictation_stream_chunk" && dictationId) {
|
||||
const seq = typeof request?.seq === "number" ? request.seq : 0;
|
||||
const requestId = request.requestId;
|
||||
failCreate = () => {
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_ack",
|
||||
payload: { dictationId, ackSeq: seq },
|
||||
type: "workspace.create.response",
|
||||
payload: {
|
||||
requestId,
|
||||
workspace: null,
|
||||
setupTerminalId: null,
|
||||
error: CREATE_FAILURE,
|
||||
},
|
||||
});
|
||||
resolveAudioChunk();
|
||||
return;
|
||||
}
|
||||
if (type === "dictation_stream_finish" && dictationId) {
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_finish_accepted",
|
||||
payload: { dictationId, timeoutMs: 5_000 },
|
||||
});
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_final",
|
||||
payload: { dictationId, text: TRANSCRIPT },
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (request && type === "workspace.create.request" && typeof request.requestId === "string") {
|
||||
const requestId = request.requestId;
|
||||
failCreate = () => {
|
||||
sendSessionMessage(ws, {
|
||||
type: "workspace.create.response",
|
||||
payload: {
|
||||
requestId,
|
||||
workspace: null,
|
||||
setupTerminalId: null,
|
||||
error: CREATE_FAILURE,
|
||||
},
|
||||
});
|
||||
};
|
||||
resolveCreateRequest();
|
||||
return;
|
||||
}
|
||||
|
||||
server.send(message);
|
||||
});
|
||||
|
||||
server.onMessage((message) => ws.send(enableDictationCapability(message)));
|
||||
};
|
||||
resolveCreateRequest();
|
||||
return true;
|
||||
},
|
||||
});
|
||||
const audioChunk = dictation.nextChunk();
|
||||
|
||||
return {
|
||||
waitForAudio: () => audioChunk,
|
||||
|
||||
@@ -0,0 +1,598 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { Locator, Page } from "@playwright/test";
|
||||
import { expect, test } from "../support/fixtures";
|
||||
import { runWorkspaceActionFromCommandCenter } from "../support/helpers/command-center-workspace-actions";
|
||||
import {
|
||||
holdMicrophone,
|
||||
installDictationWsHarness,
|
||||
installSyntheticMicrophone,
|
||||
microphoneStats,
|
||||
releaseMicrophone,
|
||||
type DictationCapability,
|
||||
type DictationWsHarness,
|
||||
} from "../support/helpers/dictation-ws";
|
||||
import { TerminalE2EHarness, type TerminalInstance } from "../support/helpers/terminal-dsl";
|
||||
import { ensureExplorerSidebar } from "../support/helpers/workspace-tabs";
|
||||
|
||||
const MAC_USER_AGENT =
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
|
||||
const LINUX_USER_AGENT =
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
|
||||
|
||||
// Payloads are echoed as JSON with whitespace escaped, because the PTY capture
|
||||
// below drops whitespace when it joins wrapped lines.
|
||||
const ENCODE = String.raw`
|
||||
const encode = (text) =>
|
||||
JSON.stringify(text).replace(
|
||||
/\s/g,
|
||||
(space) => "\\u" + space.charCodeAt(0).toString(16).padStart(4, "0"),
|
||||
);
|
||||
`;
|
||||
|
||||
// A TUI with bracketed paste on: echoes each paste body as INPUT:<json>.
|
||||
const BRACKETED_RECEIVER_SCRIPT = String.raw`
|
||||
${ENCODE}
|
||||
let buffer = "";
|
||||
process.stdin.setRawMode(true);
|
||||
process.stdin.resume();
|
||||
process.stdout.write("\x1b[?2004hDICTATION_RECEIVER_READY\r\n");
|
||||
process.stdin.on("data", (chunk) => {
|
||||
buffer += chunk.toString();
|
||||
for (;;) {
|
||||
const paste = /\x1b\[200~([^]*?)\x1b\[201~/.exec(buffer);
|
||||
if (!paste) break;
|
||||
process.stdout.write("INPUT:" + encode(paste[1]) + "\r\n");
|
||||
buffer = buffer.slice(0, paste.index) + buffer.slice(paste.index + paste[0].length);
|
||||
}
|
||||
});
|
||||
`;
|
||||
|
||||
// A program without bracketed paste: echoes every stdin byte, in raw mode so
|
||||
// Ctrl+D arrives as \x04, as RAW_INPUT:<json> after a quiet period.
|
||||
const RAW_RECEIVER_SCRIPT = String.raw`
|
||||
${ENCODE}
|
||||
let pending = "";
|
||||
let timer = null;
|
||||
process.stdin.setRawMode(true);
|
||||
process.stdin.resume();
|
||||
process.stdout.write("RAW_RECEIVER_READY\r\n");
|
||||
process.stdin.on("data", (chunk) => {
|
||||
pending += chunk.toString();
|
||||
clearTimeout(timer);
|
||||
timer = setTimeout(() => {
|
||||
process.stdout.write("RAW_INPUT:" + encode(pending) + "\r\n");
|
||||
pending = "";
|
||||
}, 200);
|
||||
});
|
||||
`;
|
||||
|
||||
type ReceiverKind = "bracketed" | "raw";
|
||||
|
||||
function visibleTestId(scope: Page | Locator, testId: string): Locator {
|
||||
return scope.getByTestId(testId).filter({ visible: true }).first();
|
||||
}
|
||||
|
||||
// Click the surface, not its helper textarea: the textarea sits at the left edge,
|
||||
// under a split or Explorer resize handle.
|
||||
function terminalSurface(scope: Page | Locator): Locator {
|
||||
return scope.getByTestId("terminal-surface").filter({ visible: true });
|
||||
}
|
||||
|
||||
function terminalInput(scope: Page | Locator): Locator {
|
||||
return scope
|
||||
.getByTestId("terminal-surface")
|
||||
.filter({ visible: true })
|
||||
.locator(".xterm-helper-textarea");
|
||||
}
|
||||
|
||||
function splitChild(page: Page, terminal: TerminalInstance): Locator {
|
||||
return page
|
||||
.getByTestId("split-group-child")
|
||||
.filter({ has: page.getByTestId(`workspace-tab-terminal_${terminal.id}`) });
|
||||
}
|
||||
|
||||
function explorerSidebar(page: Page): Locator {
|
||||
return page.getByTestId("workspace-explorer-sidebar").filter({ visible: true }).first();
|
||||
}
|
||||
|
||||
async function dragTabIntoExplorer(page: Page, terminal: TerminalInstance): Promise<void> {
|
||||
const source = page
|
||||
.getByTestId("workspace-pane-main")
|
||||
.getByTestId(`workspace-tab-terminal_${terminal.id}`);
|
||||
const destination = explorerSidebar(page).getByTestId("workspace-tab-files");
|
||||
const sourceBox = await source.boundingBox();
|
||||
const destinationBox = await destination.boundingBox();
|
||||
if (!sourceBox || !destinationBox) {
|
||||
throw new Error("Tab drag requires visible source and destination");
|
||||
}
|
||||
const start = { x: sourceBox.x + 12, y: sourceBox.y + sourceBox.height / 2 };
|
||||
await page.mouse.move(start.x, start.y);
|
||||
await page.mouse.down();
|
||||
await page.mouse.move(start.x + 12, start.y + 4);
|
||||
await page.mouse.move(destinationBox.x + 12, destinationBox.y + destinationBox.height / 2, {
|
||||
steps: 20,
|
||||
});
|
||||
await page.mouse.up();
|
||||
}
|
||||
|
||||
async function expectMicrophoneStat(
|
||||
page: Page,
|
||||
stat: "trackStops" | "getUserMediaCalls",
|
||||
count: number,
|
||||
): Promise<void> {
|
||||
await expect.poll(async () => (await microphoneStats(page))[stat]).toBe(count);
|
||||
}
|
||||
|
||||
test.describe("Terminal dictation", () => {
|
||||
test.describe.configure({ timeout: 180_000 });
|
||||
|
||||
let harness: TerminalE2EHarness;
|
||||
|
||||
test.beforeAll(async () => {
|
||||
harness = await TerminalE2EHarness.create({ tempPrefix: "terminal-dictation-" });
|
||||
await writeFile(
|
||||
path.join(harness.tempRepo.path, "bracketed-receiver.cjs"),
|
||||
BRACKETED_RECEIVER_SCRIPT,
|
||||
);
|
||||
await writeFile(path.join(harness.tempRepo.path, "raw-receiver.cjs"), RAW_RECEIVER_SCRIPT);
|
||||
});
|
||||
|
||||
test.afterAll(async () => {
|
||||
await harness?.cleanup();
|
||||
});
|
||||
|
||||
const terminals: TerminalInstance[] = [];
|
||||
|
||||
test.afterEach(async () => {
|
||||
for (const terminal of terminals.splice(0)) {
|
||||
await harness.killTerminal(terminal.id);
|
||||
}
|
||||
});
|
||||
|
||||
async function createReceiver(kind: ReceiverKind, name: string): Promise<TerminalInstance> {
|
||||
const terminal = await harness.createTerminal({
|
||||
name,
|
||||
command: process.execPath,
|
||||
args: [kind === "bracketed" ? "bracketed-receiver.cjs" : "raw-receiver.cjs"],
|
||||
});
|
||||
terminals.push(terminal);
|
||||
await expectTerminalOutput(
|
||||
terminal.id,
|
||||
kind === "bracketed" ? "DICTATION_RECEIVER_READY" : "RAW_RECEIVER_READY",
|
||||
);
|
||||
return terminal;
|
||||
}
|
||||
|
||||
async function setup(
|
||||
page: Page,
|
||||
options: { transcript?: string; capability?: DictationCapability } = {},
|
||||
): Promise<DictationWsHarness> {
|
||||
await installSyntheticMicrophone(page);
|
||||
return await installDictationWsHarness(page, {
|
||||
transcript: options.transcript ?? "first\nsecond",
|
||||
capability: options.capability,
|
||||
});
|
||||
}
|
||||
|
||||
// Reads the PTY itself, so output from one pane cannot satisfy another's check.
|
||||
async function terminalOutput(terminalId: string): Promise<string> {
|
||||
const capture = await harness.client.captureTerminal(terminalId);
|
||||
return capture.lines.join("").replace(/\s/g, "");
|
||||
}
|
||||
|
||||
async function expectTerminalOutput(terminalId: string, text: string): Promise<void> {
|
||||
await expect.poll(() => terminalOutput(terminalId), { timeout: 10_000 }).toContain(text);
|
||||
}
|
||||
|
||||
/** Everything the receiver got, in order and joined. */
|
||||
async function received(terminalId: string, kind: ReceiverKind): Promise<string> {
|
||||
const label = kind === "bracketed" ? "INPUT" : "RAW_INPUT";
|
||||
const output = await terminalOutput(terminalId);
|
||||
return [
|
||||
...output.matchAll(
|
||||
new RegExp(
|
||||
`${kind === "bracketed" ? "(?<!RAW_)" : ""}${label}:("(?:[^"\\\\]|\\\\.)*")`,
|
||||
"g",
|
||||
),
|
||||
),
|
||||
]
|
||||
.map(([, json]) => JSON.parse(json) as string)
|
||||
.join("");
|
||||
}
|
||||
|
||||
async function expectReceived(
|
||||
terminalId: string,
|
||||
kind: ReceiverKind,
|
||||
text: string,
|
||||
): Promise<void> {
|
||||
await expect.poll(() => received(terminalId, kind), { timeout: 15_000 }).toBe(text);
|
||||
}
|
||||
|
||||
/** Gives a late transcript time to land, then checks that none did. */
|
||||
async function expectNothingReceived(
|
||||
page: Page,
|
||||
...receivers: Array<[TerminalInstance, ReceiverKind]>
|
||||
): Promise<void> {
|
||||
await page.waitForTimeout(1_500);
|
||||
for (const [terminal, kind] of receivers) {
|
||||
expect(await received(terminal.id, kind)).toBe("");
|
||||
}
|
||||
}
|
||||
|
||||
async function startFromToolbar(
|
||||
page: Page,
|
||||
ws: DictationWsHarness,
|
||||
scope: Page | Locator = page,
|
||||
) {
|
||||
const chunk = ws.nextChunk();
|
||||
await visibleTestId(scope, "terminal-toolbar-dictation").click();
|
||||
await chunk;
|
||||
}
|
||||
|
||||
// The route helper needs the desktop tab row, so open there, then go compact.
|
||||
async function openCompact(page: Page, terminal: TerminalInstance) {
|
||||
await harness.openTerminal(page, { terminalId: terminal.id });
|
||||
await page.setViewportSize({ width: 390, height: 844 });
|
||||
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
|
||||
}
|
||||
|
||||
// The compact switcher labels terminals by their command, so pick by position.
|
||||
async function selectCompactTab(page: Page, index: number) {
|
||||
await page
|
||||
.getByRole("button", { name: /Switch tabs/ })
|
||||
.filter({ visible: true })
|
||||
.click();
|
||||
await expect(page.getByText("Workspace tabs", { exact: true })).toBeVisible();
|
||||
const options = page
|
||||
.getByText("bracketed-receiver.cjs", { exact: true })
|
||||
.filter({ visible: true });
|
||||
await expect(options).toHaveCount(2);
|
||||
await options.nth(index).click();
|
||||
await expect(page.getByRole("button", { name: "Bottom sheet backdrop" })).toHaveCount(0);
|
||||
}
|
||||
|
||||
async function selectTerminalTab(scope: Page | Locator, terminal: TerminalInstance) {
|
||||
await visibleTestId(scope, `workspace-tab-terminal_${terminal.id}`).click();
|
||||
}
|
||||
|
||||
test("confirm pastes the transcript on one line with a trailing space and no Enter", async ({
|
||||
page,
|
||||
}) => {
|
||||
const ws = await setup(page);
|
||||
const bracketed = await createReceiver("bracketed", "dictation-bracketed");
|
||||
const raw = await createReceiver("raw", "dictation-raw");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: bracketed.id });
|
||||
await startFromToolbar(page, ws);
|
||||
await visibleTestId(page, "terminal-dictation-confirm").click();
|
||||
await expectReceived(bracketed.id, "bracketed", "first second ");
|
||||
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
|
||||
|
||||
await selectTerminalTab(page, raw);
|
||||
await startFromToolbar(page, ws);
|
||||
await visibleTestId(page, "terminal-dictation-confirm").click();
|
||||
await expectReceived(raw.id, "raw", "first second ");
|
||||
expect(await received(bracketed.id, "bracketed")).toBe("first second ");
|
||||
});
|
||||
|
||||
test.describe("on macOS", () => {
|
||||
test.use({ userAgent: MAC_USER_AGENT });
|
||||
|
||||
test("Cmd+D starts and then confirms dictation in the focused terminal", async ({ page }) => {
|
||||
const ws = await setup(page, { transcript: "shortcut words" });
|
||||
const terminal = await createReceiver("bracketed", "dictation-shortcut");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: terminal.id });
|
||||
const chunk = ws.nextChunk();
|
||||
await terminalInput(page).press("Meta+d");
|
||||
await expect(visibleTestId(page, "terminal-dictation-confirm")).toBeVisible();
|
||||
await chunk;
|
||||
await terminalInput(page).press("Meta+d");
|
||||
|
||||
await expectReceived(terminal.id, "bracketed", "shortcut words ");
|
||||
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
|
||||
});
|
||||
|
||||
test("Cmd+D goes to the terminal with keyboard focus, main pane or Explorer", async ({
|
||||
page,
|
||||
}) => {
|
||||
const ws = await setup(page, { transcript: "main pane words" });
|
||||
const explorerTerminal = await createReceiver("bracketed", "dictation-explorer");
|
||||
const mainTerminal = await createReceiver("bracketed", "dictation-main");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: explorerTerminal.id });
|
||||
await ensureExplorerSidebar(page);
|
||||
await dragTabIntoExplorer(page, explorerTerminal);
|
||||
const explorer = explorerSidebar(page);
|
||||
await expect(
|
||||
explorer.getByTestId(`workspace-tab-terminal_${explorerTerminal.id}`),
|
||||
).toBeVisible();
|
||||
const main = page.getByTestId("workspace-pane-main");
|
||||
await selectTerminalTab(main, mainTerminal);
|
||||
await expect(explorer.getByTestId("terminal-surface")).toBeVisible();
|
||||
|
||||
let chunk = ws.nextChunk();
|
||||
await terminalSurface(main).click();
|
||||
await terminalInput(main).press("Meta+d");
|
||||
await expect(visibleTestId(main, "terminal-dictation-confirm")).toBeVisible();
|
||||
await expect(explorer.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
|
||||
await chunk;
|
||||
await terminalInput(main).press("Meta+d");
|
||||
await expectReceived(mainTerminal.id, "bracketed", "main pane words ");
|
||||
await expectNothingReceived(page, [explorerTerminal, "bracketed"]);
|
||||
|
||||
ws.setTranscript("explorer words");
|
||||
chunk = ws.nextChunk();
|
||||
await terminalSurface(explorer).click();
|
||||
await terminalInput(explorer).press("Meta+d");
|
||||
await expect(visibleTestId(explorer, "terminal-dictation-confirm")).toBeVisible();
|
||||
await expect(main.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
|
||||
await chunk;
|
||||
await terminalInput(explorer).press("Meta+d");
|
||||
await expectReceived(explorerTerminal.id, "bracketed", "explorer words ");
|
||||
expect(await received(mainTerminal.id, "bracketed")).toBe("main pane words ");
|
||||
});
|
||||
|
||||
test("Cmd+D records only in the focused split pane", async ({ page }) => {
|
||||
const ws = await setup(page, { transcript: "right pane words" });
|
||||
const left = await createReceiver("bracketed", "dictation-split-left");
|
||||
const right = await createReceiver("bracketed", "dictation-split-right");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: left.id });
|
||||
await runWorkspaceActionFromCommandCenter(page, "Split pane right");
|
||||
await visibleTestId(page, `workspace-tab-terminal_${right.id}`).click();
|
||||
await runWorkspaceActionFromCommandCenter(page, "Move tab right");
|
||||
const leftPane = splitChild(page, left);
|
||||
const rightPane = splitChild(page, right);
|
||||
await expect(terminalInput(leftPane)).toHaveCount(1);
|
||||
await expect(terminalInput(rightPane)).toHaveCount(1);
|
||||
|
||||
const chunk = ws.nextChunk();
|
||||
await terminalSurface(rightPane).click();
|
||||
await terminalInput(rightPane).press("Meta+d");
|
||||
await expect(visibleTestId(rightPane, "terminal-dictation-confirm")).toBeVisible();
|
||||
await expect(leftPane.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
|
||||
await chunk;
|
||||
await terminalInput(rightPane).press("Meta+d");
|
||||
|
||||
await expectReceived(right.id, "bracketed", "right pane words ");
|
||||
await expectNothingReceived(page, [left, "bracketed"]);
|
||||
});
|
||||
});
|
||||
|
||||
test("Ctrl+D off macOS reaches the PTY as EOF and starts nothing", async ({ page }) => {
|
||||
// Chromium keeps the host's navigator.platform unless the override sets it too.
|
||||
const session = await page.context().newCDPSession(page);
|
||||
await session.send("Emulation.setUserAgentOverride", {
|
||||
userAgent: LINUX_USER_AGENT,
|
||||
platform: "Linux x86_64",
|
||||
});
|
||||
const ws = await setup(page);
|
||||
const terminal = await createReceiver("raw", "dictation-ctrl-d");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: terminal.id });
|
||||
await terminalInput(page).press("Control+d");
|
||||
|
||||
await expectReceived(terminal.id, "raw", "\u0004");
|
||||
expect(ws.streamStarts()).toBe(0);
|
||||
expect((await microphoneStats(page)).getUserMediaCalls).toBe(0);
|
||||
await expect(page.getByTestId("terminal-dictation-confirm")).toHaveCount(0);
|
||||
});
|
||||
|
||||
test("switching the pane to another terminal while recording stops it and delivers nothing", async ({
|
||||
page,
|
||||
}) => {
|
||||
const ws = await setup(page);
|
||||
const first = await createReceiver("bracketed", "dictation-switch-a");
|
||||
const second = await createReceiver("bracketed", "dictation-switch-b");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: first.id });
|
||||
await startFromToolbar(page, ws);
|
||||
await selectTerminalTab(page, second);
|
||||
|
||||
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
|
||||
).toHaveCount(0);
|
||||
await expectMicrophoneStat(page, "trackStops", 1);
|
||||
const chunksAfterSwitch = ws.chunks();
|
||||
const startsAfterSwitch = ws.streamStarts();
|
||||
await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
|
||||
expect(ws.chunks()).toBe(chunksAfterSwitch);
|
||||
expect(ws.streamStarts()).toBe(startsAfterSwitch);
|
||||
|
||||
await selectTerminalTab(page, first);
|
||||
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
|
||||
).toHaveCount(0);
|
||||
});
|
||||
|
||||
test("switching the pane to another terminal while processing drops the late transcript", async ({
|
||||
page,
|
||||
}) => {
|
||||
const ws = await setup(page);
|
||||
const first = await createReceiver("bracketed", "dictation-processing-a");
|
||||
const second = await createReceiver("bracketed", "dictation-processing-b");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: first.id });
|
||||
await startFromToolbar(page, ws);
|
||||
ws.holdFinal();
|
||||
await visibleTestId(page, "terminal-dictation-confirm").click();
|
||||
await expect(visibleTestId(page, "terminal-dictation-confirm")).toHaveCount(0);
|
||||
await expect(visibleTestId(page, "terminal-dictation-cancel")).toBeVisible();
|
||||
await selectTerminalTab(page, second);
|
||||
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
|
||||
ws.releaseFinal();
|
||||
|
||||
await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
|
||||
});
|
||||
|
||||
test.describe("on compact web", () => {
|
||||
// Keyboard shortcuts are off on compact form factors (keyboardShortcutsAvailable), so
|
||||
// the key-bar mic is how compact web dictates.
|
||||
test("the key-bar mic shows the overlay and confirm delivers", async ({ page }) => {
|
||||
const ws = await setup(page, { transcript: "compact words" });
|
||||
const terminal = await createReceiver("bracketed", "dictation-compact-confirm");
|
||||
|
||||
await openCompact(page, terminal);
|
||||
const chunk = ws.nextChunk();
|
||||
await visibleTestId(page, "terminal-dictation").click();
|
||||
const overlay = visibleTestId(page, "terminal-dictation-overlay");
|
||||
await expect(overlay).toBeVisible();
|
||||
await expect(visibleTestId(page, "terminal-dictation")).toHaveCount(0);
|
||||
await expect(
|
||||
overlay.getByRole("button", { name: "Insert transcription and send", exact: true }),
|
||||
).toHaveCount(0);
|
||||
await chunk;
|
||||
await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
|
||||
|
||||
await expectReceived(terminal.id, "bracketed", "compact words ");
|
||||
await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
|
||||
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
|
||||
});
|
||||
|
||||
test("switching tabs while recording from the key bar stops it and delivers nothing", async ({
|
||||
page,
|
||||
}) => {
|
||||
const ws = await setup(page);
|
||||
const first = await createReceiver("bracketed", "dictation-compact-a");
|
||||
const second = await createReceiver("bracketed", "dictation-compact-b");
|
||||
|
||||
await openCompact(page, first);
|
||||
const chunk = ws.nextChunk();
|
||||
await visibleTestId(page, "terminal-dictation").click();
|
||||
await expect(visibleTestId(page, "terminal-dictation-overlay")).toBeVisible();
|
||||
await chunk;
|
||||
|
||||
await selectCompactTab(page, 1);
|
||||
await expectMicrophoneStat(page, "trackStops", 1);
|
||||
await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
|
||||
|
||||
await selectCompactTab(page, 0);
|
||||
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId("terminal-dictation-overlay").filter({ visible: true }),
|
||||
).toHaveCount(0);
|
||||
expect(await received(first.id, "bracketed")).toBe("");
|
||||
});
|
||||
|
||||
test("a failed dictation keeps the overlay with retry and discard", async ({ page }) => {
|
||||
const ws = await setup(page, { transcript: "retried words" });
|
||||
const terminal = await createReceiver("bracketed", "dictation-compact-failure");
|
||||
|
||||
await openCompact(page, terminal);
|
||||
const overlay = visibleTestId(page, "terminal-dictation-overlay");
|
||||
|
||||
ws.failNextFinish("Synthetic transcription failure");
|
||||
let chunk = ws.nextChunk();
|
||||
await visibleTestId(page, "terminal-dictation").click();
|
||||
await chunk;
|
||||
await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
|
||||
await expect(overlay).toContainText("Dictation failed: Synthetic transcription failure");
|
||||
await overlay.getByRole("button", { name: "Retry dictation", exact: true }).click();
|
||||
await expectReceived(terminal.id, "bracketed", "retried words ");
|
||||
await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
|
||||
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
|
||||
|
||||
ws.failNextFinish("Synthetic transcription failure");
|
||||
chunk = ws.nextChunk();
|
||||
await visibleTestId(page, "terminal-dictation").click();
|
||||
await chunk;
|
||||
await overlay.getByRole("button", { name: "Insert transcription", exact: true }).click();
|
||||
await expect(overlay).toContainText("Dictation failed: Synthetic transcription failure");
|
||||
await overlay.getByRole("button", { name: "Cancel dictation", exact: true }).click();
|
||||
await expect(page.getByTestId("terminal-dictation-overlay")).toHaveCount(0);
|
||||
await expect(visibleTestId(page, "terminal-dictation")).toBeVisible();
|
||||
await page.waitForTimeout(1_500);
|
||||
expect(await received(terminal.id, "bracketed")).toBe("retried words ");
|
||||
});
|
||||
});
|
||||
|
||||
for (const scenario of [
|
||||
{
|
||||
name: "a host without the dictation capability asks to update the host",
|
||||
capability: "missing" as const,
|
||||
message: "Update the host to use dictation in terminals.",
|
||||
},
|
||||
{
|
||||
name: "disabled dictation with an empty reason shows the fallback message",
|
||||
capability: { enabled: false, reason: " " },
|
||||
message: "Dictation is not available on this host.",
|
||||
},
|
||||
{
|
||||
name: "disabled dictation shows the host's reason",
|
||||
capability: { enabled: false, reason: "No speech model installed" },
|
||||
message: "No speech model installed",
|
||||
},
|
||||
]) {
|
||||
test(scenario.name, async ({ page }) => {
|
||||
const ws = await setup(page, { capability: scenario.capability });
|
||||
const terminal = await createReceiver("bracketed", "dictation-capability");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: terminal.id });
|
||||
await visibleTestId(page, "terminal-toolbar-dictation").click();
|
||||
|
||||
await expect(page.getByTestId("app-toast-message")).toHaveText(scenario.message);
|
||||
await expect(page.getByTestId("terminal-dictation-cancel")).toHaveCount(0);
|
||||
expect((await microphoneStats(page)).getUserMediaCalls).toBe(0);
|
||||
expect(ws.streamStarts()).toBe(0);
|
||||
});
|
||||
}
|
||||
|
||||
test("hiding a terminal while its microphone is still being acquired releases it", async ({
|
||||
page,
|
||||
}) => {
|
||||
const ws = await setup(page);
|
||||
const first = await createReceiver("bracketed", "dictation-held-hide-a");
|
||||
const second = await createReceiver("bracketed", "dictation-held-hide-b");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: first.id });
|
||||
await holdMicrophone(page);
|
||||
await visibleTestId(page, "terminal-toolbar-dictation").click();
|
||||
await expectMicrophoneStat(page, "getUserMediaCalls", 1);
|
||||
await expect(visibleTestId(page, "terminal-dictation-cancel")).toBeVisible();
|
||||
|
||||
await selectTerminalTab(page, second);
|
||||
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
|
||||
await releaseMicrophone(page);
|
||||
|
||||
await expectMicrophoneStat(page, "trackStops", 1);
|
||||
await expectNothingReceived(page, [first, "bracketed"], [second, "bracketed"]);
|
||||
expect(ws.streamStarts()).toBe(0);
|
||||
expect(ws.chunks()).toBe(0);
|
||||
|
||||
await selectTerminalTab(page, first);
|
||||
await expect(visibleTestId(page, "terminal-toolbar-dictation")).toBeVisible();
|
||||
await expect(
|
||||
page.getByTestId("terminal-dictation-cancel").filter({ visible: true }),
|
||||
).toHaveCount(0);
|
||||
await expect(
|
||||
page.getByTestId("terminal-dictation-confirm").filter({ visible: true }),
|
||||
).toHaveCount(0);
|
||||
});
|
||||
|
||||
test("closing a terminal while its microphone is still being acquired releases it", async ({
|
||||
page,
|
||||
}) => {
|
||||
const ws = await setup(page);
|
||||
const first = await createReceiver("bracketed", "dictation-held-close-a");
|
||||
const second = await createReceiver("bracketed", "dictation-held-close-b");
|
||||
|
||||
await harness.openTerminal(page, { terminalId: first.id });
|
||||
await holdMicrophone(page);
|
||||
await visibleTestId(page, "terminal-toolbar-dictation").click();
|
||||
await expectMicrophoneStat(page, "getUserMediaCalls", 1);
|
||||
|
||||
// Closing the terminal closes its tab, so its pane unmounts with the start pending.
|
||||
await harness.killTerminal(first.id);
|
||||
await expect(page.getByTestId(`workspace-tab-terminal_${first.id}`)).toHaveCount(0);
|
||||
await releaseMicrophone(page);
|
||||
|
||||
await expectMicrophoneStat(page, "trackStops", 1);
|
||||
await expectNothingReceived(page, [second, "bracketed"]);
|
||||
expect(ws.streamStarts()).toBe(0);
|
||||
expect(ws.chunks()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -306,7 +306,11 @@ test.describe("Terminal protocol queries", () => {
|
||||
try {
|
||||
const upload = visibleTestId(page, "terminal-toolbar-upload");
|
||||
await expect(upload).toHaveCount(1);
|
||||
// The terminal slot sits right before "...": upload, then the mic.
|
||||
expect(await controlBefore(visibleTestId(page, "workspace-split-pane-menu"))).toBe(
|
||||
"terminal-toolbar-dictation",
|
||||
);
|
||||
expect(await controlBefore(visibleTestId(page, "terminal-toolbar-dictation"))).toBe(
|
||||
"terminal-toolbar-upload",
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
import type { Page } from "@playwright/test";
|
||||
import { daemonWsRoutePattern } from "./daemon-port";
|
||||
|
||||
type WebSocketMessage = string | Buffer;
|
||||
|
||||
interface SessionSocket {
|
||||
send(message: string): void;
|
||||
}
|
||||
|
||||
/** What the rewritten `server_info` reports for dictation; `missing` drops the voice field. */
|
||||
export type DictationCapability = { enabled: boolean; reason: string } | "missing";
|
||||
|
||||
function parseEnvelope(message: WebSocketMessage): {
|
||||
type?: unknown;
|
||||
message?: Record<string, unknown>;
|
||||
} | null {
|
||||
const raw = typeof message === "string" ? message : message.toString("utf8");
|
||||
try {
|
||||
return JSON.parse(raw) as { type?: unknown; message?: Record<string, unknown> };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function sessionMessage(message: WebSocketMessage): Record<string, unknown> | null {
|
||||
const envelope = parseEnvelope(message);
|
||||
return envelope?.type === "session" && envelope.message ? envelope.message : null;
|
||||
}
|
||||
|
||||
export function sendSessionMessage(ws: SessionSocket, message: Record<string, unknown>): void {
|
||||
ws.send(JSON.stringify({ type: "session", message }));
|
||||
}
|
||||
|
||||
function rewriteDictationCapability(
|
||||
message: WebSocketMessage,
|
||||
capability: DictationCapability,
|
||||
): WebSocketMessage {
|
||||
const envelope = parseEnvelope(message);
|
||||
const payload = envelope?.message?.payload;
|
||||
if (
|
||||
envelope?.message?.type !== "status" ||
|
||||
!payload ||
|
||||
typeof payload !== "object" ||
|
||||
(payload as { status?: unknown }).status !== "server_info"
|
||||
) {
|
||||
return message;
|
||||
}
|
||||
|
||||
const serverInfo = payload as Record<string, unknown>;
|
||||
if (capability === "missing") {
|
||||
const capabilities = serverInfo.capabilities as Record<string, unknown> | undefined;
|
||||
if (capabilities) {
|
||||
delete capabilities.voice;
|
||||
}
|
||||
} else {
|
||||
serverInfo.capabilities = {
|
||||
voice: {
|
||||
dictation: capability,
|
||||
voice: { enabled: false, reason: "Realtime voice is disabled in this test." },
|
||||
},
|
||||
};
|
||||
}
|
||||
return JSON.stringify(envelope);
|
||||
}
|
||||
|
||||
interface SyntheticMicrophoneState {
|
||||
getUserMediaCalls: number;
|
||||
trackStops: number;
|
||||
hold: boolean;
|
||||
pending: Array<() => void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces getUserMedia with an oscillator stream, so the app's real web audio
|
||||
* source runs end to end. Counts calls and stopped tracks, and can hold an
|
||||
* acquisition until the test releases it (a permission prompt left open).
|
||||
*/
|
||||
export async function installSyntheticMicrophone(page: Page): Promise<void> {
|
||||
await page.addInitScript(() => {
|
||||
const state: SyntheticMicrophoneState = {
|
||||
getUserMediaCalls: 0,
|
||||
trackStops: 0,
|
||||
hold: false,
|
||||
pending: [],
|
||||
};
|
||||
(
|
||||
window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState }
|
||||
).__paseoSyntheticMicrophone = state;
|
||||
Object.defineProperty(navigator.mediaDevices, "getUserMedia", {
|
||||
configurable: true,
|
||||
value: async () => {
|
||||
state.getUserMediaCalls += 1;
|
||||
if (state.hold) {
|
||||
await new Promise<void>((resolve) => state.pending.push(resolve));
|
||||
}
|
||||
const context = new AudioContext();
|
||||
const oscillator = context.createOscillator();
|
||||
const destination = context.createMediaStreamDestination();
|
||||
oscillator.connect(destination);
|
||||
oscillator.start();
|
||||
for (const track of destination.stream.getTracks()) {
|
||||
const stop = track.stop.bind(track);
|
||||
track.stop = () => {
|
||||
state.trackStops += 1;
|
||||
stop();
|
||||
};
|
||||
}
|
||||
return destination.stream;
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function microphoneStats(
|
||||
page: Page,
|
||||
): Promise<{ getUserMediaCalls: number; trackStops: number }> {
|
||||
return await page.evaluate(() => {
|
||||
const state = (window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState })
|
||||
.__paseoSyntheticMicrophone;
|
||||
return { getUserMediaCalls: state.getUserMediaCalls, trackStops: state.trackStops };
|
||||
});
|
||||
}
|
||||
|
||||
/** Makes the next getUserMedia calls wait until `releaseMicrophone`. */
|
||||
export async function holdMicrophone(page: Page): Promise<void> {
|
||||
await page.evaluate(() => {
|
||||
(
|
||||
window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState }
|
||||
).__paseoSyntheticMicrophone.hold = true;
|
||||
});
|
||||
}
|
||||
|
||||
export async function releaseMicrophone(page: Page): Promise<void> {
|
||||
await page.evaluate(() => {
|
||||
const state = (window as unknown as { __paseoSyntheticMicrophone: SyntheticMicrophoneState })
|
||||
.__paseoSyntheticMicrophone;
|
||||
state.hold = false;
|
||||
for (const resolve of state.pending.splice(0)) {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export interface DictationWsHarness {
|
||||
/** Every `dictation_stream_start` the app sent. */
|
||||
streamStarts(): number;
|
||||
/** Audio chunks the app sent, across all dictation ids. */
|
||||
chunks(): number;
|
||||
/** Resolves on the first chunk the app sends after this call. */
|
||||
nextChunk(): Promise<void>;
|
||||
/** The final transcript the next finish replies with. */
|
||||
setTranscript(text: string): void;
|
||||
/** Keeps the next final reply until `releaseFinal`. */
|
||||
holdFinal(): void;
|
||||
releaseFinal(): void;
|
||||
/** Answers the next finish with a stream error instead of a final. */
|
||||
failNextFinish(error: string): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Answers the app's dictation stream in place of the daemon: acknowledges starts
|
||||
* and chunks, accepts finishes and replies with the scenario's transcript. Rewrites
|
||||
* `server_info` to the given dictation capability and forwards everything else, so
|
||||
* terminals, agents and workspaces keep using the real daemon.
|
||||
* `interceptClientMessage` lets a spec answer other requests itself; return true
|
||||
* when it handled one.
|
||||
*/
|
||||
export async function installDictationWsHarness(
|
||||
page: Page,
|
||||
options: {
|
||||
transcript?: string;
|
||||
capability?: DictationCapability;
|
||||
interceptClientMessage?: (request: Record<string, unknown>, ws: SessionSocket) => boolean;
|
||||
} = {},
|
||||
): Promise<DictationWsHarness> {
|
||||
const capability = options.capability ?? { enabled: true, reason: "" };
|
||||
let transcript = options.transcript ?? "";
|
||||
let starts = 0;
|
||||
let chunkCount = 0;
|
||||
let chunkWaiters: Array<() => void> = [];
|
||||
let holdNextFinal = false;
|
||||
let heldFinal: (() => void) | null = null;
|
||||
let nextFinishError: string | null = null;
|
||||
|
||||
await page.routeWebSocket(daemonWsRoutePattern(), (ws) => {
|
||||
const server = ws.connectToServer();
|
||||
|
||||
ws.onMessage((message) => {
|
||||
const request = sessionMessage(message);
|
||||
const type = request?.type;
|
||||
const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null;
|
||||
|
||||
if (type === "dictation_stream_start" && dictationId) {
|
||||
starts += 1;
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_ack",
|
||||
payload: { dictationId, ackSeq: -1 },
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (type === "dictation_stream_chunk" && dictationId) {
|
||||
chunkCount += 1;
|
||||
const seq = typeof request?.seq === "number" ? request.seq : 0;
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_ack",
|
||||
payload: { dictationId, ackSeq: seq },
|
||||
});
|
||||
const waiters = chunkWaiters;
|
||||
chunkWaiters = [];
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (type === "dictation_stream_finish" && dictationId) {
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_finish_accepted",
|
||||
payload: { dictationId, timeoutMs: 30_000 },
|
||||
});
|
||||
if (nextFinishError !== null) {
|
||||
const error = nextFinishError;
|
||||
nextFinishError = null;
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_error",
|
||||
payload: { dictationId, error, retryable: true },
|
||||
});
|
||||
return;
|
||||
}
|
||||
const text = transcript;
|
||||
const reply = () =>
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_final",
|
||||
payload: { dictationId, text },
|
||||
});
|
||||
if (holdNextFinal) {
|
||||
holdNextFinal = false;
|
||||
heldFinal = reply;
|
||||
} else {
|
||||
reply();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (type === "dictation_stream_cancel") {
|
||||
return;
|
||||
}
|
||||
if (request && options.interceptClientMessage?.(request, ws)) {
|
||||
return;
|
||||
}
|
||||
|
||||
server.send(message);
|
||||
});
|
||||
|
||||
server.onMessage((message) => ws.send(rewriteDictationCapability(message, capability)));
|
||||
});
|
||||
|
||||
return {
|
||||
streamStarts: () => starts,
|
||||
chunks: () => chunkCount,
|
||||
nextChunk: () => new Promise<void>((resolve) => chunkWaiters.push(resolve)),
|
||||
setTranscript: (text) => {
|
||||
transcript = text;
|
||||
},
|
||||
holdFinal: () => {
|
||||
holdNextFinal = true;
|
||||
},
|
||||
releaseFinal: () => {
|
||||
const reply = heldFinal;
|
||||
heldFinal = null;
|
||||
if (!reply) {
|
||||
throw new Error("No final reply is being held");
|
||||
}
|
||||
reply();
|
||||
},
|
||||
failNextFinish: (error) => {
|
||||
nextFinishError = error;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { useTranslation } from "react-i18next";
|
||||
import { VolumeMeter } from "./volume-meter";
|
||||
import { FOOTER_HEIGHT } from "@/constants/layout";
|
||||
import type { DictationStatus } from "@/hooks/use-dictation";
|
||||
import { formatDictationDuration } from "@/utils/format-dictation-duration";
|
||||
|
||||
interface DictationControlsProps {
|
||||
volume: number;
|
||||
@@ -24,12 +25,6 @@ interface DictationControlsProps {
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
function formatDuration(seconds: number): string {
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = seconds % 60;
|
||||
return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
export function DictationControls({
|
||||
volume,
|
||||
duration,
|
||||
@@ -88,7 +83,7 @@ export function DictationControls({
|
||||
<VolumeMeter volume={volume} isMuted={false} isSpeaking={false} orientation="horizontal" />
|
||||
</View>
|
||||
<Text style={timerTextStyle}>
|
||||
{status === "paused" ? t("message.dictation.paused") : formatDuration(duration)}
|
||||
{status === "paused" ? t("message.dictation.paused") : formatDictationDuration(duration)}
|
||||
</Text>
|
||||
<View style={styles.actionGroup}>
|
||||
<Pressable
|
||||
@@ -152,7 +147,13 @@ export function DictationOverlay({
|
||||
onAcceptAndSend,
|
||||
onRetry,
|
||||
onDiscard,
|
||||
}: Omit<DictationControlsProps, "onStart" | "disabled" | "transcript"> & { errorText?: string }) {
|
||||
testID,
|
||||
}: Omit<DictationControlsProps, "onStart" | "disabled" | "transcript" | "onAcceptAndSend"> & {
|
||||
errorText?: string;
|
||||
/** Omit to show insert as the only confirm action (terminals never send). */
|
||||
onAcceptAndSend?: () => void;
|
||||
testID?: string;
|
||||
}) {
|
||||
const { theme } = useUnistyles();
|
||||
const { t } = useTranslation();
|
||||
const isFailed = status === "failed";
|
||||
@@ -190,7 +191,7 @@ export function DictationOverlay({
|
||||
}
|
||||
|
||||
return (
|
||||
<View style={containerStyle}>
|
||||
<View style={containerStyle} testID={testID}>
|
||||
<Pressable
|
||||
onPress={handleCancel}
|
||||
disabled={actionsDisabled && !isFailed}
|
||||
@@ -210,7 +211,7 @@ export function DictationOverlay({
|
||||
orientation="horizontal"
|
||||
color={theme.colors.accentForeground}
|
||||
/>
|
||||
<Text style={overlayTimerTextStyle}>{formatDuration(duration)}</Text>
|
||||
<Text style={overlayTimerTextStyle}>{formatDictationDuration(duration)}</Text>
|
||||
</View>
|
||||
{status === "paused" ? (
|
||||
<Text numberOfLines={2} style={overlayTranscriptTextStyle}>
|
||||
@@ -242,7 +243,7 @@ export function DictationOverlay({
|
||||
<RefreshCcw size={theme.iconSize.lg} color={theme.colors.accent} strokeWidth={2.5} />
|
||||
</Pressable>
|
||||
) : null}
|
||||
{!actionsDisabled && !isFailed ? (
|
||||
{!actionsDisabled && !isFailed && onAcceptAndSend ? (
|
||||
<>
|
||||
<Pressable
|
||||
onPress={onAccept}
|
||||
@@ -266,6 +267,16 @@ export function DictationOverlay({
|
||||
</Pressable>
|
||||
</>
|
||||
) : null}
|
||||
{!actionsDisabled && !isFailed && !onAcceptAndSend ? (
|
||||
<Pressable
|
||||
onPress={onAccept}
|
||||
accessibilityRole="button"
|
||||
accessibilityLabel={t("message.dictation.insert")}
|
||||
style={overlayConfirmButtonStyle}
|
||||
>
|
||||
<Check size={theme.iconSize.lg} color={theme.colors.accent} strokeWidth={2.5} />
|
||||
</Pressable>
|
||||
) : null}
|
||||
</View>
|
||||
</View>
|
||||
);
|
||||
|
||||
@@ -2,7 +2,7 @@ import { TerminalFind, type TerminalPaneFindHandle } from "@/terminal/find";
|
||||
import type { TerminalFindResult } from "@/terminal/runtime/terminal-emulator-runtime";
|
||||
import { LoadingSpinner } from "@/components/ui/loading-spinner";
|
||||
import { useRetainedPanelActive } from "@/components/retained-panel";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { useCallback, useEffect, useId, useMemo, useRef, useState } from "react";
|
||||
import * as Clipboard from "expo-clipboard";
|
||||
import {
|
||||
Pressable,
|
||||
@@ -63,7 +63,20 @@ import {
|
||||
type TerminalDropDelivery,
|
||||
} from "@/terminal/drop/terminal-file-drop";
|
||||
import { preparePathsForTerminalInput } from "@/terminal/drop/terminal-path";
|
||||
import { useRegisterTerminalInputSink, type TerminalInputSink } from "@/terminal/input-sink";
|
||||
import {
|
||||
useRegisterTerminalInputSink,
|
||||
type TerminalDictationActions,
|
||||
type TerminalInputSink,
|
||||
} from "@/terminal/input-sink";
|
||||
import {
|
||||
TerminalDictationOwner,
|
||||
type TerminalDictationOwnerActions,
|
||||
} from "@/terminal/dictation/terminal-dictation-owner";
|
||||
import { isTerminalKeyboardOwner } from "@/terminal/dictation/terminal-dictation-lifecycle";
|
||||
import {
|
||||
TerminalKeyBarDictationButton,
|
||||
TerminalKeyBarDictationGate,
|
||||
} from "@/terminal/toolbar/terminal-key-bar-dictation";
|
||||
import type { SelectedFile } from "@/attachments/selected-file";
|
||||
import { TerminalUploadButton } from "@/terminal/toolbar/terminal-upload-button";
|
||||
import { toErrorMessage } from "@/utils/error-messages";
|
||||
@@ -91,7 +104,7 @@ import {
|
||||
type TerminalResizeRequest,
|
||||
} from "./terminal-resize-debouncer";
|
||||
import { useIsCompactFormFactor } from "@/constants/layout";
|
||||
import { isNative } from "@/constants/platform";
|
||||
import { isNative, isWeb } from "@/constants/platform";
|
||||
import {
|
||||
applyTerminalRendererReadyChange,
|
||||
resolveTerminalStreamTarget,
|
||||
@@ -950,13 +963,34 @@ export function TerminalPane({
|
||||
);
|
||||
});
|
||||
|
||||
// The owner mounts only while connected and remounts per terminal; the sink keeps
|
||||
// one stable forwarding object and reaches whichever owner is mounted now.
|
||||
const dictationOwnerId = useId();
|
||||
const dictationActionsRef = useRef<TerminalDictationOwnerActions | null>(null);
|
||||
const dictationActions = useMemo<TerminalDictationActions>(
|
||||
() => ({
|
||||
ownerId: dictationOwnerId,
|
||||
toggle: () => dictationActionsRef.current?.toggle(),
|
||||
cancel: () => dictationActionsRef.current?.cancel(),
|
||||
confirm: () => dictationActionsRef.current?.confirm(),
|
||||
retry: () => dictationActionsRef.current?.retry(),
|
||||
discard: () => dictationActionsRef.current?.discard(),
|
||||
}),
|
||||
[dictationOwnerId],
|
||||
);
|
||||
const terminalSurfaceRef = useRef<View>(null);
|
||||
const isDictationKeyboardOwner = useStableEvent(
|
||||
() => isWeb && isTerminalKeyboardOwner(terminalSurfaceRef.current, document.activeElement),
|
||||
);
|
||||
|
||||
const inputSink = useMemo<TerminalInputSink>(
|
||||
() => ({
|
||||
pasteText: pasteTerminalText,
|
||||
deliverFiles: deliverPickedFiles,
|
||||
focus: requestTerminalFocus,
|
||||
dictation: dictationActions,
|
||||
}),
|
||||
[pasteTerminalText, deliverPickedFiles, requestTerminalFocus],
|
||||
[pasteTerminalText, deliverPickedFiles, requestTerminalFocus, dictationActions],
|
||||
);
|
||||
useRegisterTerminalInputSink(serverId, terminalId, inputSink);
|
||||
|
||||
@@ -1203,6 +1237,8 @@ export function TerminalPane({
|
||||
) : null;
|
||||
case "upload":
|
||||
return <TerminalUploadButton key={controlId} sink={inputSink} placement="keybar" />;
|
||||
case "dictation":
|
||||
return <TerminalKeyBarDictationButton key={controlId} sink={inputSink} />;
|
||||
case "keyboardToggle":
|
||||
return (
|
||||
<KeyboardToggleButton
|
||||
@@ -1232,9 +1268,20 @@ export function TerminalPane({
|
||||
|
||||
return (
|
||||
<Animated.View style={containerStyle}>
|
||||
<TerminalDictationOwner
|
||||
key={`${serverId}:${terminalId}`}
|
||||
ownerId={dictationOwnerId}
|
||||
serverId={serverId}
|
||||
terminalId={terminalId}
|
||||
isPresented={isTerminalPresented}
|
||||
pasteText={pasteTerminalText}
|
||||
focus={requestTerminalFocus}
|
||||
isKeyboardOwner={isDictationKeyboardOwner}
|
||||
actionsRef={dictationActionsRef}
|
||||
/>
|
||||
<FileDropZone style={styles.outputContainer}>
|
||||
<TerminalFileDropSink onFiles={handleDroppedFiles} />
|
||||
<View style={styles.terminalGestureContainer}>
|
||||
<View ref={terminalSurfaceRef} style={styles.terminalGestureContainer}>
|
||||
<TerminalEmulator
|
||||
ref={emulatorRef}
|
||||
dom={TERMINAL_EMULATOR_DOM_PROPS}
|
||||
@@ -1308,14 +1355,16 @@ export function TerminalPane({
|
||||
onLayout={handleKeyBarLayout}
|
||||
testID={isNative && !isMobile ? "terminal-virtual-keyboard-native-wide" : undefined}
|
||||
>
|
||||
{virtualKeyboardRows.map((row) => (
|
||||
<View
|
||||
key={row.map(getTerminalVirtualKeyboardControlId).join(":")}
|
||||
style={styles.keyboardRow}
|
||||
>
|
||||
{row.map(renderVirtualKeyboardControl)}
|
||||
</View>
|
||||
))}
|
||||
<TerminalKeyBarDictationGate sink={inputSink}>
|
||||
{virtualKeyboardRows.map((row) => (
|
||||
<View
|
||||
key={row.map(getTerminalVirtualKeyboardControlId).join(":")}
|
||||
style={styles.keyboardRow}
|
||||
>
|
||||
{row.map(renderVirtualKeyboardControl)}
|
||||
</View>
|
||||
))}
|
||||
</TerminalKeyBarDictationGate>
|
||||
</View>
|
||||
</View>
|
||||
) : null}
|
||||
|
||||
@@ -2,6 +2,7 @@ import { resampleToPcm16 } from "@/audio/pcm";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { parsePcm16Wav } from "@/utils/pcm16-wav";
|
||||
import { isElectronRuntime } from "@/desktop/host";
|
||||
import { AttemptCancelledError } from "@/utils/attempt-guard";
|
||||
|
||||
import type {
|
||||
DictationAudioSource,
|
||||
@@ -154,6 +155,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
started: boolean;
|
||||
mode: "pcm" | "recorder";
|
||||
recorder: RecorderRefs;
|
||||
// Advanced by every stop(), so a start still waiting on getUserMedia can tell it
|
||||
// was stopped and must not install its stream.
|
||||
startGeneration: number;
|
||||
}>({
|
||||
stream: null,
|
||||
context: null,
|
||||
@@ -170,6 +174,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
stoppedResolve: null,
|
||||
stoppedReject: null,
|
||||
},
|
||||
startGeneration: 0,
|
||||
});
|
||||
|
||||
const decodeAudioData = useCallback(
|
||||
@@ -233,7 +238,8 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const rawStream = await navigator.mediaDevices.getUserMedia({
|
||||
const startGeneration = refs.current.startGeneration;
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
@@ -241,8 +247,11 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
|
||||
const stream = rawStream;
|
||||
if (refs.current.startGeneration !== startGeneration) {
|
||||
// Stopped while the permission prompt or device acquisition was pending.
|
||||
disconnectDictationAudioGraph({ processor: null, source: null, gain: null, stream });
|
||||
throw new AttemptCancelledError("Dictation start was cancelled");
|
||||
}
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
|
||||
@@ -371,6 +380,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
refs.current.started = false;
|
||||
refs.current.startGeneration += 1;
|
||||
setVolume(0);
|
||||
|
||||
const { processor, source, gain, context, stream, pending, mode, recorder } = refs.current;
|
||||
@@ -418,6 +428,7 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
stoppedResolve: null,
|
||||
stoppedReject: null,
|
||||
},
|
||||
startGeneration: refs.current.startGeneration,
|
||||
};
|
||||
}, [decodeAudioData, emitPcmSegments]);
|
||||
|
||||
@@ -436,7 +447,9 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
await start();
|
||||
} catch (err) {
|
||||
const normalized = err instanceof Error ? err : new Error(String(err));
|
||||
onErrorRef.current?.(normalized);
|
||||
if (!(normalized instanceof AttemptCancelledError)) {
|
||||
onErrorRef.current?.(normalized);
|
||||
}
|
||||
throw normalized;
|
||||
}
|
||||
},
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
/**
|
||||
* @vitest-environment jsdom
|
||||
*/
|
||||
import { act, renderHook } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { AttemptCancelledError } from "@/utils/attempt-guard";
|
||||
import type { UseDictationOptions } from "./use-dictation.shared";
|
||||
|
||||
// jsdom has no microphone: the source's start() returns a promise the test settles,
|
||||
// standing in for a permission prompt or a getUserMedia call that is still pending.
|
||||
const audioSource = vi.hoisted(() => {
|
||||
const state = {
|
||||
settleStart: null as null | { resolve: () => void; reject: (error: unknown) => void },
|
||||
start: null as unknown as ReturnType<typeof import("vitest").vi.fn>,
|
||||
stop: null as unknown as ReturnType<typeof import("vitest").vi.fn>,
|
||||
onPcmSegment: null as null | ((audio: string) => void),
|
||||
};
|
||||
return state;
|
||||
});
|
||||
|
||||
vi.mock("@/hooks/use-dictation-audio-source", () => ({
|
||||
useDictationAudioSource: (config: { onPcmSegment: (audio: string) => void }) => {
|
||||
audioSource.onPcmSegment = config.onPcmSegment;
|
||||
return {
|
||||
start: audioSource.start,
|
||||
stop: audioSource.stop,
|
||||
volume: 0,
|
||||
isPaused: false,
|
||||
};
|
||||
},
|
||||
}));
|
||||
|
||||
const { useDictation } = await import("./use-dictation");
|
||||
|
||||
function createFakeClient() {
|
||||
return {
|
||||
isConnected: true,
|
||||
startDictationStream: vi.fn(async () => {}),
|
||||
sendDictationStreamChunk: vi.fn(),
|
||||
finishDictationStream: vi.fn(async (dictationId: string) => ({ dictationId, text: "" })),
|
||||
cancelDictationStream: vi.fn(),
|
||||
subscribeRawMessages: vi.fn(() => () => {}),
|
||||
subscribeConnectionStatus: vi.fn(() => () => {}),
|
||||
on: vi.fn(() => () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
function renderDictation(client: ReturnType<typeof createFakeClient>) {
|
||||
const onError = vi.fn();
|
||||
const onTranscript = vi.fn();
|
||||
const rendered = renderHook(() =>
|
||||
useDictation({
|
||||
client: client as unknown as UseDictationOptions["client"],
|
||||
onTranscript,
|
||||
onError,
|
||||
}),
|
||||
);
|
||||
return { ...rendered, onError, onTranscript };
|
||||
}
|
||||
|
||||
async function flush(): Promise<void> {
|
||||
await act(async () => {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
audioSource.settleStart = null;
|
||||
audioSource.start = vi.fn(
|
||||
() =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
audioSource.settleStart = { resolve, reject };
|
||||
}),
|
||||
);
|
||||
audioSource.stop = vi.fn(async () => {});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe("useDictation pending startup", () => {
|
||||
it("cancels a start that is still acquiring the microphone", async () => {
|
||||
const client = createFakeClient();
|
||||
const { result, onError } = renderDictation(client);
|
||||
|
||||
act(() => {
|
||||
void result.current.startDictation();
|
||||
});
|
||||
expect(result.current.status).toBe("recording");
|
||||
expect(audioSource.settleStart).not.toBeNull();
|
||||
|
||||
await act(async () => {
|
||||
await result.current.cancelDictation();
|
||||
});
|
||||
expect(result.current.status).toBe("idle");
|
||||
// The source is stopped while acquisition is pending, so it can release a late stream itself.
|
||||
expect(audioSource.stop).toHaveBeenCalledTimes(1);
|
||||
|
||||
await act(async () => {
|
||||
audioSource.settleStart?.resolve();
|
||||
});
|
||||
await flush();
|
||||
|
||||
expect(audioSource.stop).toHaveBeenCalledTimes(2);
|
||||
expect(result.current.isRecording).toBe(false);
|
||||
expect(result.current.isRecordingActive()).toBe(false);
|
||||
expect(result.current.status).toBe("idle");
|
||||
expect(client.startDictationStream).not.toHaveBeenCalled();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
expect(result.current.error).toBeNull();
|
||||
});
|
||||
|
||||
it("treats the source's cancelled acquisition as a silent cancel", async () => {
|
||||
const client = createFakeClient();
|
||||
const { result, onError } = renderDictation(client);
|
||||
|
||||
act(() => {
|
||||
void result.current.startDictation();
|
||||
});
|
||||
await act(async () => {
|
||||
await result.current.cancelDictation();
|
||||
});
|
||||
await act(async () => {
|
||||
audioSource.settleStart?.reject(new AttemptCancelledError());
|
||||
});
|
||||
await flush();
|
||||
|
||||
expect(result.current.status).toBe("idle");
|
||||
expect(result.current.isRecording).toBe(false);
|
||||
expect(client.startDictationStream).not.toHaveBeenCalled();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
expect(result.current.error).toBeNull();
|
||||
});
|
||||
|
||||
it("stops the late-acquired microphone after an unmount during acquisition", async () => {
|
||||
const client = createFakeClient();
|
||||
const { result, unmount, onError } = renderDictation(client);
|
||||
|
||||
act(() => {
|
||||
void result.current.startDictation();
|
||||
});
|
||||
unmount();
|
||||
const stopsAtUnmount = audioSource.stop.mock.calls.length;
|
||||
expect(stopsAtUnmount).toBeGreaterThanOrEqual(1);
|
||||
|
||||
await act(async () => {
|
||||
audioSource.settleStart?.resolve();
|
||||
});
|
||||
await flush();
|
||||
|
||||
expect(audioSource.stop.mock.calls.length).toBe(stopsAtUnmount + 1);
|
||||
expect(client.startDictationStream).not.toHaveBeenCalled();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("records and opens the stream when the start is not cancelled", async () => {
|
||||
const client = createFakeClient();
|
||||
const { result, onError } = renderDictation(client);
|
||||
|
||||
act(() => {
|
||||
void result.current.startDictation();
|
||||
});
|
||||
await act(async () => {
|
||||
audioSource.settleStart?.resolve();
|
||||
});
|
||||
await flush();
|
||||
|
||||
expect(result.current.isRecording).toBe(true);
|
||||
expect(result.current.status).toBe("recording");
|
||||
expect(client.startDictationStream).toHaveBeenCalledTimes(1);
|
||||
expect(audioSource.stop).not.toHaveBeenCalled();
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("drops the partial segment the source flushes while a recording is cancelled", async () => {
|
||||
const client = createFakeClient();
|
||||
// Stopping a real source flushes the audio it still holds.
|
||||
audioSource.stop = vi.fn(async () => {
|
||||
audioSource.onPcmSegment?.("AAAA");
|
||||
});
|
||||
const { result } = renderDictation(client);
|
||||
|
||||
act(() => {
|
||||
void result.current.startDictation();
|
||||
});
|
||||
await act(async () => {
|
||||
audioSource.settleStart?.resolve();
|
||||
});
|
||||
await flush();
|
||||
expect(client.startDictationStream).toHaveBeenCalledTimes(1);
|
||||
|
||||
await act(async () => {
|
||||
await result.current.cancelDictation();
|
||||
});
|
||||
await flush();
|
||||
|
||||
expect(result.current.status).toBe("idle");
|
||||
expect(client.startDictationStream).toHaveBeenCalledTimes(1);
|
||||
expect(client.sendDictationStreamChunk).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -70,12 +70,19 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const attemptGuardRef = useRef(new AttemptGuard());
|
||||
// Advanced only by cancel and unmount, never by confirm: a confirm while the stream
|
||||
// is still starting must not make the start tear itself down.
|
||||
const startGuardRef = useRef(new AttemptGuard());
|
||||
const actionGateRef = useRef<{ starting: boolean; confirming: boolean; cancelling: boolean }>({
|
||||
starting: false,
|
||||
confirming: false,
|
||||
cancelling: false,
|
||||
});
|
||||
|
||||
// Audio is accepted from start until a cancel: stopping the source flushes its last
|
||||
// partial segment, which after a cancel would open a new stream for audio nobody wants.
|
||||
const acceptAudioRef = useRef(false);
|
||||
|
||||
const senderRef = useRef<DictationStreamSender | null>(null);
|
||||
if (!senderRef.current) {
|
||||
senderRef.current = new DictationStreamSender({
|
||||
@@ -228,6 +235,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const audio = useDictationAudioSource({
|
||||
onPcmSegment: (audioData) => {
|
||||
if (!acceptAudioRef.current) {
|
||||
return;
|
||||
}
|
||||
senderRef.current?.enqueueSegment(audioData);
|
||||
},
|
||||
onError: (err) => {
|
||||
@@ -264,15 +274,36 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
}
|
||||
|
||||
actionGateRef.current.starting = true;
|
||||
// A cancel or unmount while the microphone is still being acquired advances the
|
||||
// start guard; the checks below then release whatever the start acquired late.
|
||||
const startId = startGuardRef.current.next();
|
||||
const abandonStaleStart = async () => {
|
||||
acceptAudioRef.current = false;
|
||||
try {
|
||||
senderRef.current?.cancel();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
await audio.stop().catch(() => undefined);
|
||||
stopDurationTracking();
|
||||
isRecordingRef.current = false;
|
||||
setIsRecording(false);
|
||||
setStatus("idle");
|
||||
};
|
||||
setError(null);
|
||||
setPartialTranscript("");
|
||||
setDuration(0);
|
||||
setIsProcessing(false);
|
||||
setStatus("recording");
|
||||
clearStreamingState();
|
||||
acceptAudioRef.current = true;
|
||||
|
||||
try {
|
||||
await audio.start();
|
||||
if (!startGuardRef.current.isCurrent(startId)) {
|
||||
await abandonStaleStart();
|
||||
return;
|
||||
}
|
||||
isRecordingRef.current = true;
|
||||
setIsRecording(true);
|
||||
if (enableDuration) {
|
||||
@@ -280,8 +311,16 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
}
|
||||
if (client?.isConnected) {
|
||||
await startNewStream("start");
|
||||
if (!startGuardRef.current.isCurrent(startId)) {
|
||||
await abandonStaleStart();
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (!startGuardRef.current.isCurrent(startId)) {
|
||||
await abandonStaleStart();
|
||||
return;
|
||||
}
|
||||
await audio.stop().catch(() => undefined);
|
||||
stopDurationTracking();
|
||||
isRecordingRef.current = false;
|
||||
@@ -303,15 +342,35 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
stopDurationTracking,
|
||||
]);
|
||||
|
||||
// The start is still waiting on the microphone. Show idle at once and stop the
|
||||
// source now, so a stream acquired after this point is released by the source
|
||||
// itself; the start continuation then sees the stale attempt and tears down the rest.
|
||||
const cancelPendingStart = useCallback(async () => {
|
||||
acceptAudioRef.current = false;
|
||||
stopDurationTracking();
|
||||
setDuration(0);
|
||||
setError(null);
|
||||
setStatus("idle");
|
||||
clearStreamingState();
|
||||
await audio.stop().catch((err: unknown) => {
|
||||
reportError(err, "Failed to stop pending dictation audio");
|
||||
});
|
||||
}, [audio, clearStreamingState, reportError, stopDurationTracking]);
|
||||
|
||||
const cancelDictation = useCallback(async () => {
|
||||
attemptGuardRef.current.cancel();
|
||||
startGuardRef.current.cancel();
|
||||
if (actionGateRef.current.cancelling) {
|
||||
return;
|
||||
}
|
||||
if (!isRecordingRef.current && !isProcessingRef.current) {
|
||||
if (actionGateRef.current.starting) {
|
||||
await cancelPendingStart();
|
||||
}
|
||||
return;
|
||||
}
|
||||
actionGateRef.current.cancelling = true;
|
||||
acceptAudioRef.current = false;
|
||||
stopDurationTracking();
|
||||
setDuration(0);
|
||||
setError(null);
|
||||
@@ -334,7 +393,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
clearStreamingState();
|
||||
actionGateRef.current.cancelling = false;
|
||||
}
|
||||
}, [audio, clearStreamingState, reportError, stopDurationTracking]);
|
||||
}, [audio, cancelPendingStart, clearStreamingState, reportError, stopDurationTracking]);
|
||||
|
||||
const confirmDictation = useCallback(async () => {
|
||||
if (actionGateRef.current.confirming) {
|
||||
@@ -444,9 +503,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
useEffect(() => {
|
||||
const attemptGuard = attemptGuardRef.current;
|
||||
const startGuard = startGuardRef.current;
|
||||
const audioStop = audioStopRef;
|
||||
return () => {
|
||||
attemptGuard.cancel();
|
||||
startGuard.cancel();
|
||||
stopDurationTracking();
|
||||
void audioStop.current().catch(() => undefined);
|
||||
senderRef.current?.dispose();
|
||||
|
||||
@@ -622,6 +622,13 @@ export const ar: TranslationResources = {
|
||||
uploadingFile: "جارٍ رفع الملف…",
|
||||
actions: {
|
||||
uploadFile: "رفع ملف",
|
||||
startDictation: "بدء الإملاء",
|
||||
stopDictation: "إيقاف ولصق النص المفرغ",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "حدّث المضيف لاستخدام الإملاء في الطرفيات.",
|
||||
unavailable: "الإملاء غير متاح على هذا المضيف.",
|
||||
discard: "تجاهل التسجيل",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2607,6 +2614,7 @@ export const ar: TranslationResources = {
|
||||
cycleAgentMode: "تبديل وضع الوكيل",
|
||||
toggleVoiceMode: "تبديل الوضع الصوتي",
|
||||
startStopDictation: "بدء إملاء /stop",
|
||||
startStopTerminalDictation: "بدء/إيقاف الإملاء في الطرفية",
|
||||
interruptAgent: "عامل المقاطعة",
|
||||
sendMessage: "أرسل رسالة",
|
||||
queueMessage: "رسالة قائمة الانتظار",
|
||||
|
||||
@@ -619,6 +619,13 @@ export const en = {
|
||||
uploadingFile: "Uploading file…",
|
||||
actions: {
|
||||
uploadFile: "Upload file",
|
||||
startDictation: "Start dictation",
|
||||
stopDictation: "Stop and paste transcript",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "Update the host to use dictation in terminals.",
|
||||
unavailable: "Dictation is not available on this host.",
|
||||
discard: "Discard recording",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2694,6 +2701,7 @@ export const en = {
|
||||
cycleAgentMode: "Cycle agent mode",
|
||||
toggleVoiceMode: "Toggle voice mode",
|
||||
startStopDictation: "Start/stop dictation",
|
||||
startStopTerminalDictation: "Start/stop dictation in terminal",
|
||||
interruptAgent: "Interrupt agent",
|
||||
sendMessage: "Send message",
|
||||
queueMessage: "Queue message",
|
||||
|
||||
@@ -628,6 +628,13 @@ export const es: TranslationResources = {
|
||||
uploadingFile: "Subiendo archivo…",
|
||||
actions: {
|
||||
uploadFile: "Subir archivo",
|
||||
startDictation: "Iniciar dictado",
|
||||
stopDictation: "Detener y pegar la transcripción",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "Actualiza el host para usar el dictado en terminales.",
|
||||
unavailable: "El dictado no está disponible en este host.",
|
||||
discard: "Descartar grabación",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2625,6 +2632,7 @@ export const es: TranslationResources = {
|
||||
cycleAgentMode: "Alternar modo del agente",
|
||||
toggleVoiceMode: "Alternar modo de voz",
|
||||
startStopDictation: "Iniciar dictado/stop",
|
||||
startStopTerminalDictation: "Iniciar/detener dictado en la terminal",
|
||||
interruptAgent: "agente de interrupción",
|
||||
sendMessage: "enviar mensaje",
|
||||
queueMessage: "mensaje de cola",
|
||||
|
||||
@@ -628,6 +628,13 @@ export const fr: TranslationResources = {
|
||||
uploadingFile: "Envoi du fichier…",
|
||||
actions: {
|
||||
uploadFile: "Envoyer un fichier",
|
||||
startDictation: "Démarrer la dictée",
|
||||
stopDictation: "Arrêter et coller la transcription",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "Mettez à jour l'hôte pour utiliser la dictée dans les terminaux.",
|
||||
unavailable: "La dictée n'est pas disponible sur cet hôte.",
|
||||
discard: "Supprimer l'enregistrement",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2626,6 +2633,7 @@ export const fr: TranslationResources = {
|
||||
cycleAgentMode: "Parcourir les modes de l’agent",
|
||||
toggleVoiceMode: "Activer/désactiver le mode vocal",
|
||||
startStopDictation: "Démarrer/arrêter la dictée",
|
||||
startStopTerminalDictation: "Démarrer/arrêter la dictée dans le terminal",
|
||||
interruptAgent: "Interrompre l’agent",
|
||||
sendMessage: "Envoyer le message",
|
||||
queueMessage: "Mettre le message en file d’attente",
|
||||
|
||||
@@ -628,6 +628,13 @@ export const ja: TranslationResources = {
|
||||
uploadingFile: "ファイルをアップロード中…",
|
||||
actions: {
|
||||
uploadFile: "ファイルをアップロード",
|
||||
startDictation: "音声入力を開始",
|
||||
stopDictation: "停止して文字起こしを貼り付け",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "ターミナルで音声入力を使うにはホストを更新してください。",
|
||||
unavailable: "このホストでは音声入力を利用できません。",
|
||||
discard: "録音を破棄",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2592,6 +2599,7 @@ export const ja: TranslationResources = {
|
||||
cycleAgentMode: "エージェントモードを順に切り替え",
|
||||
toggleVoiceMode: "音声モードを切り替え",
|
||||
startStopDictation: "音声入力を開始/停止",
|
||||
startStopTerminalDictation: "ターミナルで音声入力を開始/停止",
|
||||
interruptAgent: "エージェントを中断",
|
||||
sendMessage: "メッセージを送信",
|
||||
queueMessage: "メッセージをキューに追加",
|
||||
|
||||
@@ -623,6 +623,13 @@ export const ko: TranslationResources = {
|
||||
uploadingFile: "파일 업로드 중…",
|
||||
actions: {
|
||||
uploadFile: "파일 업로드",
|
||||
startDictation: "받아쓰기 시작",
|
||||
stopDictation: "중지하고 받아쓴 내용 붙여넣기",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "터미널에서 받아쓰기를 사용하려면 호스트를 업데이트하세요.",
|
||||
unavailable: "이 호스트에서는 받아쓰기를 사용할 수 없습니다.",
|
||||
discard: "녹음 삭제",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2581,6 +2588,7 @@ export const ko: TranslationResources = {
|
||||
cycleAgentMode: "에이전트 모드 전환",
|
||||
toggleVoiceMode: "음성 모드 토글",
|
||||
startStopDictation: "받아쓰기 시작/중지",
|
||||
startStopTerminalDictation: "터미널에서 받아쓰기 시작/중지",
|
||||
interruptAgent: "에이전트 중단",
|
||||
sendMessage: "메시지 보내기",
|
||||
queueMessage: "메시지 대기열에 추가",
|
||||
|
||||
@@ -627,6 +627,13 @@ export const ptBR: TranslationResources = {
|
||||
uploadingFile: "Enviando arquivo…",
|
||||
actions: {
|
||||
uploadFile: "Enviar arquivo",
|
||||
startDictation: "Iniciar ditado",
|
||||
stopDictation: "Parar e colar a transcrição",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "Atualize o host para usar o ditado nos terminais.",
|
||||
unavailable: "O ditado não está disponível neste host.",
|
||||
discard: "Descartar gravação",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2606,6 +2613,7 @@ export const ptBR: TranslationResources = {
|
||||
cycleAgentMode: "Alternar modo do agente",
|
||||
toggleVoiceMode: "Alternar modo de voz",
|
||||
startStopDictation: "Iniciar/parar ditado",
|
||||
startStopTerminalDictation: "Iniciar/parar ditado no terminal",
|
||||
interruptAgent: "Interromper agente",
|
||||
sendMessage: "Enviar mensagem",
|
||||
queueMessage: "Enfileirar mensagem",
|
||||
|
||||
@@ -629,6 +629,13 @@ export const ru: TranslationResources = {
|
||||
uploadingFile: "Загрузка файла…",
|
||||
actions: {
|
||||
uploadFile: "Загрузить файл",
|
||||
startDictation: "Начать диктовку",
|
||||
stopDictation: "Остановить и вставить расшифровку",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "Обновите хост, чтобы использовать диктовку в терминалах.",
|
||||
unavailable: "Диктовка недоступна на этом хосте.",
|
||||
discard: "Удалить запись",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2613,6 +2620,7 @@ export const ru: TranslationResources = {
|
||||
cycleAgentMode: "Переключить режим агента",
|
||||
toggleVoiceMode: "Переключить голосовой режим",
|
||||
startStopDictation: "Начать/остановить диктовку",
|
||||
startStopTerminalDictation: "Начать/остановить диктовку в терминале",
|
||||
interruptAgent: "Прервать агента",
|
||||
sendMessage: "Отправить сообщение",
|
||||
queueMessage: "Поставить сообщение в очередь",
|
||||
|
||||
@@ -622,6 +622,13 @@ export const zhCN: TranslationResources = {
|
||||
uploadingFile: "正在上传文件…",
|
||||
actions: {
|
||||
uploadFile: "上传文件",
|
||||
startDictation: "开始听写",
|
||||
stopDictation: "停止并粘贴转写内容",
|
||||
},
|
||||
dictation: {
|
||||
updateHost: "请更新主机以在终端中使用听写。",
|
||||
unavailable: "此主机不支持听写。",
|
||||
discard: "丢弃录音",
|
||||
},
|
||||
},
|
||||
sheet: {
|
||||
@@ -2543,6 +2550,7 @@ export const zhCN: TranslationResources = {
|
||||
cycleAgentMode: "循环切换代理模式",
|
||||
toggleVoiceMode: "切换语音模式",
|
||||
startStopDictation: "开始/停止听写",
|
||||
startStopTerminalDictation: "在终端中开始/停止听写",
|
||||
interruptAgent: "中断 Agent",
|
||||
sendMessage: "发送消息",
|
||||
queueMessage: "消息排队",
|
||||
|
||||
@@ -48,6 +48,7 @@ export type KeyboardActionId =
|
||||
| "command-center.files"
|
||||
| "shortcuts.dialog.toggle"
|
||||
| "workspace.terminal.new"
|
||||
| "terminal.dictation-toggle"
|
||||
| "workspace.new"
|
||||
| "workspace.project.pick"
|
||||
| "worktree.new"
|
||||
|
||||
@@ -44,6 +44,7 @@ export type KeyboardActionId =
|
||||
| "workspace.pane.close"
|
||||
| "workspace.focus.toggle"
|
||||
| "workspace.terminal.new"
|
||||
| "terminal.dictation-toggle"
|
||||
| "workspace.browser.new"
|
||||
| "sidebar.toggle.right"
|
||||
| "sidebar.toggle.both"
|
||||
@@ -102,6 +103,7 @@ export type KeyboardActionDefinition =
|
||||
| { id: "workspace.pane.close"; scope: KeyboardActionScope }
|
||||
| { id: "workspace.focus.toggle"; scope: KeyboardActionScope }
|
||||
| { id: "workspace.terminal.new"; scope: KeyboardActionScope }
|
||||
| { id: "terminal.dictation-toggle"; scope: KeyboardActionScope }
|
||||
| { id: "workspace.browser.new"; scope: KeyboardActionScope }
|
||||
| { id: "sidebar.toggle.right"; scope: KeyboardActionScope }
|
||||
| { id: "sidebar.toggle.both"; scope: KeyboardActionScope }
|
||||
|
||||
@@ -323,6 +323,12 @@ describe("keyboard-shortcuts", () => {
|
||||
action: "message-input.action",
|
||||
payload: { kind: "dictation-toggle" },
|
||||
},
|
||||
{
|
||||
name: "routes Cmd+D to terminal dictation, not the message input, when terminal is focused on macOS",
|
||||
event: { key: "d", code: "KeyD", metaKey: true },
|
||||
context: { isMac: true, focusScope: "terminal" },
|
||||
action: "terminal.dictation-toggle",
|
||||
},
|
||||
{
|
||||
name: "routes Shift+Tab to cycle agent mode from the message input",
|
||||
event: { key: "Tab", code: "Tab", shiftKey: true },
|
||||
@@ -481,9 +487,14 @@ describe("keyboard-shortcuts", () => {
|
||||
context: { isMac: false, focusScope: "terminal" },
|
||||
},
|
||||
{
|
||||
name: "does not route message-input actions when terminal is focused",
|
||||
name: "leaves Ctrl+D to the PTY as EOF when terminal is focused on non-mac",
|
||||
event: { key: "d", code: "KeyD", ctrlKey: true },
|
||||
context: { isMac: false, focusScope: "terminal" },
|
||||
},
|
||||
{
|
||||
name: "does not bind Cmd+D to terminal dictation on non-mac",
|
||||
event: { key: "d", code: "KeyD", metaKey: true },
|
||||
context: { isMac: true, focusScope: "terminal" },
|
||||
context: { isMac: false, focusScope: "terminal" },
|
||||
},
|
||||
{
|
||||
name: "does not cycle agent mode outside the message input",
|
||||
|
||||
@@ -199,6 +199,7 @@ export const SHORTCUT_HELP_ROW_ORDER: Record<ShortcutSectionId, readonly string[
|
||||
"cycle-agent-mode",
|
||||
"voice-toggle",
|
||||
"dictation-toggle",
|
||||
"terminal-dictation-toggle",
|
||||
"agent-interrupt",
|
||||
"voice-mute-toggle",
|
||||
],
|
||||
@@ -246,6 +247,7 @@ const SHORTCUT_HELP_LABEL_KEYS: Record<string, string> = {
|
||||
"cycle-agent-mode": "settings.shortcuts.help.cycleAgentMode",
|
||||
"voice-toggle": "settings.shortcuts.help.toggleVoiceMode",
|
||||
"dictation-toggle": "settings.shortcuts.help.startStopDictation",
|
||||
"terminal-dictation-toggle": "settings.shortcuts.help.startStopTerminalDictation",
|
||||
"agent-interrupt": "settings.shortcuts.help.interruptAgent",
|
||||
"voice-mute-toggle": "settings.shortcuts.help.muteUnmuteVoiceMode",
|
||||
};
|
||||
@@ -1136,6 +1138,18 @@ const SHORTCUT_BINDINGS: readonly ShortcutBinding[] = [
|
||||
label: "Start/stop dictation",
|
||||
},
|
||||
},
|
||||
// Terminals take Cmd+D on macOS only: elsewhere Ctrl+D is the PTY's EOF and passes through.
|
||||
{
|
||||
id: "terminal-dictation-toggle-cmd-d-mac",
|
||||
action: "terminal.dictation-toggle",
|
||||
combo: "Cmd+D",
|
||||
when: { mac: true, commandCenter: false, focusScope: "terminal" },
|
||||
help: {
|
||||
id: "terminal-dictation-toggle",
|
||||
section: "agent-input",
|
||||
label: "Start/stop dictation in terminal",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "agent-interrupt",
|
||||
action: "agent.interrupt",
|
||||
|
||||
@@ -39,6 +39,7 @@ describe("routeKeyboardShortcut — dispatch passthroughs", () => {
|
||||
["workspace.pin", { id: "workspace.pin", scope: "sidebar" }],
|
||||
["worktree.new", { id: "worktree.new", scope: "sidebar" }],
|
||||
["workspace.terminal.new", { id: "workspace.terminal.new", scope: "workspace" }],
|
||||
["terminal.dictation-toggle", { id: "terminal.dictation-toggle", scope: "workspace" }],
|
||||
["workspace.tab.close.current", { id: "workspace.tab.close-current", scope: "workspace" }],
|
||||
["sidebar.toggle.right", { id: "sidebar.toggle.right", scope: "sidebar" }],
|
||||
["workspace.pane.split.right", { id: "workspace.pane.split.right", scope: "workspace" }],
|
||||
|
||||
@@ -51,6 +51,7 @@ const PASSTHROUGH_DISPATCH: Record<string, KeyboardActionDefinition> = {
|
||||
"workspace.pin": { id: "workspace.pin", scope: "sidebar" },
|
||||
"worktree.new": { id: "worktree.new", scope: "sidebar" },
|
||||
"workspace.terminal.new": { id: "workspace.terminal.new", scope: "workspace" },
|
||||
"terminal.dictation-toggle": { id: "terminal.dictation-toggle", scope: "workspace" },
|
||||
"workspace.tab.close.current": { id: "workspace.tab.close-current", scope: "workspace" },
|
||||
"sidebar.toggle.right": { id: "sidebar.toggle.right", scope: "sidebar" },
|
||||
"workspace.pane.split.right": { id: "workspace.pane.split.right", scope: "workspace" },
|
||||
|
||||
@@ -105,8 +105,9 @@ const PANE_SPLIT_ACTIONS_RESERVED_WIDTH =
|
||||
PANE_SPLIT_ACTIONS_HORIZONTAL_PADDING * 2 +
|
||||
PANE_SPLIT_ACTIONS_OUTER_MARGIN;
|
||||
const PANE_MAXIMIZE_ACTION_RESERVED_WIDTH = smallIconButtonChromeFrameSize(false) + 1;
|
||||
// The terminal slot holds one icon button (upload).
|
||||
const TERMINAL_SLOT_RESERVED_WIDTH = smallIconButtonChromeFrameSize(false) + 1;
|
||||
// The terminal slot holds two icon buttons (upload, mic). While the terminal dictates the
|
||||
// mic grows into a pill; the row re-measures then instead of reserving that width always.
|
||||
const TERMINAL_SLOT_RESERVED_WIDTH = 2 * (smallIconButtonChromeFrameSize(false) + 1);
|
||||
// Chip geometry. `layoutMetrics` measures tabs from these same numbers, so a chip that changes
|
||||
// shape without changing them mis-measures and drops the row into the overflow-scroll fallback at
|
||||
// the wrong width. Keep them together.
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { formatTerminalTranscript } from "./format-terminal-transcript";
|
||||
|
||||
describe("formatTerminalTranscript", () => {
|
||||
it.each([
|
||||
["first\nsecond", "first second "],
|
||||
["first\r\nsecond", "first second "],
|
||||
["first\rsecond", "first second "],
|
||||
["\n hello \r\n", "hello "],
|
||||
["héllo 日本語", "héllo 日本語 "],
|
||||
])("formats %j as %j", (input, expected) => {
|
||||
expect(formatTerminalTranscript(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it.each(["", " ", "\r\n"])("returns null for %j", (input) => {
|
||||
expect(formatTerminalTranscript(input)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
/**
|
||||
* Turns a dictation transcript into terminal input: embedded line breaks become
|
||||
* spaces (a newline would submit the line in most TUIs), the ends are trimmed, and
|
||||
* one trailing space separates it from whatever is typed next. Never adds Enter.
|
||||
* Returns null when nothing is left to paste.
|
||||
*/
|
||||
export function formatTerminalTranscript(text: string): string | null {
|
||||
const singleLine = text.replace(/\r\n|\r|\n/g, " ").trim();
|
||||
return singleLine.length > 0 ? `${singleLine} ` : null;
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { DaemonServerInfo } from "@/stores/session-store";
|
||||
import { resolveTerminalDictationAvailability } from "./terminal-dictation-availability";
|
||||
|
||||
function serverInfo(capabilities?: DaemonServerInfo["capabilities"]): DaemonServerInfo {
|
||||
return {
|
||||
serverId: "server-a",
|
||||
hostname: null,
|
||||
version: "0.13.2",
|
||||
...(capabilities ? { capabilities } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function voice(dictation: { enabled: boolean; reason: string }): DaemonServerInfo["capabilities"] {
|
||||
return { voice: { dictation, voice: { enabled: false, reason: "" } } };
|
||||
}
|
||||
|
||||
describe("resolveTerminalDictationAvailability", () => {
|
||||
it("is disconnected while the host is not connected, whatever it reports", () => {
|
||||
expect(
|
||||
resolveTerminalDictationAvailability({
|
||||
serverInfo: serverInfo(voice({ enabled: true, reason: "" })),
|
||||
isConnected: false,
|
||||
}),
|
||||
).toEqual({ kind: "disconnected" });
|
||||
});
|
||||
|
||||
it("asks to update a host that reports no capabilities", () => {
|
||||
expect(
|
||||
resolveTerminalDictationAvailability({ serverInfo: serverInfo(), isConnected: true }),
|
||||
).toEqual({ kind: "unavailable", reason: "update-host" });
|
||||
expect(resolveTerminalDictationAvailability({ serverInfo: null, isConnected: true })).toEqual({
|
||||
kind: "unavailable",
|
||||
reason: "update-host",
|
||||
});
|
||||
});
|
||||
|
||||
it("asks to update a host whose capabilities have no voice field", () => {
|
||||
expect(
|
||||
resolveTerminalDictationAvailability({ serverInfo: serverInfo({}), isConnected: true }),
|
||||
).toEqual({ kind: "unavailable", reason: "update-host" });
|
||||
});
|
||||
|
||||
it("carries the host's trimmed reason when dictation is disabled", () => {
|
||||
expect(
|
||||
resolveTerminalDictationAvailability({
|
||||
serverInfo: serverInfo(voice({ enabled: false, reason: " No speech model installed " })),
|
||||
isConnected: true,
|
||||
}),
|
||||
).toEqual({ kind: "unavailable", reason: "disabled", message: "No speech model installed" });
|
||||
});
|
||||
|
||||
it.each(["", " "])("leaves the message empty for a disabled reason of %j", (reason) => {
|
||||
expect(
|
||||
resolveTerminalDictationAvailability({
|
||||
serverInfo: serverInfo(voice({ enabled: false, reason })),
|
||||
isConnected: true,
|
||||
}),
|
||||
).toEqual({ kind: "unavailable", reason: "disabled", message: null });
|
||||
});
|
||||
|
||||
it("is ready when the host enables dictation", () => {
|
||||
expect(
|
||||
resolveTerminalDictationAvailability({
|
||||
serverInfo: serverInfo(voice({ enabled: true, reason: "" })),
|
||||
isConnected: true,
|
||||
}),
|
||||
).toEqual({ kind: "ready" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
import type { DaemonServerInfo } from "@/stores/session-store";
|
||||
import { getVoiceReadinessState } from "@/utils/server-info-capabilities";
|
||||
|
||||
export type TerminalDictationAvailability =
|
||||
| { kind: "ready" }
|
||||
| { kind: "disconnected" }
|
||||
| { kind: "unavailable"; reason: "update-host" }
|
||||
| { kind: "unavailable"; reason: "disabled"; message: string | null };
|
||||
|
||||
/**
|
||||
* Whether a terminal can dictate on this host. Gates on the host's explicit
|
||||
* dictation capability only: a host too old to report it is told to update, so a
|
||||
* terminal never falls back to the composer's agent-directory heuristic.
|
||||
*/
|
||||
export function resolveTerminalDictationAvailability(input: {
|
||||
serverInfo: DaemonServerInfo | null | undefined;
|
||||
isConnected: boolean;
|
||||
}): TerminalDictationAvailability {
|
||||
if (!input.isConnected) {
|
||||
return { kind: "disconnected" };
|
||||
}
|
||||
const dictation = getVoiceReadinessState({ serverInfo: input.serverInfo, mode: "dictation" });
|
||||
if (!dictation) {
|
||||
return { kind: "unavailable", reason: "update-host" };
|
||||
}
|
||||
if (!dictation.enabled) {
|
||||
const message = dictation.reason.trim();
|
||||
return {
|
||||
kind: "unavailable",
|
||||
reason: "disabled",
|
||||
message: message.length > 0 ? message : null,
|
||||
};
|
||||
}
|
||||
return { kind: "ready" };
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* @vitest-environment jsdom
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { DictationStatus } from "@/hooks/use-dictation";
|
||||
import {
|
||||
isTerminalKeyboardOwner,
|
||||
resolveTerminalDictationLifecycle,
|
||||
} from "./terminal-dictation-lifecycle";
|
||||
|
||||
describe("resolveTerminalDictationLifecycle", () => {
|
||||
const active: DictationStatus[] = ["recording", "paused", "uploading"];
|
||||
|
||||
it("keeps an idle terminal whatever happens to it", () => {
|
||||
expect(
|
||||
resolveTerminalDictationLifecycle({
|
||||
isPresented: false,
|
||||
isRecordingOwner: false,
|
||||
status: "idle",
|
||||
}),
|
||||
).toBe("keep");
|
||||
});
|
||||
|
||||
it.each(active)("keeps a presented recording owner while %s", (status) => {
|
||||
expect(
|
||||
resolveTerminalDictationLifecycle({ isPresented: true, isRecordingOwner: true, status }),
|
||||
).toBe("keep");
|
||||
});
|
||||
|
||||
it.each(active)("cancels when the terminal stops being presented while %s", (status) => {
|
||||
expect(
|
||||
resolveTerminalDictationLifecycle({ isPresented: false, isRecordingOwner: true, status }),
|
||||
).toBe("cancel");
|
||||
});
|
||||
|
||||
it.each(active)("cancels when another terminal takes the recording while %s", (status) => {
|
||||
expect(
|
||||
resolveTerminalDictationLifecycle({ isPresented: true, isRecordingOwner: false, status }),
|
||||
).toBe("cancel");
|
||||
});
|
||||
|
||||
it("discards a failed attempt once hidden or superseded", () => {
|
||||
expect(
|
||||
resolveTerminalDictationLifecycle({
|
||||
isPresented: false,
|
||||
isRecordingOwner: true,
|
||||
status: "failed",
|
||||
}),
|
||||
).toBe("discard");
|
||||
expect(
|
||||
resolveTerminalDictationLifecycle({
|
||||
isPresented: true,
|
||||
isRecordingOwner: false,
|
||||
status: "failed",
|
||||
}),
|
||||
).toBe("discard");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isTerminalKeyboardOwner", () => {
|
||||
function surfaceWithInput() {
|
||||
const surface = document.createElement("div");
|
||||
const input = document.createElement("textarea");
|
||||
surface.append(input);
|
||||
document.body.append(surface);
|
||||
return { surface, input };
|
||||
}
|
||||
|
||||
it("is true when focus is inside the surface", () => {
|
||||
const { surface, input } = surfaceWithInput();
|
||||
expect(isTerminalKeyboardOwner(surface, input)).toBe(true);
|
||||
surface.remove();
|
||||
});
|
||||
|
||||
it("is false when focus is in another surface", () => {
|
||||
const first = surfaceWithInput();
|
||||
const second = surfaceWithInput();
|
||||
expect(isTerminalKeyboardOwner(first.surface, second.input)).toBe(false);
|
||||
first.surface.remove();
|
||||
second.surface.remove();
|
||||
});
|
||||
|
||||
it("is false without a surface or a focused element", () => {
|
||||
const { surface, input } = surfaceWithInput();
|
||||
expect(isTerminalKeyboardOwner(surface, null)).toBe(false);
|
||||
expect(isTerminalKeyboardOwner(null, input)).toBe(false);
|
||||
surface.remove();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { DictationStatus } from "@/hooks/use-dictation";
|
||||
|
||||
export type TerminalDictationLifecycleDecision = "keep" | "cancel" | "discard";
|
||||
|
||||
/**
|
||||
* What a terminal's dictation does when its pane stops being presented (another
|
||||
* tab or terminal selected, workspace left, explorer closed) or another terminal
|
||||
* takes the recording. Recording, paused and processing are cancelled, a failed
|
||||
* attempt is discarded. Status `recording` also covers a start still acquiring
|
||||
* the microphone, because the hook sets it before awaiting the source.
|
||||
*/
|
||||
export function resolveTerminalDictationLifecycle(input: {
|
||||
isPresented: boolean;
|
||||
isRecordingOwner: boolean;
|
||||
status: DictationStatus;
|
||||
}): TerminalDictationLifecycleDecision {
|
||||
if (input.status === "idle") {
|
||||
return "keep";
|
||||
}
|
||||
if (input.isPresented && input.isRecordingOwner) {
|
||||
return "keep";
|
||||
}
|
||||
return input.status === "failed" ? "discard" : "cancel";
|
||||
}
|
||||
|
||||
/**
|
||||
* True when DOM keyboard focus is inside this pane's terminal surface: the same
|
||||
* focus that gives a shortcut the `terminal` scope, so exactly one pane owns it.
|
||||
*/
|
||||
export function isTerminalKeyboardOwner(surface: unknown, activeElement: unknown): boolean {
|
||||
if (typeof Element === "undefined") {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
surface instanceof Element &&
|
||||
activeElement instanceof Element &&
|
||||
surface.contains(activeElement)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import { useEffect, useRef, type MutableRefObject } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useToast } from "@/contexts/toast-context";
|
||||
import { useDictation } from "@/hooks/use-dictation";
|
||||
import { useKeyboardActionHandler } from "@/hooks/use-keyboard-action-handler";
|
||||
import { useStableEvent } from "@/hooks/use-stable-event";
|
||||
import { useHostRuntimeClient, useHostRuntimeIsConnected } from "@/runtime/host-runtime";
|
||||
import { useSessionStore } from "@/stores/session-store";
|
||||
import {
|
||||
claimTerminalRecording,
|
||||
clearTerminalDictationState,
|
||||
getTerminalRecordingOwnerId,
|
||||
publishTerminalDictationState,
|
||||
releaseTerminalRecording,
|
||||
useTerminalRecordingOwnerId,
|
||||
type TerminalDictationActions,
|
||||
} from "@/terminal/input-sink";
|
||||
import { formatTerminalTranscript } from "./format-terminal-transcript";
|
||||
import {
|
||||
resolveTerminalDictationAvailability,
|
||||
type TerminalDictationAvailability,
|
||||
} from "./terminal-dictation-availability";
|
||||
import { resolveTerminalDictationLifecycle } from "./terminal-dictation-lifecycle";
|
||||
|
||||
export type TerminalDictationOwnerActions = Omit<TerminalDictationActions, "ownerId">;
|
||||
|
||||
interface TerminalDictationOwnerProps {
|
||||
ownerId: string;
|
||||
serverId: string;
|
||||
terminalId: string;
|
||||
/** The pane's terminal is the one on screen: its tab is selected and its workspace focused. */
|
||||
isPresented: boolean;
|
||||
pasteText: (text: string) => void;
|
||||
focus: () => void;
|
||||
/** Evaluated when the shortcut fires: true while DOM focus is inside this pane's terminal. */
|
||||
isKeyboardOwner: () => boolean;
|
||||
/** Where the owner hands its actions to the pane's input sink. */
|
||||
actionsRef: MutableRefObject<TerminalDictationOwnerActions | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns one terminal pane's dictation. Mounted inside the pane and keyed by its
|
||||
* terminal, so a transcript can only reach the pane that recorded it; the tab-bar
|
||||
* control and the key bar are views of the state it publishes. Renders nothing,
|
||||
* and lives apart from the pane so volume ticks re-render only this component.
|
||||
*
|
||||
* The recording ends, microphone and pending result included, when the pane
|
||||
* unmounts or changes terminal (remount through the key), when the terminal stops
|
||||
* being presented, and when another terminal starts recording.
|
||||
*/
|
||||
export function TerminalDictationOwner({
|
||||
ownerId,
|
||||
serverId,
|
||||
terminalId,
|
||||
isPresented,
|
||||
pasteText,
|
||||
focus,
|
||||
isKeyboardOwner,
|
||||
actionsRef,
|
||||
}: TerminalDictationOwnerProps) {
|
||||
const { t } = useTranslation();
|
||||
const toast = useToast();
|
||||
const client = useHostRuntimeClient(serverId);
|
||||
const isConnected = useHostRuntimeIsConnected(serverId);
|
||||
const serverInfo = useSessionStore((state) => state.sessions[serverId]?.serverInfo ?? null);
|
||||
const recordingOwnerId = useTerminalRecordingOwnerId();
|
||||
const isPresentedRef = useRef(isPresented);
|
||||
isPresentedRef.current = isPresented;
|
||||
|
||||
const handleTranscript = useStableEvent((text: string) => {
|
||||
// A result resolving in the same tick as a cancel: drop it unless this pane is
|
||||
// still on screen and still the recording terminal.
|
||||
if (!isPresentedRef.current || getTerminalRecordingOwnerId() !== ownerId) {
|
||||
return;
|
||||
}
|
||||
const formatted = formatTerminalTranscript(text);
|
||||
if (formatted !== null) {
|
||||
pasteText(formatted);
|
||||
}
|
||||
focus();
|
||||
});
|
||||
const handleError = useStableEvent((error: Error) => {
|
||||
toast.error(error.message);
|
||||
});
|
||||
|
||||
const dictation = useDictation({
|
||||
client,
|
||||
onTranscript: handleTranscript,
|
||||
onError: handleError,
|
||||
enableDuration: true,
|
||||
});
|
||||
const statusRef = useRef(dictation.status);
|
||||
statusRef.current = dictation.status;
|
||||
|
||||
const unavailableMessage = (
|
||||
availability: Exclude<TerminalDictationAvailability, { kind: "ready" }>,
|
||||
) => {
|
||||
if (availability.kind === "disconnected") {
|
||||
return t("workspace.terminal.hostDisconnected");
|
||||
}
|
||||
if (availability.reason === "update-host") {
|
||||
return t("workspace.terminal.dictation.updateHost");
|
||||
}
|
||||
return availability.message ?? t("workspace.terminal.dictation.unavailable");
|
||||
};
|
||||
|
||||
const toggle = useStableEvent(() => {
|
||||
if (!isPresentedRef.current) {
|
||||
return;
|
||||
}
|
||||
const status = statusRef.current;
|
||||
if (status === "recording" || status === "paused") {
|
||||
void dictation.confirmDictation();
|
||||
return;
|
||||
}
|
||||
if (status !== "idle") {
|
||||
return;
|
||||
}
|
||||
const availability = resolveTerminalDictationAvailability({ serverInfo, isConnected });
|
||||
if (availability.kind !== "ready") {
|
||||
toast.error(unavailableMessage(availability));
|
||||
return;
|
||||
}
|
||||
claimTerminalRecording(ownerId);
|
||||
void dictation.startDictation();
|
||||
});
|
||||
const cancel = useStableEvent(() => {
|
||||
void dictation.cancelDictation();
|
||||
focus();
|
||||
});
|
||||
const confirm = useStableEvent(() => {
|
||||
void dictation.confirmDictation();
|
||||
});
|
||||
const retry = useStableEvent(() => {
|
||||
void dictation.retryFailedDictation();
|
||||
});
|
||||
const discard = useStableEvent(() => {
|
||||
dictation.discardFailedDictation();
|
||||
focus();
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
const actions: TerminalDictationOwnerActions = { toggle, cancel, confirm, retry, discard };
|
||||
actionsRef.current = actions;
|
||||
return () => {
|
||||
if (actionsRef.current === actions) {
|
||||
actionsRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [actionsRef, cancel, confirm, discard, retry, toggle]);
|
||||
|
||||
// Only a change of presentation or recording owner ends the recording, so the
|
||||
// status is read at that moment instead of re-running on every status change.
|
||||
const applyLifecycle = useStableEvent(
|
||||
(input: { isPresented: boolean; isRecordingOwner: boolean }) => {
|
||||
const decision = resolveTerminalDictationLifecycle({ ...input, status: statusRef.current });
|
||||
if (decision === "cancel") {
|
||||
void dictation.cancelDictation();
|
||||
} else if (decision === "discard") {
|
||||
dictation.discardFailedDictation();
|
||||
}
|
||||
},
|
||||
);
|
||||
useEffect(() => {
|
||||
applyLifecycle({ isPresented, isRecordingOwner: recordingOwnerId === ownerId });
|
||||
}, [applyLifecycle, isPresented, ownerId, recordingOwnerId]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => releaseTerminalRecording(ownerId);
|
||||
}, [ownerId]);
|
||||
|
||||
const { status, isRecording, isProcessing, error, duration, volume } = dictation;
|
||||
useEffect(() => {
|
||||
publishTerminalDictationState(ownerId, {
|
||||
status,
|
||||
isRecording,
|
||||
isProcessing,
|
||||
error,
|
||||
duration,
|
||||
volume,
|
||||
});
|
||||
}, [duration, error, isProcessing, isRecording, ownerId, status, volume]);
|
||||
useEffect(() => () => clearTerminalDictationState(ownerId), [ownerId]);
|
||||
|
||||
useKeyboardActionHandler({
|
||||
handlerId: `terminal-dictation:${serverId}:${terminalId}:${ownerId}`,
|
||||
actions: ["terminal.dictation-toggle"],
|
||||
enabled: isPresented,
|
||||
priority: 0,
|
||||
isActive: isKeyboardOwner,
|
||||
handle: () => {
|
||||
toggle();
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -4,15 +4,45 @@
|
||||
import { act, renderHook } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
claimTerminalRecording,
|
||||
clearTerminalDictationState,
|
||||
getTerminalInputSink,
|
||||
getTerminalRecordingOwnerId,
|
||||
publishTerminalDictationState,
|
||||
registerTerminalInputSink,
|
||||
releaseTerminalRecording,
|
||||
useRegisterTerminalInputSink,
|
||||
useTerminalDictationState,
|
||||
useTerminalInputSink,
|
||||
type TerminalDictationState,
|
||||
type TerminalInputSink,
|
||||
} from "./index";
|
||||
|
||||
function fakeSink(): TerminalInputSink {
|
||||
return { pasteText: () => {}, deliverFiles: () => {}, focus: () => {} };
|
||||
return {
|
||||
pasteText: () => {},
|
||||
deliverFiles: () => {},
|
||||
focus: () => {},
|
||||
dictation: {
|
||||
ownerId: "owner",
|
||||
toggle: () => {},
|
||||
cancel: () => {},
|
||||
confirm: () => {},
|
||||
retry: () => {},
|
||||
discard: () => {},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function dictationState(volume: number): TerminalDictationState {
|
||||
return {
|
||||
status: "recording",
|
||||
isRecording: true,
|
||||
isProcessing: false,
|
||||
error: null,
|
||||
duration: 0,
|
||||
volume,
|
||||
};
|
||||
}
|
||||
|
||||
const cleanups: Array<() => void> = [];
|
||||
@@ -95,3 +125,47 @@ describe("terminal input sinks", () => {
|
||||
expect(getTerminalInputSink("server-a", "term-1")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("terminal dictation state", () => {
|
||||
afterEach(() => {
|
||||
act(() => {
|
||||
clearTerminalDictationState("owner-a");
|
||||
clearTerminalDictationState("owner-b");
|
||||
releaseTerminalRecording("owner-a");
|
||||
releaseTerminalRecording("owner-b");
|
||||
});
|
||||
});
|
||||
|
||||
it("re-renders an owner's controls only for that owner's state", () => {
|
||||
let renders = 0;
|
||||
const { result } = renderHook(() => {
|
||||
renders += 1;
|
||||
return useTerminalDictationState("owner-a");
|
||||
});
|
||||
expect(result.current).toBeNull();
|
||||
const rendersBefore = renders;
|
||||
|
||||
act(() => publishTerminalDictationState("owner-b", dictationState(0.5)));
|
||||
expect(renders).toBe(rendersBefore);
|
||||
|
||||
const published = dictationState(0.2);
|
||||
act(() => publishTerminalDictationState("owner-a", published));
|
||||
expect(result.current).toBe(published);
|
||||
|
||||
act(() => clearTerminalDictationState("owner-a"));
|
||||
expect(result.current).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps one recording owner and releases it only for that owner", () => {
|
||||
act(() => claimTerminalRecording("owner-a"));
|
||||
expect(getTerminalRecordingOwnerId()).toBe("owner-a");
|
||||
|
||||
act(() => claimTerminalRecording("owner-b"));
|
||||
expect(getTerminalRecordingOwnerId()).toBe("owner-b");
|
||||
|
||||
act(() => releaseTerminalRecording("owner-a"));
|
||||
expect(getTerminalRecordingOwnerId()).toBe("owner-b");
|
||||
act(() => releaseTerminalRecording("owner-b"));
|
||||
expect(getTerminalRecordingOwnerId()).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
import { useEffect } from "react";
|
||||
import { create } from "zustand";
|
||||
import type { SelectedFile } from "@/attachments/selected-file";
|
||||
import type { DictationStatus } from "@/hooks/use-dictation";
|
||||
|
||||
/**
|
||||
* What a mounted terminal pane accepts from controls rendered outside it: the
|
||||
* pane's tab-bar slot and its key bar. Those controls are siblings of the pane
|
||||
* under three different hosts (split pane, fallback row, explorer sidebar), so
|
||||
* they find the pane here by terminal identity instead of through props.
|
||||
* What a mounted terminal pane accepts from its controls. The tab-bar slot is a
|
||||
* sibling of the pane under three different hosts (split pane, fallback row,
|
||||
* explorer sidebar), so it finds the pane here by terminal identity instead of
|
||||
* through props. The key bar renders inside the pane and receives the pane's own
|
||||
* sink as a prop: when two panes show one terminal the registry holds only the
|
||||
* newer one, and each key bar must still act on its own pane.
|
||||
*/
|
||||
export interface TerminalInputSink {
|
||||
/** Pastes text the way the terminal's TUI expects it (bracketed when enabled). */
|
||||
@@ -15,13 +18,44 @@ export interface TerminalInputSink {
|
||||
deliverFiles(files: SelectedFile[]): void;
|
||||
/** Returns keyboard focus to the terminal. */
|
||||
focus(): void;
|
||||
/** Forwards to the pane's dictation owner; does nothing while no owner is mounted. */
|
||||
dictation: TerminalDictationActions;
|
||||
}
|
||||
|
||||
export interface TerminalDictationActions {
|
||||
/** Identifies the pane's owner; its published state is under this id. */
|
||||
ownerId: string;
|
||||
/** Starts dictation, or confirms it while recording. */
|
||||
toggle(): void;
|
||||
cancel(): void;
|
||||
confirm(): void;
|
||||
retry(): void;
|
||||
discard(): void;
|
||||
}
|
||||
|
||||
/** What a dictation owner publishes for its controls. */
|
||||
export interface TerminalDictationState {
|
||||
status: DictationStatus;
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
error: string | null;
|
||||
duration: number;
|
||||
volume: number;
|
||||
}
|
||||
|
||||
interface TerminalInputSinkState {
|
||||
sinks: Record<string, TerminalInputSink>;
|
||||
/** Keyed by owner id, apart from `sinks` so volume ticks never change a sink. */
|
||||
dictationStates: Record<string, TerminalDictationState>;
|
||||
/** The owner whose terminal is recording; one terminal records at a time. */
|
||||
recordingOwnerId: string | null;
|
||||
}
|
||||
|
||||
const useTerminalInputSinkStore = create<TerminalInputSinkState>(() => ({ sinks: {} }));
|
||||
const useTerminalInputSinkStore = create<TerminalInputSinkState>(() => ({
|
||||
sinks: {},
|
||||
dictationStates: {},
|
||||
recordingOwnerId: null,
|
||||
}));
|
||||
|
||||
export function terminalInputSinkKey(serverId: string, terminalId: string): string {
|
||||
return `${serverId}:${terminalId}`;
|
||||
@@ -79,3 +113,47 @@ export function useTerminalInputSink(
|
||||
const key = terminalInputSinkKey(serverId, terminalId);
|
||||
return useTerminalInputSinkStore((state) => state.sinks[key] ?? null);
|
||||
}
|
||||
|
||||
export function publishTerminalDictationState(
|
||||
ownerId: string,
|
||||
dictationState: TerminalDictationState,
|
||||
): void {
|
||||
useTerminalInputSinkStore.setState((state) => ({
|
||||
dictationStates: { ...state.dictationStates, [ownerId]: dictationState },
|
||||
}));
|
||||
}
|
||||
|
||||
export function clearTerminalDictationState(ownerId: string): void {
|
||||
useTerminalInputSinkStore.setState((state) => {
|
||||
if (!(ownerId in state.dictationStates)) {
|
||||
return state;
|
||||
}
|
||||
const { [ownerId]: _removed, ...dictationStates } = state.dictationStates;
|
||||
return { dictationStates };
|
||||
});
|
||||
}
|
||||
|
||||
/** The owner's published state, or null while no owner with this id is mounted. */
|
||||
export function useTerminalDictationState(ownerId: string): TerminalDictationState | null {
|
||||
return useTerminalInputSinkStore((state) => state.dictationStates[ownerId] ?? null);
|
||||
}
|
||||
|
||||
/** Makes `ownerId` the recording terminal; the previous owner cancels itself. */
|
||||
export function claimTerminalRecording(ownerId: string): void {
|
||||
useTerminalInputSinkStore.setState({ recordingOwnerId: ownerId });
|
||||
}
|
||||
|
||||
/** Clears the recording owner only while it is still `ownerId`. */
|
||||
export function releaseTerminalRecording(ownerId: string): void {
|
||||
useTerminalInputSinkStore.setState((state) =>
|
||||
state.recordingOwnerId === ownerId ? { recordingOwnerId: null } : state,
|
||||
);
|
||||
}
|
||||
|
||||
export function getTerminalRecordingOwnerId(): string | null {
|
||||
return useTerminalInputSinkStore.getState().recordingOwnerId;
|
||||
}
|
||||
|
||||
export function useTerminalRecordingOwnerId(): string | null {
|
||||
return useTerminalInputSinkStore((state) => state.recordingOwnerId);
|
||||
}
|
||||
|
||||
@@ -82,12 +82,13 @@ describe("terminal virtual keyboard policy", () => {
|
||||
"terminal-key-esc",
|
||||
"terminal-key-tab",
|
||||
"terminal-key-ctrl",
|
||||
"terminal-key-up",
|
||||
"terminal-key-shift",
|
||||
"terminal-key-up",
|
||||
"terminal-key-alt",
|
||||
"terminal-keyboard-toggle",
|
||||
"terminal-paste",
|
||||
"terminal-upload",
|
||||
"terminal-dictation",
|
||||
"terminal-key-left",
|
||||
"terminal-key-down",
|
||||
"terminal-key-right",
|
||||
@@ -104,14 +105,14 @@ describe("terminal virtual keyboard policy", () => {
|
||||
down: positions.get("terminal-key-down"),
|
||||
right: positions.get("terminal-key-right"),
|
||||
}).toEqual({
|
||||
up: { row: 0, col: 3 },
|
||||
left: { row: 1, col: 2 },
|
||||
down: { row: 1, col: 3 },
|
||||
right: { row: 1, col: 4 },
|
||||
up: { row: 0, col: 4 },
|
||||
left: { row: 1, col: 3 },
|
||||
down: { row: 1, col: 4 },
|
||||
right: { row: 1, col: 5 },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps Paste and Upload next to each other in every arrangement", () => {
|
||||
it("keeps Paste, Upload and Dictation next to each other in every arrangement", () => {
|
||||
for (const rows of [
|
||||
TERMINAL_VIRTUAL_KEYBOARD_ROWS,
|
||||
TERMINAL_VIRTUAL_KEYBOARD_WIDE_ROWS,
|
||||
@@ -120,6 +121,7 @@ describe("terminal virtual keyboard policy", () => {
|
||||
const rowIds = rows.map((row) => row.map(getTerminalVirtualKeyboardControlId));
|
||||
const ids = rowIds.find((row) => row.includes("terminal-paste")) ?? [];
|
||||
expect(ids.indexOf("terminal-upload")).toBe(ids.indexOf("terminal-paste") + 1);
|
||||
expect(ids.indexOf("terminal-dictation")).toBe(ids.indexOf("terminal-paste") + 2);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -21,24 +21,28 @@ export type TerminalVirtualKeyboardControl =
|
||||
| {
|
||||
type: "upload";
|
||||
}
|
||||
| {
|
||||
type: "dictation";
|
||||
}
|
||||
| {
|
||||
type: "keyboardToggle";
|
||||
};
|
||||
|
||||
// Paste and upload stay next to each other in every arrangement.
|
||||
// Paste, upload and dictation stay next to each other in every arrangement.
|
||||
export const TERMINAL_VIRTUAL_KEYBOARD_ROWS = [
|
||||
[
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.esc },
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.tab },
|
||||
{ type: "modifier", modifier: "ctrl" },
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up },
|
||||
{ type: "modifier", modifier: "shift" },
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up },
|
||||
{ type: "modifier", modifier: "alt" },
|
||||
{ type: "keyboardToggle" },
|
||||
],
|
||||
[
|
||||
{ type: "paste" },
|
||||
{ type: "upload" },
|
||||
{ type: "dictation" },
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.left },
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.down },
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.right },
|
||||
@@ -59,6 +63,7 @@ export const TERMINAL_VIRTUAL_KEYBOARD_WIDE_ROWS = [
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.right },
|
||||
{ type: "paste" },
|
||||
{ type: "upload" },
|
||||
{ type: "dictation" },
|
||||
{ type: "keyboardToggle" },
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.enter },
|
||||
],
|
||||
@@ -75,7 +80,7 @@ export const TERMINAL_VIRTUAL_KEYBOARD_THREE_COLUMN_ROWS = [
|
||||
{ type: "modifier", modifier: "shift" },
|
||||
{ type: "keyboardToggle" },
|
||||
],
|
||||
[{ type: "paste" }, { type: "upload" }],
|
||||
[{ type: "paste" }, { type: "upload" }, { type: "dictation" }],
|
||||
[
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.up },
|
||||
{ type: "key", button: TERMINAL_VIRTUAL_KEY_BUTTONS.enter },
|
||||
@@ -121,6 +126,8 @@ export function getTerminalVirtualKeyboardControlId(
|
||||
return "terminal-paste";
|
||||
case "upload":
|
||||
return "terminal-upload";
|
||||
case "dictation":
|
||||
return "terminal-dictation";
|
||||
case "keyboardToggle":
|
||||
return "terminal-keyboard-toggle";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import { Check, Mic, RefreshCcw, X } from "lucide-react-native";
|
||||
import { Text, View } from "react-native";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { StyleSheet, withUnistyles } from "react-native-unistyles";
|
||||
import { LoadingSpinner } from "@/components/ui/loading-spinner";
|
||||
import { ToolbarButton } from "@/components/ui/pane-content-toolbar";
|
||||
import { VolumeMeter } from "@/components/volume-meter";
|
||||
import { useShortcutKeys } from "@/hooks/use-shortcut-keys";
|
||||
import type { Theme } from "@/styles/theme";
|
||||
import { formatDictationDuration } from "@/utils/format-dictation-duration";
|
||||
import type { TerminalDictationState, TerminalInputSink } from "@/terminal/input-sink";
|
||||
|
||||
const ThemedMic = withUnistyles(Mic);
|
||||
const ThemedX = withUnistyles(X);
|
||||
const ThemedCheck = withUnistyles(Check);
|
||||
const ThemedRefresh = withUnistyles(RefreshCcw);
|
||||
const ThemedVolumeMeter = withUnistyles(VolumeMeter);
|
||||
const ThemedSpinner = withUnistyles(LoadingSpinner);
|
||||
const iconColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundExtraMuted });
|
||||
const activeIconColorMapping = (theme: Theme) => ({ color: theme.colors.foreground });
|
||||
const accentIconColorMapping = (theme: Theme) => ({ color: theme.colors.accent });
|
||||
|
||||
interface TerminalDictationControlProps {
|
||||
sink: TerminalInputSink;
|
||||
/** Null while the pane has no dictation owner mounted (host disconnected). */
|
||||
state: TerminalDictationState | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The terminal's mic in its pane's tab bar. Idle it is one button; while the
|
||||
* terminal records, processes or has failed, the same slot becomes a pill with
|
||||
* cancel, meter and duration, and confirm (or retry and discard after a failure).
|
||||
* A view of the pane's dictation owner: every press goes through the sink.
|
||||
*/
|
||||
export function TerminalDictationControl({ sink, state }: TerminalDictationControlProps) {
|
||||
const { t } = useTranslation();
|
||||
const shortcut = useShortcutKeys("terminal-dictation-toggle");
|
||||
const status = state?.status ?? "idle";
|
||||
|
||||
if (!state || status === "idle") {
|
||||
return (
|
||||
<ToolbarButton
|
||||
label={t("workspace.terminal.actions.startDictation")}
|
||||
shortcut={shortcut}
|
||||
testID="terminal-toolbar-dictation"
|
||||
disabled={!state}
|
||||
onPress={sink.dictation.toggle}
|
||||
>
|
||||
<ThemedMic size={14} uniProps={iconColorMapping} />
|
||||
</ToolbarButton>
|
||||
);
|
||||
}
|
||||
|
||||
const isFailed = status === "failed";
|
||||
const isProcessing = state.isProcessing;
|
||||
|
||||
return (
|
||||
<View style={styles.pill} testID="terminal-dictation-pill">
|
||||
{isFailed ? (
|
||||
<ToolbarButton
|
||||
label={t("workspace.terminal.dictation.discard")}
|
||||
testID="terminal-dictation-discard"
|
||||
round
|
||||
onPress={sink.dictation.discard}
|
||||
>
|
||||
<ThemedX size={14} uniProps={activeIconColorMapping} />
|
||||
</ToolbarButton>
|
||||
) : (
|
||||
<ToolbarButton
|
||||
label={t("message.dictation.cancel")}
|
||||
testID="terminal-dictation-cancel"
|
||||
round
|
||||
onPress={sink.dictation.cancel}
|
||||
>
|
||||
<ThemedX size={14} uniProps={activeIconColorMapping} />
|
||||
</ToolbarButton>
|
||||
)}
|
||||
{isFailed ? (
|
||||
<Text style={styles.errorText} numberOfLines={1} testID="terminal-dictation-error">
|
||||
{state.error
|
||||
? t("message.dictation.failed", { error: state.error })
|
||||
: t("message.dictation.failedRetry")}
|
||||
</Text>
|
||||
) : (
|
||||
<View style={styles.meter}>
|
||||
<ThemedVolumeMeter
|
||||
volume={state.volume}
|
||||
orientation="horizontal"
|
||||
variant="compact"
|
||||
uniProps={accentIconColorMapping}
|
||||
/>
|
||||
<Text style={styles.duration}>
|
||||
{status === "paused"
|
||||
? t("message.dictation.paused")
|
||||
: formatDictationDuration(state.duration)}
|
||||
</Text>
|
||||
</View>
|
||||
)}
|
||||
{isProcessing ? (
|
||||
<View style={styles.spinner}>
|
||||
<ThemedSpinner size="small" uniProps={activeIconColorMapping} />
|
||||
</View>
|
||||
) : null}
|
||||
{!isProcessing && isFailed ? (
|
||||
<ToolbarButton
|
||||
label={t("message.dictation.retry")}
|
||||
testID="terminal-dictation-retry"
|
||||
round
|
||||
onPress={sink.dictation.retry}
|
||||
>
|
||||
<ThemedRefresh size={14} uniProps={accentIconColorMapping} />
|
||||
</ToolbarButton>
|
||||
) : null}
|
||||
{!isProcessing && !isFailed ? (
|
||||
<ToolbarButton
|
||||
label={t("workspace.terminal.actions.stopDictation")}
|
||||
shortcut={shortcut}
|
||||
testID="terminal-dictation-confirm"
|
||||
round
|
||||
onPress={sink.dictation.confirm}
|
||||
>
|
||||
<ThemedCheck size={14} uniProps={accentIconColorMapping} />
|
||||
</ToolbarButton>
|
||||
) : null}
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create((theme) => ({
|
||||
pill: {
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
gap: theme.spacing[1],
|
||||
paddingHorizontal: theme.spacing[1],
|
||||
borderRadius: theme.borderRadius.full,
|
||||
borderWidth: 1,
|
||||
borderColor: theme.colors.accent,
|
||||
backgroundColor: theme.colors.surface1,
|
||||
},
|
||||
meter: {
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
gap: theme.spacing[1],
|
||||
},
|
||||
duration: {
|
||||
color: theme.colors.foreground,
|
||||
fontSize: theme.fontSize.sm,
|
||||
fontVariant: ["tabular-nums"],
|
||||
},
|
||||
errorText: {
|
||||
maxWidth: 160,
|
||||
color: theme.colors.destructive,
|
||||
fontSize: theme.fontSize.sm,
|
||||
},
|
||||
spinner: {
|
||||
width: 20,
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
},
|
||||
}));
|
||||
@@ -0,0 +1,66 @@
|
||||
import { useMemo, type ReactNode } from "react";
|
||||
import { Mic } from "lucide-react-native";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { withUnistyles } from "react-native-unistyles";
|
||||
import { DictationOverlay } from "@/components/dictation-controls";
|
||||
import { TerminalKeyBarIconAction } from "@/components/terminal-copy-paste-actions";
|
||||
import type { Theme } from "@/styles/theme";
|
||||
import { useTerminalDictationState, type TerminalInputSink } from "@/terminal/input-sink";
|
||||
|
||||
const ThemedMic = withUnistyles(Mic);
|
||||
const keyBarIconColorMapping = (theme: Theme) => ({ color: theme.colors.foregroundMuted });
|
||||
|
||||
interface TerminalKeyBarDictationProps {
|
||||
/** The pane's own sink: a key bar acts on its pane, never on a newer pane of the same terminal. */
|
||||
sink: TerminalInputSink;
|
||||
}
|
||||
|
||||
/** The key-bar mic. Disabled while the pane has no dictation owner (host disconnected). */
|
||||
export function TerminalKeyBarDictationButton({ sink }: TerminalKeyBarDictationProps) {
|
||||
const { t } = useTranslation();
|
||||
const state = useTerminalDictationState(sink.dictation.ownerId);
|
||||
const icon = useMemo(() => <ThemedMic size={16} uniProps={keyBarIconColorMapping} />, []);
|
||||
return (
|
||||
<TerminalKeyBarIconAction
|
||||
accessibilityLabel={t("workspace.terminal.actions.startDictation")}
|
||||
testID="terminal-dictation"
|
||||
icon={icon}
|
||||
disabled={!state}
|
||||
onPress={sink.dictation.toggle}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shows the key bar's rows while the pane's terminal is not dictating, and the
|
||||
* dictation overlay in their place through recording, paused, processing and
|
||||
* failed. The rows come back once the attempt is delivered, cancelled or discarded.
|
||||
* Lives apart from the pane so volume ticks re-render only this component.
|
||||
*/
|
||||
export function TerminalKeyBarDictationGate({
|
||||
sink,
|
||||
children,
|
||||
}: TerminalKeyBarDictationProps & { children: ReactNode }) {
|
||||
const state = useTerminalDictationState(sink.dictation.ownerId);
|
||||
if (!state || state.status === "idle") {
|
||||
return children;
|
||||
}
|
||||
// Also recording while the microphone is still being acquired, so cancel is reachable then.
|
||||
const isRecording =
|
||||
state.isRecording || state.status === "recording" || state.status === "paused";
|
||||
return (
|
||||
<DictationOverlay
|
||||
testID="terminal-dictation-overlay"
|
||||
volume={state.volume}
|
||||
duration={state.duration}
|
||||
isRecording={isRecording}
|
||||
isProcessing={state.isProcessing}
|
||||
status={state.status}
|
||||
errorText={state.status === "failed" ? (state.error ?? undefined) : undefined}
|
||||
onCancel={sink.dictation.cancel}
|
||||
onAccept={sink.dictation.confirm}
|
||||
onRetry={sink.dictation.retry}
|
||||
onDiscard={sink.dictation.discard}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useTerminalInputSink } from "@/terminal/input-sink";
|
||||
import { useTerminalDictationState, useTerminalInputSink } from "@/terminal/input-sink";
|
||||
import { TerminalDictationControl } from "./terminal-dictation-control";
|
||||
import { TerminalUploadButton } from "./terminal-upload-button";
|
||||
|
||||
interface TerminalToolbarSlotProps {
|
||||
@@ -8,13 +9,28 @@ interface TerminalToolbarSlotProps {
|
||||
|
||||
/**
|
||||
* The terminal controls in a pane's tab bar, shown while the pane's active tab is
|
||||
* a terminal. A view over the mounted pane's input sink: it owns no terminal
|
||||
* state and renders nothing until the pane has registered.
|
||||
* a terminal: upload, then the mic. A view over the mounted pane's input sink and
|
||||
* its dictation state: it owns no terminal state and renders nothing until the
|
||||
* pane has registered.
|
||||
*/
|
||||
export function TerminalToolbarSlot({ serverId, terminalId }: TerminalToolbarSlotProps) {
|
||||
const sink = useTerminalInputSink(serverId, terminalId);
|
||||
if (!sink) {
|
||||
return null;
|
||||
}
|
||||
return <TerminalUploadButton sink={sink} placement="toolbar" />;
|
||||
return <TerminalToolbarSlotControls sink={sink} />;
|
||||
}
|
||||
|
||||
function TerminalToolbarSlotControls({
|
||||
sink,
|
||||
}: {
|
||||
sink: NonNullable<ReturnType<typeof useTerminalInputSink>>;
|
||||
}) {
|
||||
const state = useTerminalDictationState(sink.dictation.ownerId);
|
||||
return (
|
||||
<>
|
||||
<TerminalUploadButton sink={sink} placement="toolbar" />
|
||||
<TerminalDictationControl sink={sink} state={state} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
/** Formats a dictation's elapsed seconds as `mm:ss`. */
|
||||
export function formatDictationDuration(seconds: number): string {
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = seconds % 60;
|
||||
return `${mins.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
|
||||
}
|
||||
Reference in New Issue
Block a user