From 0132f9f3e309e44310d1640118d717ecb121d46a Mon Sep 17 00:00:00 2001 From: NightMachinery Date: Fri, 30 Oct 2026 23:38:01 +0100 Subject: [PATCH] fix(app): explain empty dictation results and retain recordings Keep empty transcription results visible and preserve captured audio for retry. --- .../browser/dictation-empty-result.spec.ts | 64 ++++++++ .../new-workspace-dictation-submit.spec.ts | 70 +------- packages/app/e2e/support/helpers/dictation.ts | 131 +++++++++++++++ .../app/src/components/dictation-controls.tsx | 47 +++++- packages/app/src/composer/input/input.tsx | 18 ++- .../dictation/empty-transcript-reason.test.ts | 50 ++++++ .../src/dictation/empty-transcript-reason.ts | 24 +++ packages/app/src/dictation/pcm-level.ts | 24 +++ .../app/src/hooks/use-dictation.shared.ts | 11 +- packages/app/src/hooks/use-dictation.test.tsx | 151 ++++++++++++++++++ packages/app/src/hooks/use-dictation.ts | 23 ++- packages/app/src/i18n/resources/ar.ts | 5 + packages/app/src/i18n/resources/en.ts | 5 + packages/app/src/i18n/resources/es.ts | 5 + packages/app/src/i18n/resources/fr.ts | 5 + packages/app/src/i18n/resources/ja.ts | 5 + packages/app/src/i18n/resources/ko.ts | 5 + packages/app/src/i18n/resources/pt-BR.ts | 5 + packages/app/src/i18n/resources/ru.ts | 5 + packages/app/src/i18n/resources/zh-CN.ts | 5 + 20 files changed, 582 insertions(+), 76 deletions(-) create mode 100644 packages/app/e2e/browser/dictation-empty-result.spec.ts create mode 100644 packages/app/e2e/support/helpers/dictation.ts create mode 100644 packages/app/src/dictation/empty-transcript-reason.test.ts create mode 100644 packages/app/src/dictation/empty-transcript-reason.ts create mode 100644 packages/app/src/dictation/pcm-level.ts create mode 100644 packages/app/src/hooks/use-dictation.test.tsx diff --git a/packages/app/e2e/browser/dictation-empty-result.spec.ts b/packages/app/e2e/browser/dictation-empty-result.spec.ts new file mode 100644 index 0000000000..5c868feb3b --- /dev/null +++ b/packages/app/e2e/browser/dictation-empty-result.spec.ts @@ -0,0 +1,64 @@ +import { expect, test } from "../support/fixtures"; +import { composerLocator, expectComposerVisible } from "../support/helpers/composer"; +import { + installDictationTranscriptHarness, + installSyntheticMicrophone, +} from "../support/helpers/dictation"; +import { openAgentRoute, seedMockAgentWorkspace } from "../support/helpers/mock-agent"; + +const TRANSCRIPT = "Recovered on the second attempt"; + +test.describe("Dictation empty result", () => { + test.describe.configure({ timeout: 240_000 }); + + test("explains an empty transcript and retries the same recording", async ({ page }) => { + const agent = await seedMockAgentWorkspace({ + repoPrefix: "dictation-empty-", + title: "Empty dictation", + }); + await installSyntheticMicrophone(page); + const harness = await installDictationTranscriptHarness(page, ["", TRANSCRIPT]); + try { + await openAgentRoute(page, agent); + await expectComposerVisible(page); + + await page.getByRole("button", { name: "Start dictation" }).click(); + await harness.waitForAudio(); + await page.getByRole("button", { name: "Insert transcription", exact: true }).click(); + + await expect( + page.getByText("No speech was recognized. Tap retry to try again.", { exact: true }), + ).toBeVisible(); + await page.getByRole("button", { name: "Retry dictation", exact: true }).click(); + + await expect(composerLocator(page)).toHaveValue(TRANSCRIPT); + } finally { + await agent.cleanup(); + } + }); + + test("says when the microphone only delivered silence", async ({ page }) => { + const agent = await seedMockAgentWorkspace({ + repoPrefix: "dictation-silent-", + title: "Silent dictation", + }); + await installSyntheticMicrophone(page, { silent: true }); + const harness = await installDictationTranscriptHarness(page, [""]); + try { + await openAgentRoute(page, agent); + await expectComposerVisible(page); + + await page.getByRole("button", { name: "Start dictation" }).click(); + await harness.waitForAudio(); + await page.getByRole("button", { name: "Insert transcription", exact: true }).click(); + + await expect( + page.getByText("Only silence was recorded. Check your microphone.", { exact: true }), + ).toBeVisible(); + await page.getByRole("button", { name: "Cancel dictation", exact: true }).click(); + await expect(composerLocator(page)).toBeEditable(); + } finally { + await agent.cleanup(); + } + }); +}); diff --git a/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts b/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts index 22b2b8a6c5..b5ac8f6a18 100644 --- a/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts +++ b/packages/app/e2e/browser/new-workspace-dictation-submit.spec.ts @@ -16,74 +16,16 @@ import { expectComposerFullscreenControls, } from "../support/helpers/composer"; import { waitForSidebarHydration } from "../support/helpers/workspace-ui"; - -type WebSocketMessage = string | Buffer; +import { + enableDictationCapability, + installSyntheticMicrophone, + sendSessionMessage, + sessionMessage, +} from "../support/helpers/dictation"; const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace"; const CREATE_FAILURE = "Synthetic workspace creation failure"; -function parseEnvelope(message: WebSocketMessage): { - type?: unknown; - message?: Record; -} | null { - const raw = typeof message === "string" ? message : message.toString("utf8"); - try { - return JSON.parse(raw) as { type?: unknown; message?: Record }; - } catch { - return null; - } -} - -function sessionMessage(message: WebSocketMessage): Record | null { - const envelope = parseEnvelope(message); - return envelope?.type === "session" && envelope.message ? envelope.message : null; -} - -function sendSessionMessage( - ws: { send(message: string): void }, - message: Record, -): void { - ws.send(JSON.stringify({ type: "session", message })); -} - -function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { - const envelope = parseEnvelope(message); - const payload = envelope?.message?.payload; - if ( - envelope?.message?.type !== "status" || - !payload || - typeof payload !== "object" || - (payload as { status?: unknown }).status !== "server_info" - ) { - return message; - } - - (payload as Record).capabilities = { - voice: { - dictation: { enabled: true, reason: "" }, - voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, - }, - }; - return JSON.stringify(envelope); -} - -async function installSyntheticMicrophone(page: Page): Promise { - await page.addInitScript(() => { - const mediaDevices = navigator.mediaDevices; - Object.defineProperty(mediaDevices, "getUserMedia", { - configurable: true, - value: async () => { - const context = new AudioContext(); - const oscillator = context.createOscillator(); - const destination = context.createMediaStreamDestination(); - oscillator.connect(destination); - oscillator.start(); - return destination.stream; - }, - }); - }); -} - async function installDictationFailureHarness(page: Page) { let resolveAudioChunk!: () => void; let resolveCreateRequest!: () => void; diff --git a/packages/app/e2e/support/helpers/dictation.ts b/packages/app/e2e/support/helpers/dictation.ts new file mode 100644 index 0000000000..68395e574c --- /dev/null +++ b/packages/app/e2e/support/helpers/dictation.ts @@ -0,0 +1,131 @@ +import type { Page } from "../fixtures"; +import { daemonWsRoutePattern } from "./daemon-port"; + +type WebSocketMessage = string | Buffer; + +function parseEnvelope(message: WebSocketMessage): { + type?: unknown; + message?: Record; +} | null { + const raw = typeof message === "string" ? message : message.toString("utf8"); + try { + return JSON.parse(raw) as { type?: unknown; message?: Record }; + } catch { + return null; + } +} + +export function sessionMessage(message: WebSocketMessage): Record | null { + const envelope = parseEnvelope(message); + return envelope?.type === "session" && envelope.message ? envelope.message : null; +} + +export function sendSessionMessage( + ws: { send(message: string): void }, + message: Record, +): void { + ws.send(JSON.stringify({ type: "session", message })); +} + +export function enableDictationCapability(message: WebSocketMessage): WebSocketMessage { + const envelope = parseEnvelope(message); + const payload = envelope?.message?.payload; + if ( + envelope?.message?.type !== "status" || + !payload || + typeof payload !== "object" || + (payload as { status?: unknown }).status !== "server_info" + ) { + return message; + } + (payload as Record).capabilities = { + voice: { + dictation: { enabled: true, reason: "" }, + voice: { enabled: false, reason: "Realtime voice is disabled in this test." }, + }, + }; + return JSON.stringify(envelope); +} + +/** + * Replaces the browser microphone with an oscillator. `silent` keeps the stream + * running at zero gain, which is what a muted or wrongly routed input delivers. + */ +export async function installSyntheticMicrophone( + page: Page, + options: { silent?: boolean } = {}, +): Promise { + await page.addInitScript((silent: boolean) => { + const mediaDevices = navigator.mediaDevices; + Object.defineProperty(mediaDevices, "getUserMedia", { + configurable: true, + value: async () => { + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const gain = context.createGain(); + gain.gain.value = silent ? 0 : 1; + const destination = context.createMediaStreamDestination(); + oscillator.connect(gain); + gain.connect(destination); + oscillator.start(); + return destination.stream; + }, + }); + }, options.silent ?? false); +} + +/** + * Answers dictation stream requests in place of the daemon. Each finish is + * answered with the next queued transcript; everything else reaches the daemon. + */ +export async function installDictationTranscriptHarness(page: Page, transcripts: string[]) { + const queue = [...transcripts]; + let resolveAudioChunk!: () => void; + const audioChunk = new Promise((resolve) => { + resolveAudioChunk = resolve; + }); + + await page.routeWebSocket(daemonWsRoutePattern(), (ws) => { + const server = ws.connectToServer(); + + ws.onMessage((message) => { + const request = sessionMessage(message); + const type = request?.type; + const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null; + + if (type === "dictation_stream_start" && dictationId) { + sendSessionMessage(ws, { + type: "dictation_stream_ack", + payload: { dictationId, ackSeq: -1 }, + }); + return; + } + if (type === "dictation_stream_chunk" && dictationId) { + const seq = typeof request?.seq === "number" ? request.seq : 0; + sendSessionMessage(ws, { + type: "dictation_stream_ack", + payload: { dictationId, ackSeq: seq }, + }); + resolveAudioChunk(); + return; + } + if (type === "dictation_stream_finish" && dictationId) { + sendSessionMessage(ws, { + type: "dictation_stream_finish_accepted", + payload: { dictationId, timeoutMs: 5_000 }, + }); + sendSessionMessage(ws, { + type: "dictation_stream_final", + payload: { dictationId, text: queue.shift() ?? "" }, + }); + return; + } + + server.send(message); + }); + + server.onMessage((message) => ws.send(enableDictationCapability(message))); + }); + + return { waitForAudio: () => audioChunk }; +} diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index 2191f0d480..95e75497d4 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -4,9 +4,11 @@ import { View, Text, Pressable } from "react-native"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { X, ArrowUp, RefreshCcw, Check, Mic, Pencil } from "lucide-react-native"; import { useTranslation } from "react-i18next"; +import type { TFunction } from "i18next"; import { VolumeMeter } from "./volume-meter"; import { FOOTER_HEIGHT } from "@/constants/layout"; import type { DictationStatus } from "@/hooks/use-dictation"; +import type { DictationEmptyReason } from "@/dictation/empty-transcript-reason"; interface DictationControlsProps { volume: number; @@ -24,6 +26,31 @@ interface DictationControlsProps { disabled?: boolean; } +const EMPTY_REASON_MESSAGE_KEYS = { + "no-audio": "message.dictation.empty.noAudio", + "silent-audio": "message.dictation.empty.silentAudio", + "no-speech": "message.dictation.empty.noSpeech", +} as const satisfies Record; + +// The line under the meter once an attempt has finished without text. +function resolveResultText(input: { + t: TFunction; + status: DictationStatus; + errorText?: string; + emptyReason?: DictationEmptyReason | null; +}): string | null { + const { t, status, errorText, emptyReason } = input; + if (status === "empty") { + return t(EMPTY_REASON_MESSAGE_KEYS[emptyReason ?? "no-speech"]); + } + if (status !== "failed") { + return null; + } + return errorText + ? t("message.dictation.failed", { error: errorText }) + : t("message.dictation.failedRetry"); +} + function formatDuration(seconds: number): string { const mins = Math.floor(seconds / 60); const secs = seconds % 60; @@ -147,18 +174,26 @@ export function DictationOverlay({ isProcessing, status, errorText, + emptyReason, onCancel, onAccept, onAcceptAndSend, onRetry, onDiscard, -}: Omit & { errorText?: string }) { +}: Omit & { + errorText?: string; + emptyReason?: DictationEmptyReason | null; +}) { const { theme } = useUnistyles(); const { t } = useTranslation(); - const isFailed = status === "failed"; + const isEmpty = status === "empty"; + // An empty result is a finished attempt the user has to acknowledge, like a failure. + const isFailed = status === "failed" || isEmpty; + const canRetry = !isEmpty || emptyReason !== "no-audio"; const showActiveState = isRecording || isProcessing || isFailed; const actionsDisabled = isProcessing; const handleCancel = isFailed && onDiscard ? onDiscard : onCancel; + const resultText = resolveResultText({ t, status, errorText, emptyReason }); const containerStyle = useMemo( () => [overlayStyles.container, { backgroundColor: theme.colors.accent }], @@ -217,11 +252,9 @@ export function DictationOverlay({ {t("message.dictation.paused")} ) : null} - {isFailed ? ( + {resultText ? ( - {errorText - ? t("message.dictation.failed", { error: errorText }) - : t("message.dictation.failedRetry")} + {resultText} ) : null} @@ -232,7 +265,7 @@ export function DictationOverlay({ ) : null} - {!actionsDisabled && isFailed ? ( + {!actionsDisabled && isFailed && canRetry ? ( ["status"]; dictationError: string | null; + dictationEmptyReason: React.ComponentProps["emptyReason"]; onCancelRecording: () => Promise; onAcceptRecording: () => Promise; onAcceptAndSendRecording: () => Promise; @@ -593,6 +595,7 @@ function MessageInputOverlay({ onRealtimeVoiceStop: () => void; }) { if (showDictationOverlay) { + const awaitsAcknowledgement = dictationStatus === "failed" || dictationStatus === "empty"; return ( ); } @@ -1032,7 +1036,12 @@ function computeShouldShowDictationOverlay( isDictationProcessing: boolean, dictationStatus: string, ): boolean { - return isDictating || isDictationProcessing || dictationStatus === "failed"; + return ( + isDictating || + isDictationProcessing || + dictationStatus === "failed" || + dictationStatus === "empty" + ); } function computeIsDictationStartEnabled( @@ -1533,6 +1542,7 @@ export const MessageInput = forwardRef( duration: dictationDuration, error: dictationError, status: dictationStatus, + emptyReason: dictationEmptyReason, startDictation, cancelDictation, confirmDictation, @@ -2180,6 +2190,7 @@ export const MessageInput = forwardRef( isDictationProcessing={isDictationProcessing} dictationStatus={dictationStatus} dictationError={dictationError} + dictationEmptyReason={dictationEmptyReason} onCancelRecording={handleCancelRecording} onAcceptRecording={handleAcceptRecording} onAcceptAndSendRecording={handleAcceptAndSendRecording} @@ -2201,6 +2212,7 @@ export const MessageInput = forwardRef( isDictationProcessing, dictationStatus, dictationError, + dictationEmptyReason, handleCancelRecording, handleAcceptRecording, handleAcceptAndSendRecording, diff --git a/packages/app/src/dictation/empty-transcript-reason.test.ts b/packages/app/src/dictation/empty-transcript-reason.test.ts new file mode 100644 index 0000000000..f316d27daf --- /dev/null +++ b/packages/app/src/dictation/empty-transcript-reason.test.ts @@ -0,0 +1,50 @@ +import { Buffer } from "buffer"; +import { describe, expect, it } from "vitest"; +import { resolveDictationEmptyReason } from "@/dictation/empty-transcript-reason"; +import { + DICTATION_SILENCE_PEAK_THRESHOLD, + isAudiblePeak, + pcm16PeakFromBase64, +} from "@/dictation/pcm-level"; + +function pcm16Base64(samples: number[]): string { + const bytes = Buffer.alloc(samples.length * 2); + samples.forEach((sample, index) => bytes.writeInt16LE(sample, index * 2)); + return bytes.toString("base64"); +} + +describe("pcm16PeakFromBase64", () => { + it("returns the largest absolute sample, including negative ones", () => { + expect(pcm16PeakFromBase64(pcm16Base64([0, 120, -4000, 2500]))).toBe(4000); + }); + + it("reads digital silence as zero", () => { + expect(pcm16PeakFromBase64(pcm16Base64([0, 0, 0, 0]))).toBe(0); + }); + + it("ignores a trailing odd byte", () => { + const bytes = Buffer.concat([Buffer.from(pcm16Base64([-700]), "base64"), Buffer.from([0x7f])]); + expect(pcm16PeakFromBase64(bytes.toString("base64"))).toBe(700); + }); +}); + +describe("isAudiblePeak", () => { + it("uses the daemon's default silence threshold", () => { + expect(isAudiblePeak(DICTATION_SILENCE_PEAK_THRESHOLD - 1)).toBe(false); + expect(isAudiblePeak(DICTATION_SILENCE_PEAK_THRESHOLD)).toBe(true); + }); +}); + +describe("resolveDictationEmptyReason", () => { + it("reports missing audio when the microphone delivered nothing", () => { + expect(resolveDictationEmptyReason({ segmentCount: 0, peak: 0 })).toBe("no-audio"); + }); + + it("reports silent audio when nothing crossed the silence threshold", () => { + expect(resolveDictationEmptyReason({ segmentCount: 40, peak: 120 })).toBe("silent-audio"); + }); + + it("reports no recognized speech when audible audio produced no text", () => { + expect(resolveDictationEmptyReason({ segmentCount: 40, peak: 9000 })).toBe("no-speech"); + }); +}); diff --git a/packages/app/src/dictation/empty-transcript-reason.ts b/packages/app/src/dictation/empty-transcript-reason.ts new file mode 100644 index 0000000000..98e7633292 --- /dev/null +++ b/packages/app/src/dictation/empty-transcript-reason.ts @@ -0,0 +1,24 @@ +import { isAudiblePeak } from "./pcm-level"; + +/** + * Why a finished dictation produced no text. + * + * - `no-audio`: the microphone delivered no audio at all. + * - `silent-audio`: audio arrived, but it never rose above the daemon's silence + * threshold, so the daemon dropped it before speech-to-text. + * - `no-speech`: audible audio reached speech-to-text, which returned no words. + */ +export type DictationEmptyReason = "no-audio" | "silent-audio" | "no-speech"; + +export function resolveDictationEmptyReason(input: { + segmentCount: number; + peak: number; +}): DictationEmptyReason { + if (input.segmentCount === 0) { + return "no-audio"; + } + if (!isAudiblePeak(input.peak)) { + return "silent-audio"; + } + return "no-speech"; +} diff --git a/packages/app/src/dictation/pcm-level.ts b/packages/app/src/dictation/pcm-level.ts new file mode 100644 index 0000000000..245585155f --- /dev/null +++ b/packages/app/src/dictation/pcm-level.ts @@ -0,0 +1,24 @@ +import { Buffer } from "buffer"; + +// Mirrors the daemon's default `PASEO_DICTATION_SILENCE_PEAK_THRESHOLD` +// (packages/server/src/server/dictation/dictation-stream-manager.ts). Audio whose +// peak stays below it is cleared by the daemon and never reaches speech-to-text. +// The `volume` level from the audio sources can't be used for this: native reports +// a dB curve and web a linear RMS, so the same input reads differently per platform. +export const DICTATION_SILENCE_PEAK_THRESHOLD = 300; + +export function pcm16PeakFromBase64(base64Pcm: string): number { + const bytes = Buffer.from(base64Pcm, "base64"); + let peak = 0; + for (let offset = 0; offset + 1 < bytes.length; offset += 2) { + const sample = Math.abs(bytes.readInt16LE(offset)); + if (sample > peak) { + peak = sample; + } + } + return peak; +} + +export function isAudiblePeak(peak: number): boolean { + return peak >= DICTATION_SILENCE_PEAK_THRESHOLD; +} diff --git a/packages/app/src/hooks/use-dictation.shared.ts b/packages/app/src/hooks/use-dictation.shared.ts index fee3da46c1..d2b6d3748b 100644 --- a/packages/app/src/hooks/use-dictation.shared.ts +++ b/packages/app/src/hooks/use-dictation.shared.ts @@ -1,6 +1,14 @@ +import type { DictationEmptyReason } from "@/dictation/empty-transcript-reason"; import { i18n } from "@/i18n/i18next"; -export type DictationStatus = "idle" | "recording" | "paused" | "uploading" | "failed"; +export type DictationStatus = + | "idle" + | "recording" + | "paused" + | "uploading" + | "failed" + // Finished, but produced no text. `emptyReason` says why; the audio is kept for retry. + | "empty"; export interface UseDictationOptions { client: import("@getpaseo/client/internal/daemon-client").DaemonClient | null; @@ -22,6 +30,7 @@ export interface UseDictationResult { duration: number; error: string | null; status: DictationStatus; + emptyReason: DictationEmptyReason | null; startDictation: () => Promise; cancelDictation: () => Promise; confirmDictation: () => Promise; diff --git a/packages/app/src/hooks/use-dictation.test.tsx b/packages/app/src/hooks/use-dictation.test.tsx new file mode 100644 index 0000000000..0b0ce158ad --- /dev/null +++ b/packages/app/src/hooks/use-dictation.test.tsx @@ -0,0 +1,151 @@ +/** @vitest-environment jsdom */ +import { Buffer } from "buffer"; +import { act, renderHook } from "@testing-library/react"; +import type { SessionOutboundMessage } from "@getpaseo/protocol/messages"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { DictationAudioSourceConfig } from "./use-dictation-audio-source.types"; +import { useDictation, type UseDictationOptions } from "./use-dictation"; + +const audioSource = vi.hoisted(() => ({ + config: null as DictationAudioSourceConfig | null, +})); + +vi.mock("@/hooks/use-dictation-audio-source", () => ({ + useDictationAudioSource: (config: DictationAudioSourceConfig) => { + audioSource.config = config; + return { + start: async () => undefined, + stop: async () => undefined, + volume: 0, + isPaused: false, + }; + }, +})); + +function pcm16Base64(samples: number[]): string { + const bytes = Buffer.alloc(samples.length * 2); + samples.forEach((sample, index) => bytes.writeInt16LE(sample, index * 2)); + return bytes.toString("base64"); +} + +class FakeDictationClient { + isConnected = true; + transcript = ""; + finishes: Array<{ dictationId: string; finalSeq: number }> = []; + private readonly rawListeners = new Set<(message: SessionOutboundMessage) => void>(); + + async startDictationStream(): Promise {} + + sendDictationStreamChunk(dictationId: string, seq: number): void { + const ack = { type: "dictation_stream_ack", payload: { dictationId, ackSeq: seq } }; + for (const listener of this.rawListeners) { + listener(ack as SessionOutboundMessage); + } + } + + async finishDictationStream(dictationId: string, finalSeq: number) { + this.finishes.push({ dictationId, finalSeq }); + return { dictationId, text: this.transcript }; + } + + cancelDictationStream(): void {} + + subscribeRawMessages(handler: (message: SessionOutboundMessage) => void): () => void { + this.rawListeners.add(handler); + return () => this.rawListeners.delete(handler); + } + + subscribeConnectionStatus(): () => void { + return () => undefined; + } + + on(): () => void { + return () => undefined; + } +} + +function renderDictation(client: FakeDictationClient) { + const onTranscript = vi.fn(); + const view = renderHook(() => + useDictation({ + client: client as unknown as UseDictationOptions["client"], + onTranscript, + }), + ); + return { view, onTranscript }; +} + +async function record(view: ReturnType["view"], segments: string[]) { + await act(async () => { + await view.result.current.startDictation(); + }); + await act(async () => { + for (const segment of segments) { + audioSource.config?.onPcmSegment(segment); + } + }); + await act(async () => { + await view.result.current.confirmDictation(); + }); +} + +describe("useDictation empty results", () => { + beforeEach(() => { + audioSource.config = null; + }); + + it("explains a silent recording and keeps its audio for retry", async () => { + const client = new FakeDictationClient(); + const { view, onTranscript } = renderDictation(client); + + await record(view, [pcm16Base64([0, 40, -60]), pcm16Base64([10, -20, 0])]); + + expect(view.result.current.status).toBe("empty"); + expect(view.result.current.emptyReason).toBe("silent-audio"); + expect(onTranscript).not.toHaveBeenCalled(); + + client.transcript = "hello there"; + await act(async () => { + await view.result.current.retryFailedDictation(); + }); + + expect(client.finishes.map((finish) => finish.finalSeq)).toEqual([1, 1]); + expect(onTranscript).toHaveBeenCalledWith("hello there", expect.anything()); + expect(view.result.current.status).toBe("idle"); + expect(view.result.current.emptyReason).toBeNull(); + }); + + it("says when audible speech produced no text", async () => { + const client = new FakeDictationClient(); + const { view } = renderDictation(client); + + await record(view, [pcm16Base64([0, 9000, -12000])]); + + expect(view.result.current.status).toBe("empty"); + expect(view.result.current.emptyReason).toBe("no-speech"); + }); + + it("says when the microphone delivered no audio", async () => { + const client = new FakeDictationClient(); + const { view } = renderDictation(client); + + await record(view, []); + + expect(view.result.current.status).toBe("empty"); + expect(view.result.current.emptyReason).toBe("no-audio"); + expect(client.finishes).toEqual([]); + }); + + it("clears the empty result when dismissed", async () => { + const client = new FakeDictationClient(); + const { view } = renderDictation(client); + + await record(view, [pcm16Base64([5, 5])]); + act(() => { + view.result.current.discardFailedDictation(); + }); + + expect(view.result.current.status).toBe("idle"); + expect(view.result.current.emptyReason).toBeNull(); + }); +}); diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index a2a3f46bcc..c1c27e575d 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -2,6 +2,11 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { DictationStreamSender } from "@/dictation/dictation-stream-sender"; +import { + resolveDictationEmptyReason, + type DictationEmptyReason, +} from "@/dictation/empty-transcript-reason"; +import { pcm16PeakFromBase64 } from "@/dictation/pcm-level"; import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source"; import { generateMessageId } from "@/types/stream"; import { AttemptGuard } from "@/utils/attempt-guard"; @@ -33,7 +38,10 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const [duration, setDuration] = useState(0); const [error, setError] = useState(null); const [status, setStatus] = useState("idle"); + const [emptyReason, setEmptyReason] = useState(null); const latestPartialTranscriptRef = useRef(""); + // Loudest sample captured in the current recording; explains an empty transcript. + const recordingPeakRef = useRef(0); const onTranscriptRef = useRef(onTranscript); useEffect(() => { @@ -133,8 +141,10 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const clearStreamingState = useCallback(() => { senderRef.current?.clearAll(); + recordingPeakRef.current = 0; latestPartialTranscriptRef.current = ""; setPartialTranscript(""); + setEmptyReason(null); }, []); const startNewStream = useCallback(async (reason: string) => { @@ -194,11 +204,19 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const transcriptText = text.trim().length > 0 ? text.trim() : latestPartialTranscriptRef.current.trim(); - clearStreamingState(); if (!transcriptText) { + // Keep the audio so the user can retry, and say why nothing came back. + setStatus("empty"); + setEmptyReason( + resolveDictationEmptyReason({ + segmentCount: senderRef.current?.getSegmentCount() ?? 0, + peak: recordingPeakRef.current, + }), + ); return; } + clearStreamingState(); onTranscriptRef.current?.(transcriptText, { requestId }); }, [clearStreamingState], @@ -228,6 +246,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const audio = useDictationAudioSource({ onPcmSegment: (audioData) => { + recordingPeakRef.current = Math.max(recordingPeakRef.current, pcm16PeakFromBase64(audioData)); senderRef.current?.enqueueSegment(audioData); }, onError: (err) => { @@ -395,6 +414,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { return; } setError(null); + setEmptyReason(null); setStatus("uploading"); setIsProcessing(true); isProcessingRef.current = true; @@ -462,6 +482,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { duration, error, status: status === "recording" && audio.isPaused ? "paused" : status, + emptyReason, startDictation, cancelDictation, confirmDictation, diff --git a/packages/app/src/i18n/resources/ar.ts b/packages/app/src/i18n/resources/ar.ts index 8d7c2a2950..91d09ee263 100644 --- a/packages/app/src/i18n/resources/ar.ts +++ b/packages/app/src/i18n/resources/ar.ts @@ -371,6 +371,11 @@ export const ar: TranslationResources = { insertAndSend: "أدخل النسخ وأرسل", failed: "فشل الإملاء:{{error}}", failedRetry: "فشل الإملاء. اضغط على إعادة المحاولة.", + empty: { + noAudio: "لم يُسجَّل أي صوت. هل يستخدم تطبيق آخر الميكروفون؟", + silentAudio: "سُجِّل صمت فقط. تحقّق من الميكروفون.", + noSpeech: "لم يُتعرَّف على أي كلام. اضغط على إعادة المحاولة للمحاولة مجددًا.", + }, }, question: { responseFailed: "تعذر إرسال ردك. حاول مرة أخرى.", diff --git a/packages/app/src/i18n/resources/en.ts b/packages/app/src/i18n/resources/en.ts index 1f61a83f12..2b8e97a914 100644 --- a/packages/app/src/i18n/resources/en.ts +++ b/packages/app/src/i18n/resources/en.ts @@ -368,6 +368,11 @@ export const en = { insertAndSend: "Insert transcription and send", failed: "Dictation failed: {{error}}", failedRetry: "Dictation failed. Tap retry.", + empty: { + noAudio: "No audio was recorded. Is another app using the microphone?", + silentAudio: "Only silence was recorded. Check your microphone.", + noSpeech: "No speech was recognized. Tap retry to try again.", + }, }, question: { responseFailed: "Could not send your response. Try again.", diff --git a/packages/app/src/i18n/resources/es.ts b/packages/app/src/i18n/resources/es.ts index 7e5a1d41e6..433c7d125a 100644 --- a/packages/app/src/i18n/resources/es.ts +++ b/packages/app/src/i18n/resources/es.ts @@ -376,6 +376,11 @@ export const es: TranslationResources = { insertAndSend: "Insertar transcripción y enviar", failed: "Fallo en el dictado:{{error}}", failedRetry: "El dictado falló. Toca reintentar.", + empty: { + noAudio: "No se grabó audio. ¿Otra app está usando el micrófono?", + silentAudio: "Solo se grabó silencio. Revisa tu micrófono.", + noSpeech: "No se reconoció ninguna voz. Toca reintentar para volver a intentarlo.", + }, }, question: { responseFailed: "No se pudo enviar tu respuesta. Inténtalo de nuevo.", diff --git a/packages/app/src/i18n/resources/fr.ts b/packages/app/src/i18n/resources/fr.ts index 551f4a4158..e411f97cfe 100644 --- a/packages/app/src/i18n/resources/fr.ts +++ b/packages/app/src/i18n/resources/fr.ts @@ -376,6 +376,11 @@ export const fr: TranslationResources = { insertAndSend: "Insérer la transcription et envoyer", failed: "Échec de la dictée : {{error}}", failedRetry: "La dictée a échoué. Appuyez sur Réessayer.", + empty: { + noAudio: "Aucun son n'a été enregistré. Une autre app utilise-t-elle le micro ?", + silentAudio: "Seul du silence a été enregistré. Vérifiez votre micro.", + noSpeech: "Aucune parole reconnue. Appuyez sur Réessayer pour recommencer.", + }, }, question: { responseFailed: "Impossible d’envoyer votre réponse. Réessayez.", diff --git a/packages/app/src/i18n/resources/ja.ts b/packages/app/src/i18n/resources/ja.ts index 4b1b8b6f64..dbcd34d77d 100644 --- a/packages/app/src/i18n/resources/ja.ts +++ b/packages/app/src/i18n/resources/ja.ts @@ -376,6 +376,11 @@ export const ja: TranslationResources = { insertAndSend: "文字起こしを挿入して送信", failed: "音声入力に失敗しました: {{error}}", failedRetry: "音声入力に失敗しました。再試行をタップしてください。", + empty: { + noAudio: "音声が録音されませんでした。別のアプリがマイクを使用していませんか？", + silentAudio: "無音しか録音されませんでした。マイクを確認してください。", + noSpeech: "音声を認識できませんでした。再試行をタップしてください。", + }, }, question: { responseFailed: "回答を送信できませんでした。もう一度お試しください。", diff --git a/packages/app/src/i18n/resources/ko.ts b/packages/app/src/i18n/resources/ko.ts index b9e33b4eba..d5e22aea9d 100644 --- a/packages/app/src/i18n/resources/ko.ts +++ b/packages/app/src/i18n/resources/ko.ts @@ -372,6 +372,11 @@ export const ko: TranslationResources = { insertAndSend: "전사 삽입 후 보내기", failed: "받아쓰기 실패: {{error}}", failedRetry: "받아쓰기에 실패했습니다. 다시 시도하려면 누르세요.", + empty: { + noAudio: "녹음된 오디오가 없습니다. 다른 앱이 마이크를 사용 중인가요?", + silentAudio: "무음만 녹음되었습니다. 마이크를 확인하세요.", + noSpeech: "음성이 인식되지 않았습니다. 다시 시도하려면 누르세요.", + }, }, question: { responseFailed: "응답을 보내지 못했습니다. 다시 시도해 주세요.", diff --git a/packages/app/src/i18n/resources/pt-BR.ts b/packages/app/src/i18n/resources/pt-BR.ts index 2fa6c30c8e..4dab47d71e 100644 --- a/packages/app/src/i18n/resources/pt-BR.ts +++ b/packages/app/src/i18n/resources/pt-BR.ts @@ -375,6 +375,11 @@ export const ptBR: TranslationResources = { insertAndSend: "Inserir transcrição e enviar", failed: "Falha no ditado: {{error}}", failedRetry: "Falha no ditado. Toque para tentar novamente.", + empty: { + noAudio: "Nenhum áudio foi gravado. Outro app está usando o microfone?", + silentAudio: "Só foi gravado silêncio. Verifique seu microfone.", + noSpeech: "Nenhuma fala foi reconhecida. Toque para tentar novamente.", + }, }, question: { responseFailed: "Não foi possível enviar sua resposta. Tente novamente.", diff --git a/packages/app/src/i18n/resources/ru.ts b/packages/app/src/i18n/resources/ru.ts index 9a6eb5b0e2..da7fee5150 100644 --- a/packages/app/src/i18n/resources/ru.ts +++ b/packages/app/src/i18n/resources/ru.ts @@ -375,6 +375,11 @@ export const ru: TranslationResources = { insertAndSend: "Вставить транскрипцию и отправить", failed: "Диктовка не удалась: {{error}}", failedRetry: "Диктовка не удалась. Нажмите «Повторить».", + empty: { + noAudio: "Звук не записан. Возможно, микрофон занят другим приложением?", + silentAudio: "Записана только тишина. Проверьте микрофон.", + noSpeech: "Речь не распознана. Нажмите «Повторить», чтобы попробовать снова.", + }, }, question: { responseFailed: "Не удалось отправить ответ. Попробуйте ещё раз.", diff --git a/packages/app/src/i18n/resources/zh-CN.ts b/packages/app/src/i18n/resources/zh-CN.ts index 553d8212b4..65e8416d95 100644 --- a/packages/app/src/i18n/resources/zh-CN.ts +++ b/packages/app/src/i18n/resources/zh-CN.ts @@ -371,6 +371,11 @@ export const zhCN: TranslationResources = { insertAndSend: "插入转写并发送", failed: "听写失败：{{error}}", failedRetry: "听写失败。点按重试。", + empty: { + noAudio: "未录到音频。是否有其他应用正在使用麦克风？", + silentAudio: "只录到了静音。请检查麦克风。", + noSpeech: "未识别到语音。点按重试再试一次。", + }, }, question: { responseFailed: "未能发送你的回复，请重试。",