mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-31 13:42:50 +00:00
fix(app): explain empty dictation results and retain recordings
Keep empty transcription results visible and preserve captured audio for retry.
This commit is contained in:
committed by
paseo-bot[bot]
parent
9c5b47421a
commit
0132f9f3e3
@@ -0,0 +1,64 @@
|
||||
import { expect, test } from "../support/fixtures";
|
||||
import { composerLocator, expectComposerVisible } from "../support/helpers/composer";
|
||||
import {
|
||||
installDictationTranscriptHarness,
|
||||
installSyntheticMicrophone,
|
||||
} from "../support/helpers/dictation";
|
||||
import { openAgentRoute, seedMockAgentWorkspace } from "../support/helpers/mock-agent";
|
||||
|
||||
const TRANSCRIPT = "Recovered on the second attempt";
|
||||
|
||||
test.describe("Dictation empty result", () => {
|
||||
test.describe.configure({ timeout: 240_000 });
|
||||
|
||||
test("explains an empty transcript and retries the same recording", async ({ page }) => {
|
||||
const agent = await seedMockAgentWorkspace({
|
||||
repoPrefix: "dictation-empty-",
|
||||
title: "Empty dictation",
|
||||
});
|
||||
await installSyntheticMicrophone(page);
|
||||
const harness = await installDictationTranscriptHarness(page, ["", TRANSCRIPT]);
|
||||
try {
|
||||
await openAgentRoute(page, agent);
|
||||
await expectComposerVisible(page);
|
||||
|
||||
await page.getByRole("button", { name: "Start dictation" }).click();
|
||||
await harness.waitForAudio();
|
||||
await page.getByRole("button", { name: "Insert transcription", exact: true }).click();
|
||||
|
||||
await expect(
|
||||
page.getByText("No speech was recognized. Tap retry to try again.", { exact: true }),
|
||||
).toBeVisible();
|
||||
await page.getByRole("button", { name: "Retry dictation", exact: true }).click();
|
||||
|
||||
await expect(composerLocator(page)).toHaveValue(TRANSCRIPT);
|
||||
} finally {
|
||||
await agent.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
test("says when the microphone only delivered silence", async ({ page }) => {
|
||||
const agent = await seedMockAgentWorkspace({
|
||||
repoPrefix: "dictation-silent-",
|
||||
title: "Silent dictation",
|
||||
});
|
||||
await installSyntheticMicrophone(page, { silent: true });
|
||||
const harness = await installDictationTranscriptHarness(page, [""]);
|
||||
try {
|
||||
await openAgentRoute(page, agent);
|
||||
await expectComposerVisible(page);
|
||||
|
||||
await page.getByRole("button", { name: "Start dictation" }).click();
|
||||
await harness.waitForAudio();
|
||||
await page.getByRole("button", { name: "Insert transcription", exact: true }).click();
|
||||
|
||||
await expect(
|
||||
page.getByText("Only silence was recorded. Check your microphone.", { exact: true }),
|
||||
).toBeVisible();
|
||||
await page.getByRole("button", { name: "Cancel dictation", exact: true }).click();
|
||||
await expect(composerLocator(page)).toBeEditable();
|
||||
} finally {
|
||||
await agent.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -16,74 +16,16 @@ import {
|
||||
expectComposerFullscreenControls,
|
||||
} from "../support/helpers/composer";
|
||||
import { waitForSidebarHydration } from "../support/helpers/workspace-ui";
|
||||
|
||||
type WebSocketMessage = string | Buffer;
|
||||
import {
|
||||
enableDictationCapability,
|
||||
installSyntheticMicrophone,
|
||||
sendSessionMessage,
|
||||
sessionMessage,
|
||||
} from "../support/helpers/dictation";
|
||||
|
||||
const TRANSCRIPT = "Keep this spoken prompt visible while creating the workspace";
|
||||
const CREATE_FAILURE = "Synthetic workspace creation failure";
|
||||
|
||||
function parseEnvelope(message: WebSocketMessage): {
|
||||
type?: unknown;
|
||||
message?: Record<string, unknown>;
|
||||
} | null {
|
||||
const raw = typeof message === "string" ? message : message.toString("utf8");
|
||||
try {
|
||||
return JSON.parse(raw) as { type?: unknown; message?: Record<string, unknown> };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sessionMessage(message: WebSocketMessage): Record<string, unknown> | null {
|
||||
const envelope = parseEnvelope(message);
|
||||
return envelope?.type === "session" && envelope.message ? envelope.message : null;
|
||||
}
|
||||
|
||||
function sendSessionMessage(
|
||||
ws: { send(message: string): void },
|
||||
message: Record<string, unknown>,
|
||||
): void {
|
||||
ws.send(JSON.stringify({ type: "session", message }));
|
||||
}
|
||||
|
||||
function enableDictationCapability(message: WebSocketMessage): WebSocketMessage {
|
||||
const envelope = parseEnvelope(message);
|
||||
const payload = envelope?.message?.payload;
|
||||
if (
|
||||
envelope?.message?.type !== "status" ||
|
||||
!payload ||
|
||||
typeof payload !== "object" ||
|
||||
(payload as { status?: unknown }).status !== "server_info"
|
||||
) {
|
||||
return message;
|
||||
}
|
||||
|
||||
(payload as Record<string, unknown>).capabilities = {
|
||||
voice: {
|
||||
dictation: { enabled: true, reason: "" },
|
||||
voice: { enabled: false, reason: "Realtime voice is disabled in this test." },
|
||||
},
|
||||
};
|
||||
return JSON.stringify(envelope);
|
||||
}
|
||||
|
||||
async function installSyntheticMicrophone(page: Page): Promise<void> {
|
||||
await page.addInitScript(() => {
|
||||
const mediaDevices = navigator.mediaDevices;
|
||||
Object.defineProperty(mediaDevices, "getUserMedia", {
|
||||
configurable: true,
|
||||
value: async () => {
|
||||
const context = new AudioContext();
|
||||
const oscillator = context.createOscillator();
|
||||
const destination = context.createMediaStreamDestination();
|
||||
oscillator.connect(destination);
|
||||
oscillator.start();
|
||||
return destination.stream;
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function installDictationFailureHarness(page: Page) {
|
||||
let resolveAudioChunk!: () => void;
|
||||
let resolveCreateRequest!: () => void;
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
import type { Page } from "../fixtures";
|
||||
import { daemonWsRoutePattern } from "./daemon-port";
|
||||
|
||||
type WebSocketMessage = string | Buffer;
|
||||
|
||||
function parseEnvelope(message: WebSocketMessage): {
|
||||
type?: unknown;
|
||||
message?: Record<string, unknown>;
|
||||
} | null {
|
||||
const raw = typeof message === "string" ? message : message.toString("utf8");
|
||||
try {
|
||||
return JSON.parse(raw) as { type?: unknown; message?: Record<string, unknown> };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function sessionMessage(message: WebSocketMessage): Record<string, unknown> | null {
|
||||
const envelope = parseEnvelope(message);
|
||||
return envelope?.type === "session" && envelope.message ? envelope.message : null;
|
||||
}
|
||||
|
||||
export function sendSessionMessage(
|
||||
ws: { send(message: string): void },
|
||||
message: Record<string, unknown>,
|
||||
): void {
|
||||
ws.send(JSON.stringify({ type: "session", message }));
|
||||
}
|
||||
|
||||
export function enableDictationCapability(message: WebSocketMessage): WebSocketMessage {
|
||||
const envelope = parseEnvelope(message);
|
||||
const payload = envelope?.message?.payload;
|
||||
if (
|
||||
envelope?.message?.type !== "status" ||
|
||||
!payload ||
|
||||
typeof payload !== "object" ||
|
||||
(payload as { status?: unknown }).status !== "server_info"
|
||||
) {
|
||||
return message;
|
||||
}
|
||||
(payload as Record<string, unknown>).capabilities = {
|
||||
voice: {
|
||||
dictation: { enabled: true, reason: "" },
|
||||
voice: { enabled: false, reason: "Realtime voice is disabled in this test." },
|
||||
},
|
||||
};
|
||||
return JSON.stringify(envelope);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the browser microphone with an oscillator. `silent` keeps the stream
|
||||
* running at zero gain, which is what a muted or wrongly routed input delivers.
|
||||
*/
|
||||
export async function installSyntheticMicrophone(
|
||||
page: Page,
|
||||
options: { silent?: boolean } = {},
|
||||
): Promise<void> {
|
||||
await page.addInitScript((silent: boolean) => {
|
||||
const mediaDevices = navigator.mediaDevices;
|
||||
Object.defineProperty(mediaDevices, "getUserMedia", {
|
||||
configurable: true,
|
||||
value: async () => {
|
||||
const context = new AudioContext();
|
||||
const oscillator = context.createOscillator();
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = silent ? 0 : 1;
|
||||
const destination = context.createMediaStreamDestination();
|
||||
oscillator.connect(gain);
|
||||
gain.connect(destination);
|
||||
oscillator.start();
|
||||
return destination.stream;
|
||||
},
|
||||
});
|
||||
}, options.silent ?? false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Answers dictation stream requests in place of the daemon. Each finish is
|
||||
* answered with the next queued transcript; everything else reaches the daemon.
|
||||
*/
|
||||
export async function installDictationTranscriptHarness(page: Page, transcripts: string[]) {
|
||||
const queue = [...transcripts];
|
||||
let resolveAudioChunk!: () => void;
|
||||
const audioChunk = new Promise<void>((resolve) => {
|
||||
resolveAudioChunk = resolve;
|
||||
});
|
||||
|
||||
await page.routeWebSocket(daemonWsRoutePattern(), (ws) => {
|
||||
const server = ws.connectToServer();
|
||||
|
||||
ws.onMessage((message) => {
|
||||
const request = sessionMessage(message);
|
||||
const type = request?.type;
|
||||
const dictationId = typeof request?.dictationId === "string" ? request.dictationId : null;
|
||||
|
||||
if (type === "dictation_stream_start" && dictationId) {
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_ack",
|
||||
payload: { dictationId, ackSeq: -1 },
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (type === "dictation_stream_chunk" && dictationId) {
|
||||
const seq = typeof request?.seq === "number" ? request.seq : 0;
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_ack",
|
||||
payload: { dictationId, ackSeq: seq },
|
||||
});
|
||||
resolveAudioChunk();
|
||||
return;
|
||||
}
|
||||
if (type === "dictation_stream_finish" && dictationId) {
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_finish_accepted",
|
||||
payload: { dictationId, timeoutMs: 5_000 },
|
||||
});
|
||||
sendSessionMessage(ws, {
|
||||
type: "dictation_stream_final",
|
||||
payload: { dictationId, text: queue.shift() ?? "" },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
server.send(message);
|
||||
});
|
||||
|
||||
server.onMessage((message) => ws.send(enableDictationCapability(message)));
|
||||
});
|
||||
|
||||
return { waitForAudio: () => audioChunk };
|
||||
}
|
||||
@@ -4,9 +4,11 @@ import { View, Text, Pressable } from "react-native";
|
||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||
import { X, ArrowUp, RefreshCcw, Check, Mic, Pencil } from "lucide-react-native";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import type { TFunction } from "i18next";
|
||||
import { VolumeMeter } from "./volume-meter";
|
||||
import { FOOTER_HEIGHT } from "@/constants/layout";
|
||||
import type { DictationStatus } from "@/hooks/use-dictation";
|
||||
import type { DictationEmptyReason } from "@/dictation/empty-transcript-reason";
|
||||
|
||||
interface DictationControlsProps {
|
||||
volume: number;
|
||||
@@ -24,6 +26,31 @@ interface DictationControlsProps {
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
const EMPTY_REASON_MESSAGE_KEYS = {
|
||||
"no-audio": "message.dictation.empty.noAudio",
|
||||
"silent-audio": "message.dictation.empty.silentAudio",
|
||||
"no-speech": "message.dictation.empty.noSpeech",
|
||||
} as const satisfies Record<DictationEmptyReason, string>;
|
||||
|
||||
// The line under the meter once an attempt has finished without text.
|
||||
function resolveResultText(input: {
|
||||
t: TFunction;
|
||||
status: DictationStatus;
|
||||
errorText?: string;
|
||||
emptyReason?: DictationEmptyReason | null;
|
||||
}): string | null {
|
||||
const { t, status, errorText, emptyReason } = input;
|
||||
if (status === "empty") {
|
||||
return t(EMPTY_REASON_MESSAGE_KEYS[emptyReason ?? "no-speech"]);
|
||||
}
|
||||
if (status !== "failed") {
|
||||
return null;
|
||||
}
|
||||
return errorText
|
||||
? t("message.dictation.failed", { error: errorText })
|
||||
: t("message.dictation.failedRetry");
|
||||
}
|
||||
|
||||
function formatDuration(seconds: number): string {
|
||||
const mins = Math.floor(seconds / 60);
|
||||
const secs = seconds % 60;
|
||||
@@ -147,18 +174,26 @@ export function DictationOverlay({
|
||||
isProcessing,
|
||||
status,
|
||||
errorText,
|
||||
emptyReason,
|
||||
onCancel,
|
||||
onAccept,
|
||||
onAcceptAndSend,
|
||||
onRetry,
|
||||
onDiscard,
|
||||
}: Omit<DictationControlsProps, "onStart" | "disabled" | "transcript"> & { errorText?: string }) {
|
||||
}: Omit<DictationControlsProps, "onStart" | "disabled" | "transcript"> & {
|
||||
errorText?: string;
|
||||
emptyReason?: DictationEmptyReason | null;
|
||||
}) {
|
||||
const { theme } = useUnistyles();
|
||||
const { t } = useTranslation();
|
||||
const isFailed = status === "failed";
|
||||
const isEmpty = status === "empty";
|
||||
// An empty result is a finished attempt the user has to acknowledge, like a failure.
|
||||
const isFailed = status === "failed" || isEmpty;
|
||||
const canRetry = !isEmpty || emptyReason !== "no-audio";
|
||||
const showActiveState = isRecording || isProcessing || isFailed;
|
||||
const actionsDisabled = isProcessing;
|
||||
const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
|
||||
const resultText = resolveResultText({ t, status, errorText, emptyReason });
|
||||
|
||||
const containerStyle = useMemo(
|
||||
() => [overlayStyles.container, { backgroundColor: theme.colors.accent }],
|
||||
@@ -217,11 +252,9 @@ export function DictationOverlay({
|
||||
{t("message.dictation.paused")}
|
||||
</Text>
|
||||
) : null}
|
||||
{isFailed ? (
|
||||
{resultText ? (
|
||||
<Text numberOfLines={2} style={overlayTranscriptTextStyle}>
|
||||
{errorText
|
||||
? t("message.dictation.failed", { error: errorText })
|
||||
: t("message.dictation.failedRetry")}
|
||||
{resultText}
|
||||
</Text>
|
||||
) : null}
|
||||
</View>
|
||||
@@ -232,7 +265,7 @@ export function DictationOverlay({
|
||||
<LoadingSpinner size="small" color={theme.colors.accentForeground} />
|
||||
</View>
|
||||
) : null}
|
||||
{!actionsDisabled && isFailed ? (
|
||||
{!actionsDisabled && isFailed && canRetry ? (
|
||||
<Pressable
|
||||
onPress={onRetry}
|
||||
accessibilityRole="button"
|
||||
|
||||
@@ -562,6 +562,7 @@ function MessageInputOverlay({
|
||||
isDictationProcessing,
|
||||
dictationStatus,
|
||||
dictationError,
|
||||
dictationEmptyReason,
|
||||
onCancelRecording,
|
||||
onAcceptRecording,
|
||||
onAcceptAndSendRecording,
|
||||
@@ -585,6 +586,7 @@ function MessageInputOverlay({
|
||||
isDictationProcessing: boolean;
|
||||
dictationStatus: React.ComponentProps<typeof DictationOverlay>["status"];
|
||||
dictationError: string | null;
|
||||
dictationEmptyReason: React.ComponentProps<typeof DictationOverlay>["emptyReason"];
|
||||
onCancelRecording: () => Promise<void>;
|
||||
onAcceptRecording: () => Promise<void>;
|
||||
onAcceptAndSendRecording: () => Promise<void>;
|
||||
@@ -593,6 +595,7 @@ function MessageInputOverlay({
|
||||
onRealtimeVoiceStop: () => void;
|
||||
}) {
|
||||
if (showDictationOverlay) {
|
||||
const awaitsAcknowledgement = dictationStatus === "failed" || dictationStatus === "empty";
|
||||
return (
|
||||
<DictationOverlay
|
||||
volume={dictationVolume}
|
||||
@@ -601,11 +604,12 @@ function MessageInputOverlay({
|
||||
isProcessing={isDictationProcessing}
|
||||
status={dictationStatus}
|
||||
errorText={dictationStatus === "failed" ? (dictationError ?? undefined) : undefined}
|
||||
emptyReason={dictationEmptyReason}
|
||||
onCancel={onCancelRecording}
|
||||
onAccept={onAcceptRecording}
|
||||
onAcceptAndSend={onAcceptAndSendRecording}
|
||||
onRetry={dictationStatus === "failed" ? onRetryFailedRecording : undefined}
|
||||
onDiscard={dictationStatus === "failed" ? onDiscardFailedRecording : undefined}
|
||||
onRetry={awaitsAcknowledgement ? onRetryFailedRecording : undefined}
|
||||
onDiscard={awaitsAcknowledgement ? onDiscardFailedRecording : undefined}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -1032,7 +1036,12 @@ function computeShouldShowDictationOverlay(
|
||||
isDictationProcessing: boolean,
|
||||
dictationStatus: string,
|
||||
): boolean {
|
||||
return isDictating || isDictationProcessing || dictationStatus === "failed";
|
||||
return (
|
||||
isDictating ||
|
||||
isDictationProcessing ||
|
||||
dictationStatus === "failed" ||
|
||||
dictationStatus === "empty"
|
||||
);
|
||||
}
|
||||
|
||||
function computeIsDictationStartEnabled(
|
||||
@@ -1533,6 +1542,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
duration: dictationDuration,
|
||||
error: dictationError,
|
||||
status: dictationStatus,
|
||||
emptyReason: dictationEmptyReason,
|
||||
startDictation,
|
||||
cancelDictation,
|
||||
confirmDictation,
|
||||
@@ -2180,6 +2190,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
isDictationProcessing={isDictationProcessing}
|
||||
dictationStatus={dictationStatus}
|
||||
dictationError={dictationError}
|
||||
dictationEmptyReason={dictationEmptyReason}
|
||||
onCancelRecording={handleCancelRecording}
|
||||
onAcceptRecording={handleAcceptRecording}
|
||||
onAcceptAndSendRecording={handleAcceptAndSendRecording}
|
||||
@@ -2201,6 +2212,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
isDictationProcessing,
|
||||
dictationStatus,
|
||||
dictationError,
|
||||
dictationEmptyReason,
|
||||
handleCancelRecording,
|
||||
handleAcceptRecording,
|
||||
handleAcceptAndSendRecording,
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { Buffer } from "buffer";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveDictationEmptyReason } from "@/dictation/empty-transcript-reason";
|
||||
import {
|
||||
DICTATION_SILENCE_PEAK_THRESHOLD,
|
||||
isAudiblePeak,
|
||||
pcm16PeakFromBase64,
|
||||
} from "@/dictation/pcm-level";
|
||||
|
||||
function pcm16Base64(samples: number[]): string {
|
||||
const bytes = Buffer.alloc(samples.length * 2);
|
||||
samples.forEach((sample, index) => bytes.writeInt16LE(sample, index * 2));
|
||||
return bytes.toString("base64");
|
||||
}
|
||||
|
||||
describe("pcm16PeakFromBase64", () => {
|
||||
it("returns the largest absolute sample, including negative ones", () => {
|
||||
expect(pcm16PeakFromBase64(pcm16Base64([0, 120, -4000, 2500]))).toBe(4000);
|
||||
});
|
||||
|
||||
it("reads digital silence as zero", () => {
|
||||
expect(pcm16PeakFromBase64(pcm16Base64([0, 0, 0, 0]))).toBe(0);
|
||||
});
|
||||
|
||||
it("ignores a trailing odd byte", () => {
|
||||
const bytes = Buffer.concat([Buffer.from(pcm16Base64([-700]), "base64"), Buffer.from([0x7f])]);
|
||||
expect(pcm16PeakFromBase64(bytes.toString("base64"))).toBe(700);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isAudiblePeak", () => {
|
||||
it("uses the daemon's default silence threshold", () => {
|
||||
expect(isAudiblePeak(DICTATION_SILENCE_PEAK_THRESHOLD - 1)).toBe(false);
|
||||
expect(isAudiblePeak(DICTATION_SILENCE_PEAK_THRESHOLD)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveDictationEmptyReason", () => {
|
||||
it("reports missing audio when the microphone delivered nothing", () => {
|
||||
expect(resolveDictationEmptyReason({ segmentCount: 0, peak: 0 })).toBe("no-audio");
|
||||
});
|
||||
|
||||
it("reports silent audio when nothing crossed the silence threshold", () => {
|
||||
expect(resolveDictationEmptyReason({ segmentCount: 40, peak: 120 })).toBe("silent-audio");
|
||||
});
|
||||
|
||||
it("reports no recognized speech when audible audio produced no text", () => {
|
||||
expect(resolveDictationEmptyReason({ segmentCount: 40, peak: 9000 })).toBe("no-speech");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
import { isAudiblePeak } from "./pcm-level";
|
||||
|
||||
/**
|
||||
* Why a finished dictation produced no text.
|
||||
*
|
||||
* - `no-audio`: the microphone delivered no audio at all.
|
||||
* - `silent-audio`: audio arrived, but it never rose above the daemon's silence
|
||||
* threshold, so the daemon dropped it before speech-to-text.
|
||||
* - `no-speech`: audible audio reached speech-to-text, which returned no words.
|
||||
*/
|
||||
export type DictationEmptyReason = "no-audio" | "silent-audio" | "no-speech";
|
||||
|
||||
export function resolveDictationEmptyReason(input: {
|
||||
segmentCount: number;
|
||||
peak: number;
|
||||
}): DictationEmptyReason {
|
||||
if (input.segmentCount === 0) {
|
||||
return "no-audio";
|
||||
}
|
||||
if (!isAudiblePeak(input.peak)) {
|
||||
return "silent-audio";
|
||||
}
|
||||
return "no-speech";
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Buffer } from "buffer";
|
||||
|
||||
// Mirrors the daemon's default `PASEO_DICTATION_SILENCE_PEAK_THRESHOLD`
|
||||
// (packages/server/src/server/dictation/dictation-stream-manager.ts). Audio whose
|
||||
// peak stays below it is cleared by the daemon and never reaches speech-to-text.
|
||||
// The `volume` level from the audio sources can't be used for this: native reports
|
||||
// a dB curve and web a linear RMS, so the same input reads differently per platform.
|
||||
export const DICTATION_SILENCE_PEAK_THRESHOLD = 300;
|
||||
|
||||
export function pcm16PeakFromBase64(base64Pcm: string): number {
|
||||
const bytes = Buffer.from(base64Pcm, "base64");
|
||||
let peak = 0;
|
||||
for (let offset = 0; offset + 1 < bytes.length; offset += 2) {
|
||||
const sample = Math.abs(bytes.readInt16LE(offset));
|
||||
if (sample > peak) {
|
||||
peak = sample;
|
||||
}
|
||||
}
|
||||
return peak;
|
||||
}
|
||||
|
||||
export function isAudiblePeak(peak: number): boolean {
|
||||
return peak >= DICTATION_SILENCE_PEAK_THRESHOLD;
|
||||
}
|
||||
@@ -1,6 +1,14 @@
|
||||
import type { DictationEmptyReason } from "@/dictation/empty-transcript-reason";
|
||||
import { i18n } from "@/i18n/i18next";
|
||||
|
||||
export type DictationStatus = "idle" | "recording" | "paused" | "uploading" | "failed";
|
||||
export type DictationStatus =
|
||||
| "idle"
|
||||
| "recording"
|
||||
| "paused"
|
||||
| "uploading"
|
||||
| "failed"
|
||||
// Finished, but produced no text. `emptyReason` says why; the audio is kept for retry.
|
||||
| "empty";
|
||||
|
||||
export interface UseDictationOptions {
|
||||
client: import("@getpaseo/client/internal/daemon-client").DaemonClient | null;
|
||||
@@ -22,6 +30,7 @@ export interface UseDictationResult {
|
||||
duration: number;
|
||||
error: string | null;
|
||||
status: DictationStatus;
|
||||
emptyReason: DictationEmptyReason | null;
|
||||
startDictation: () => Promise<void>;
|
||||
cancelDictation: () => Promise<void>;
|
||||
confirmDictation: () => Promise<void>;
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
/** @vitest-environment jsdom */
|
||||
import { Buffer } from "buffer";
|
||||
import { act, renderHook } from "@testing-library/react";
|
||||
import type { SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
|
||||
import { useDictation, type UseDictationOptions } from "./use-dictation";
|
||||
|
||||
const audioSource = vi.hoisted(() => ({
|
||||
config: null as DictationAudioSourceConfig | null,
|
||||
}));
|
||||
|
||||
vi.mock("@/hooks/use-dictation-audio-source", () => ({
|
||||
useDictationAudioSource: (config: DictationAudioSourceConfig) => {
|
||||
audioSource.config = config;
|
||||
return {
|
||||
start: async () => undefined,
|
||||
stop: async () => undefined,
|
||||
volume: 0,
|
||||
isPaused: false,
|
||||
};
|
||||
},
|
||||
}));
|
||||
|
||||
function pcm16Base64(samples: number[]): string {
|
||||
const bytes = Buffer.alloc(samples.length * 2);
|
||||
samples.forEach((sample, index) => bytes.writeInt16LE(sample, index * 2));
|
||||
return bytes.toString("base64");
|
||||
}
|
||||
|
||||
class FakeDictationClient {
|
||||
isConnected = true;
|
||||
transcript = "";
|
||||
finishes: Array<{ dictationId: string; finalSeq: number }> = [];
|
||||
private readonly rawListeners = new Set<(message: SessionOutboundMessage) => void>();
|
||||
|
||||
async startDictationStream(): Promise<void> {}
|
||||
|
||||
sendDictationStreamChunk(dictationId: string, seq: number): void {
|
||||
const ack = { type: "dictation_stream_ack", payload: { dictationId, ackSeq: seq } };
|
||||
for (const listener of this.rawListeners) {
|
||||
listener(ack as SessionOutboundMessage);
|
||||
}
|
||||
}
|
||||
|
||||
async finishDictationStream(dictationId: string, finalSeq: number) {
|
||||
this.finishes.push({ dictationId, finalSeq });
|
||||
return { dictationId, text: this.transcript };
|
||||
}
|
||||
|
||||
cancelDictationStream(): void {}
|
||||
|
||||
subscribeRawMessages(handler: (message: SessionOutboundMessage) => void): () => void {
|
||||
this.rawListeners.add(handler);
|
||||
return () => this.rawListeners.delete(handler);
|
||||
}
|
||||
|
||||
subscribeConnectionStatus(): () => void {
|
||||
return () => undefined;
|
||||
}
|
||||
|
||||
on(): () => void {
|
||||
return () => undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function renderDictation(client: FakeDictationClient) {
|
||||
const onTranscript = vi.fn();
|
||||
const view = renderHook(() =>
|
||||
useDictation({
|
||||
client: client as unknown as UseDictationOptions["client"],
|
||||
onTranscript,
|
||||
}),
|
||||
);
|
||||
return { view, onTranscript };
|
||||
}
|
||||
|
||||
async function record(view: ReturnType<typeof renderDictation>["view"], segments: string[]) {
|
||||
await act(async () => {
|
||||
await view.result.current.startDictation();
|
||||
});
|
||||
await act(async () => {
|
||||
for (const segment of segments) {
|
||||
audioSource.config?.onPcmSegment(segment);
|
||||
}
|
||||
});
|
||||
await act(async () => {
|
||||
await view.result.current.confirmDictation();
|
||||
});
|
||||
}
|
||||
|
||||
describe("useDictation empty results", () => {
|
||||
beforeEach(() => {
|
||||
audioSource.config = null;
|
||||
});
|
||||
|
||||
it("explains a silent recording and keeps its audio for retry", async () => {
|
||||
const client = new FakeDictationClient();
|
||||
const { view, onTranscript } = renderDictation(client);
|
||||
|
||||
await record(view, [pcm16Base64([0, 40, -60]), pcm16Base64([10, -20, 0])]);
|
||||
|
||||
expect(view.result.current.status).toBe("empty");
|
||||
expect(view.result.current.emptyReason).toBe("silent-audio");
|
||||
expect(onTranscript).not.toHaveBeenCalled();
|
||||
|
||||
client.transcript = "hello there";
|
||||
await act(async () => {
|
||||
await view.result.current.retryFailedDictation();
|
||||
});
|
||||
|
||||
expect(client.finishes.map((finish) => finish.finalSeq)).toEqual([1, 1]);
|
||||
expect(onTranscript).toHaveBeenCalledWith("hello there", expect.anything());
|
||||
expect(view.result.current.status).toBe("idle");
|
||||
expect(view.result.current.emptyReason).toBeNull();
|
||||
});
|
||||
|
||||
it("says when audible speech produced no text", async () => {
|
||||
const client = new FakeDictationClient();
|
||||
const { view } = renderDictation(client);
|
||||
|
||||
await record(view, [pcm16Base64([0, 9000, -12000])]);
|
||||
|
||||
expect(view.result.current.status).toBe("empty");
|
||||
expect(view.result.current.emptyReason).toBe("no-speech");
|
||||
});
|
||||
|
||||
it("says when the microphone delivered no audio", async () => {
|
||||
const client = new FakeDictationClient();
|
||||
const { view } = renderDictation(client);
|
||||
|
||||
await record(view, []);
|
||||
|
||||
expect(view.result.current.status).toBe("empty");
|
||||
expect(view.result.current.emptyReason).toBe("no-audio");
|
||||
expect(client.finishes).toEqual([]);
|
||||
});
|
||||
|
||||
it("clears the empty result when dismissed", async () => {
|
||||
const client = new FakeDictationClient();
|
||||
const { view } = renderDictation(client);
|
||||
|
||||
await record(view, [pcm16Base64([5, 5])]);
|
||||
act(() => {
|
||||
view.result.current.discardFailedDictation();
|
||||
});
|
||||
|
||||
expect(view.result.current.status).toBe("idle");
|
||||
expect(view.result.current.emptyReason).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,11 @@ import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
|
||||
import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
|
||||
import {
|
||||
resolveDictationEmptyReason,
|
||||
type DictationEmptyReason,
|
||||
} from "@/dictation/empty-transcript-reason";
|
||||
import { pcm16PeakFromBase64 } from "@/dictation/pcm-level";
|
||||
import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source";
|
||||
import { generateMessageId } from "@/types/stream";
|
||||
import { AttemptGuard } from "@/utils/attempt-guard";
|
||||
@@ -33,7 +38,10 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [status, setStatus] = useState<DictationStatus>("idle");
|
||||
const [emptyReason, setEmptyReason] = useState<DictationEmptyReason | null>(null);
|
||||
const latestPartialTranscriptRef = useRef("");
|
||||
// Loudest sample captured in the current recording; explains an empty transcript.
|
||||
const recordingPeakRef = useRef(0);
|
||||
|
||||
const onTranscriptRef = useRef(onTranscript);
|
||||
useEffect(() => {
|
||||
@@ -133,8 +141,10 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const clearStreamingState = useCallback(() => {
|
||||
senderRef.current?.clearAll();
|
||||
recordingPeakRef.current = 0;
|
||||
latestPartialTranscriptRef.current = "";
|
||||
setPartialTranscript("");
|
||||
setEmptyReason(null);
|
||||
}, []);
|
||||
|
||||
const startNewStream = useCallback(async (reason: string) => {
|
||||
@@ -194,11 +204,19 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const transcriptText =
|
||||
text.trim().length > 0 ? text.trim() : latestPartialTranscriptRef.current.trim();
|
||||
clearStreamingState();
|
||||
|
||||
if (!transcriptText) {
|
||||
// Keep the audio so the user can retry, and say why nothing came back.
|
||||
setStatus("empty");
|
||||
setEmptyReason(
|
||||
resolveDictationEmptyReason({
|
||||
segmentCount: senderRef.current?.getSegmentCount() ?? 0,
|
||||
peak: recordingPeakRef.current,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
clearStreamingState();
|
||||
onTranscriptRef.current?.(transcriptText, { requestId });
|
||||
},
|
||||
[clearStreamingState],
|
||||
@@ -228,6 +246,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const audio = useDictationAudioSource({
|
||||
onPcmSegment: (audioData) => {
|
||||
recordingPeakRef.current = Math.max(recordingPeakRef.current, pcm16PeakFromBase64(audioData));
|
||||
senderRef.current?.enqueueSegment(audioData);
|
||||
},
|
||||
onError: (err) => {
|
||||
@@ -395,6 +414,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
return;
|
||||
}
|
||||
setError(null);
|
||||
setEmptyReason(null);
|
||||
setStatus("uploading");
|
||||
setIsProcessing(true);
|
||||
isProcessingRef.current = true;
|
||||
@@ -462,6 +482,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
duration,
|
||||
error,
|
||||
status: status === "recording" && audio.isPaused ? "paused" : status,
|
||||
emptyReason,
|
||||
startDictation,
|
||||
cancelDictation,
|
||||
confirmDictation,
|
||||
|
||||
@@ -371,6 +371,11 @@ export const ar: TranslationResources = {
|
||||
insertAndSend: "أدخل النسخ وأرسل",
|
||||
failed: "فشل الإملاء:{{error}}",
|
||||
failedRetry: "فشل الإملاء. اضغط على إعادة المحاولة.",
|
||||
empty: {
|
||||
noAudio: "لم يُسجَّل أي صوت. هل يستخدم تطبيق آخر الميكروفون؟",
|
||||
silentAudio: "سُجِّل صمت فقط. تحقّق من الميكروفون.",
|
||||
noSpeech: "لم يُتعرَّف على أي كلام. اضغط على إعادة المحاولة للمحاولة مجددًا.",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "تعذر إرسال ردك. حاول مرة أخرى.",
|
||||
|
||||
@@ -368,6 +368,11 @@ export const en = {
|
||||
insertAndSend: "Insert transcription and send",
|
||||
failed: "Dictation failed: {{error}}",
|
||||
failedRetry: "Dictation failed. Tap retry.",
|
||||
empty: {
|
||||
noAudio: "No audio was recorded. Is another app using the microphone?",
|
||||
silentAudio: "Only silence was recorded. Check your microphone.",
|
||||
noSpeech: "No speech was recognized. Tap retry to try again.",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "Could not send your response. Try again.",
|
||||
|
||||
@@ -376,6 +376,11 @@ export const es: TranslationResources = {
|
||||
insertAndSend: "Insertar transcripción y enviar",
|
||||
failed: "Fallo en el dictado:{{error}}",
|
||||
failedRetry: "El dictado falló. Toca reintentar.",
|
||||
empty: {
|
||||
noAudio: "No se grabó audio. ¿Otra app está usando el micrófono?",
|
||||
silentAudio: "Solo se grabó silencio. Revisa tu micrófono.",
|
||||
noSpeech: "No se reconoció ninguna voz. Toca reintentar para volver a intentarlo.",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "No se pudo enviar tu respuesta. Inténtalo de nuevo.",
|
||||
|
||||
@@ -376,6 +376,11 @@ export const fr: TranslationResources = {
|
||||
insertAndSend: "Insérer la transcription et envoyer",
|
||||
failed: "Échec de la dictée : {{error}}",
|
||||
failedRetry: "La dictée a échoué. Appuyez sur Réessayer.",
|
||||
empty: {
|
||||
noAudio: "Aucun son n'a été enregistré. Une autre app utilise-t-elle le micro ?",
|
||||
silentAudio: "Seul du silence a été enregistré. Vérifiez votre micro.",
|
||||
noSpeech: "Aucune parole reconnue. Appuyez sur Réessayer pour recommencer.",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "Impossible d’envoyer votre réponse. Réessayez.",
|
||||
|
||||
@@ -376,6 +376,11 @@ export const ja: TranslationResources = {
|
||||
insertAndSend: "文字起こしを挿入して送信",
|
||||
failed: "音声入力に失敗しました: {{error}}",
|
||||
failedRetry: "音声入力に失敗しました。再試行をタップしてください。",
|
||||
empty: {
|
||||
noAudio: "音声が録音されませんでした。別のアプリがマイクを使用していませんか?",
|
||||
silentAudio: "無音しか録音されませんでした。マイクを確認してください。",
|
||||
noSpeech: "音声を認識できませんでした。再試行をタップしてください。",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "回答を送信できませんでした。もう一度お試しください。",
|
||||
|
||||
@@ -372,6 +372,11 @@ export const ko: TranslationResources = {
|
||||
insertAndSend: "전사 삽입 후 보내기",
|
||||
failed: "받아쓰기 실패: {{error}}",
|
||||
failedRetry: "받아쓰기에 실패했습니다. 다시 시도하려면 누르세요.",
|
||||
empty: {
|
||||
noAudio: "녹음된 오디오가 없습니다. 다른 앱이 마이크를 사용 중인가요?",
|
||||
silentAudio: "무음만 녹음되었습니다. 마이크를 확인하세요.",
|
||||
noSpeech: "음성이 인식되지 않았습니다. 다시 시도하려면 누르세요.",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "응답을 보내지 못했습니다. 다시 시도해 주세요.",
|
||||
|
||||
@@ -375,6 +375,11 @@ export const ptBR: TranslationResources = {
|
||||
insertAndSend: "Inserir transcrição e enviar",
|
||||
failed: "Falha no ditado: {{error}}",
|
||||
failedRetry: "Falha no ditado. Toque para tentar novamente.",
|
||||
empty: {
|
||||
noAudio: "Nenhum áudio foi gravado. Outro app está usando o microfone?",
|
||||
silentAudio: "Só foi gravado silêncio. Verifique seu microfone.",
|
||||
noSpeech: "Nenhuma fala foi reconhecida. Toque para tentar novamente.",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "Não foi possível enviar sua resposta. Tente novamente.",
|
||||
|
||||
@@ -375,6 +375,11 @@ export const ru: TranslationResources = {
|
||||
insertAndSend: "Вставить транскрипцию и отправить",
|
||||
failed: "Диктовка не удалась: {{error}}",
|
||||
failedRetry: "Диктовка не удалась. Нажмите «Повторить».",
|
||||
empty: {
|
||||
noAudio: "Звук не записан. Возможно, микрофон занят другим приложением?",
|
||||
silentAudio: "Записана только тишина. Проверьте микрофон.",
|
||||
noSpeech: "Речь не распознана. Нажмите «Повторить», чтобы попробовать снова.",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "Не удалось отправить ответ. Попробуйте ещё раз.",
|
||||
|
||||
@@ -371,6 +371,11 @@ export const zhCN: TranslationResources = {
|
||||
insertAndSend: "插入转写并发送",
|
||||
failed: "听写失败:{{error}}",
|
||||
failedRetry: "听写失败。点按重试。",
|
||||
empty: {
|
||||
noAudio: "未录到音频。是否有其他应用正在使用麦克风?",
|
||||
silentAudio: "只录到了静音。请检查麦克风。",
|
||||
noSpeech: "未识别到语音。点按重试再试一次。",
|
||||
},
|
||||
},
|
||||
question: {
|
||||
responseFailed: "未能发送你的回复,请重试。",
|
||||
|
||||
Reference in New Issue
Block a user