mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-01 14:12:51 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e635d9b46 | ||
|
|
3e859625ed | ||
|
|
83b1a3cbe2 | ||
|
|
0132f9f3e3 |
@@ -0,0 +1,80 @@
|
|||||||
|
import { expect, test } from "../support/fixtures";
|
||||||
|
import { composerLocator, expectComposerVisible } from "../support/helpers/composer";
|
||||||
|
import {
|
||||||
|
discardDictation,
|
||||||
|
expectEmptyDictation,
|
||||||
|
installSyntheticMicrophone,
|
||||||
|
observeDictation,
|
||||||
|
recordDictation,
|
||||||
|
retryDictation,
|
||||||
|
} from "../support/helpers/dictation";
|
||||||
|
import { openAgentRoute, seedMockAgentWorkspace } from "../support/helpers/mock-agent";
|
||||||
|
|
||||||
|
// Silence is finalized by the real daemon without calling a transcription service.
|
||||||
|
// A closed local endpoint makes an unexpected transcription request fail the test.
|
||||||
|
test.use({
|
||||||
|
e2eDaemonEnvironment: {
|
||||||
|
PASEO_DICTATION_ENABLED: "1",
|
||||||
|
PASEO_DICTATION_SILENCE_PEAK_THRESHOLD: "300",
|
||||||
|
OPENAI_API_KEY: "unused-for-silent-recordings",
|
||||||
|
OPENAI_BASE_URL: "http://127.0.0.1:9/v1",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
test.describe("Dictation empty result", () => {
|
||||||
|
test("the daemon accepts the same recording through repeated empty-result retries", async ({
|
||||||
|
page,
|
||||||
|
}) => {
|
||||||
|
const agent = await seedMockAgentWorkspace({
|
||||||
|
repoPrefix: "dictation-empty-",
|
||||||
|
title: "Empty dictation",
|
||||||
|
});
|
||||||
|
await installSyntheticMicrophone(page, { silent: true });
|
||||||
|
const observed = observeDictation(page);
|
||||||
|
try {
|
||||||
|
await openAgentRoute(page, agent);
|
||||||
|
await expectComposerVisible(page);
|
||||||
|
await recordDictation(page, observed);
|
||||||
|
await expectEmptyDictation(page);
|
||||||
|
await retryDictation(page);
|
||||||
|
await expectEmptyDictation(page);
|
||||||
|
await retryDictation(page);
|
||||||
|
await expectEmptyDictation(page);
|
||||||
|
|
||||||
|
expect(observed.recordings).toHaveLength(3);
|
||||||
|
expect(observed.recordings[0].audio.length).toBeGreaterThan(0);
|
||||||
|
expect(observed.recordings[0].finalSeq).toBe(observed.recordings[0].audio.length - 1);
|
||||||
|
expect(observed.recordings[0].text).toBe("");
|
||||||
|
expect(observed.recordings[1]).toEqual(observed.recordings[0]);
|
||||||
|
expect(observed.recordings[2]).toEqual(observed.recordings[0]);
|
||||||
|
await discardDictation(page);
|
||||||
|
await expect(composerLocator(page)).toBeEditable();
|
||||||
|
await expect(composerLocator(page)).toHaveValue("");
|
||||||
|
} finally {
|
||||||
|
await agent.cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test.describe("narrow composer", () => {
|
||||||
|
test.use({ viewport: { width: 390, height: 844 } });
|
||||||
|
test("explains and discards an empty result", async ({ page }) => {
|
||||||
|
const agent = await seedMockAgentWorkspace({
|
||||||
|
repoPrefix: "dictation-silent-",
|
||||||
|
title: "Silent dictation",
|
||||||
|
});
|
||||||
|
await installSyntheticMicrophone(page, { silent: true });
|
||||||
|
const observed = observeDictation(page);
|
||||||
|
try {
|
||||||
|
await openAgentRoute(page, agent);
|
||||||
|
await expectComposerVisible(page);
|
||||||
|
await recordDictation(page, observed);
|
||||||
|
await expectEmptyDictation(page);
|
||||||
|
await discardDictation(page);
|
||||||
|
await expect(composerLocator(page)).toBeEditable();
|
||||||
|
await expect(composerLocator(page)).toHaveValue("");
|
||||||
|
} finally {
|
||||||
|
await agent.cleanup();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
import { expect, type Page } from "../fixtures";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Replaces the browser microphone with an oscillator. `silent` keeps the stream
|
||||||
|
* running at zero gain, which is what a muted or wrongly routed input delivers.
|
||||||
|
*/
|
||||||
|
export async function installSyntheticMicrophone(
|
||||||
|
page: Page,
|
||||||
|
options: { silent?: boolean } = {},
|
||||||
|
): Promise<void> {
|
||||||
|
await page.addInitScript((silent: boolean) => {
|
||||||
|
const mediaDevices = navigator.mediaDevices;
|
||||||
|
Object.defineProperty(mediaDevices, "getUserMedia", {
|
||||||
|
configurable: true,
|
||||||
|
value: async () => {
|
||||||
|
const context = new AudioContext();
|
||||||
|
const oscillator = context.createOscillator();
|
||||||
|
const gain = context.createGain();
|
||||||
|
gain.gain.value = silent ? 0 : 1;
|
||||||
|
const destination = context.createMediaStreamDestination();
|
||||||
|
oscillator.connect(gain);
|
||||||
|
gain.connect(destination);
|
||||||
|
oscillator.start();
|
||||||
|
return destination.stream;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}, options.silent ?? false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Observe real browser/daemon traffic without intercepting or answering it. */
|
||||||
|
export function observeDictation(page: Page) {
|
||||||
|
const streams = new Map<string, { audio: Map<number, string>; finalSeq: number }>();
|
||||||
|
const recordings: Array<{ audio: string[]; finalSeq: number; text: string }> = [];
|
||||||
|
let resolveAudio!: () => void;
|
||||||
|
const audio = new Promise<void>((resolve) => {
|
||||||
|
resolveAudio = resolve;
|
||||||
|
});
|
||||||
|
|
||||||
|
page.on("websocket", (socket) => {
|
||||||
|
socket.on("framesent", ({ payload }) => {
|
||||||
|
const request = sessionMessage(payload);
|
||||||
|
if (!request || typeof request.dictationId !== "string") return;
|
||||||
|
const id = request.dictationId;
|
||||||
|
if (request.type === "dictation_stream_start") {
|
||||||
|
streams.set(id, { audio: new Map(), finalSeq: -1 });
|
||||||
|
}
|
||||||
|
if (request.type === "dictation_stream_chunk") {
|
||||||
|
streams.get(id)!.audio.set(request.seq as number, request.audio as string);
|
||||||
|
resolveAudio();
|
||||||
|
}
|
||||||
|
if (request.type === "dictation_stream_finish") {
|
||||||
|
streams.get(id)!.finalSeq = request.finalSeq as number;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
socket.on("framereceived", ({ payload }) => {
|
||||||
|
const response = sessionMessage(payload);
|
||||||
|
if (response?.type !== "dictation_stream_final") return;
|
||||||
|
const final = response.payload as { dictationId: string; text: string };
|
||||||
|
const stream = streams.get(final.dictationId)!;
|
||||||
|
recordings.push({
|
||||||
|
audio: [...stream.audio.entries()].sort(([a], [b]) => a - b).map(([, chunk]) => chunk),
|
||||||
|
finalSeq: stream.finalSeq,
|
||||||
|
text: final.text,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return { waitForAudio: () => audio, recordings };
|
||||||
|
}
|
||||||
|
|
||||||
|
function sessionMessage(payload: string | Buffer): Record<string, unknown> | null {
|
||||||
|
try {
|
||||||
|
const envelope = JSON.parse(typeof payload === "string" ? payload : payload.toString("utf8"));
|
||||||
|
return envelope.type === "session" ? envelope.message : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function recordDictation(page: Page, observed: { waitForAudio(): Promise<void> }) {
|
||||||
|
await page.getByRole("button", { name: "Start dictation", exact: true }).click();
|
||||||
|
await observed.waitForAudio();
|
||||||
|
await page.getByRole("button", { name: "Insert transcription", exact: true }).click();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function expectEmptyDictation(page: Page) {
|
||||||
|
const message = page.getByText("No transcription was returned. Tap retry to try again.", {
|
||||||
|
exact: true,
|
||||||
|
});
|
||||||
|
await expect(message).toBeVisible();
|
||||||
|
const bounds = await message.boundingBox();
|
||||||
|
const cancel = await page
|
||||||
|
.getByRole("button", { name: "Cancel dictation", exact: true })
|
||||||
|
.boundingBox();
|
||||||
|
expect(bounds).not.toBeNull();
|
||||||
|
expect(cancel).not.toBeNull();
|
||||||
|
expect(bounds!.y).toBeGreaterThanOrEqual(cancel!.y);
|
||||||
|
expect(bounds!.y + bounds!.height).toBeLessThanOrEqual(cancel!.y + cancel!.height);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function retryDictation(page: Page) {
|
||||||
|
await page.getByRole("button", { name: "Retry dictation", exact: true }).click();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function discardDictation(page: Page) {
|
||||||
|
await page.getByRole("button", { name: "Cancel dictation", exact: true }).click();
|
||||||
|
}
|
||||||
@@ -182,8 +182,8 @@ export async function startIsolatedHostDaemon(
|
|||||||
const spawnDaemon = async (): Promise<ChildProcess> => {
|
const spawnDaemon = async (): Promise<ChildProcess> => {
|
||||||
const spawnOptions: SpawnOptions = {
|
const spawnOptions: SpawnOptions = {
|
||||||
cwd: serverDir,
|
cwd: serverDir,
|
||||||
env: withDisabledE2ESpeechEnv({
|
env: {
|
||||||
...process.env,
|
...withDisabledE2ESpeechEnv(process.env),
|
||||||
...options.environment,
|
...options.environment,
|
||||||
PASEO_HOME: paseoHome,
|
PASEO_HOME: paseoHome,
|
||||||
PASEO_SERVER_ID: serverId,
|
PASEO_SERVER_ID: serverId,
|
||||||
@@ -192,7 +192,7 @@ export async function startIsolatedHostDaemon(
|
|||||||
PASEO_RELAY_ENABLED: options.mutableRelay ? undefined : "0",
|
PASEO_RELAY_ENABLED: options.mutableRelay ? undefined : "0",
|
||||||
PASEO_NODE_ENV: "development",
|
PASEO_NODE_ENV: "development",
|
||||||
NODE_ENV: "development",
|
NODE_ENV: "development",
|
||||||
}),
|
},
|
||||||
stdio: ["ignore", "ignore", "pipe"],
|
stdio: ["ignore", "ignore", "pipe"],
|
||||||
detached: false,
|
detached: false,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -4,9 +4,11 @@ import { View, Text, Pressable } from "react-native";
|
|||||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||||
import { X, ArrowUp, RefreshCcw, Check, Mic, Pencil } from "lucide-react-native";
|
import { X, ArrowUp, RefreshCcw, Check, Mic, Pencil } from "lucide-react-native";
|
||||||
import { useTranslation } from "react-i18next";
|
import { useTranslation } from "react-i18next";
|
||||||
|
import type { TFunction } from "i18next";
|
||||||
import { VolumeMeter } from "./volume-meter";
|
import { VolumeMeter } from "./volume-meter";
|
||||||
import { FOOTER_HEIGHT } from "@/constants/layout";
|
import { FOOTER_HEIGHT } from "@/constants/layout";
|
||||||
import type { DictationStatus } from "@/hooks/use-dictation";
|
import type { DictationStatus } from "@/hooks/use-dictation";
|
||||||
|
import type { DictationEmptyReason } from "@/dictation/result";
|
||||||
|
|
||||||
interface DictationControlsProps {
|
interface DictationControlsProps {
|
||||||
volume: number;
|
volume: number;
|
||||||
@@ -24,6 +26,30 @@ interface DictationControlsProps {
|
|||||||
disabled?: boolean;
|
disabled?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const EMPTY_REASON_MESSAGE_KEYS = {
|
||||||
|
"no-audio": "message.dictation.empty.noAudio",
|
||||||
|
"no-transcript": "message.dictation.empty.noTranscript",
|
||||||
|
} as const satisfies Record<DictationEmptyReason, string>;
|
||||||
|
|
||||||
|
// The line under the meter once an attempt has finished without text.
|
||||||
|
function resolveResultText(input: {
|
||||||
|
t: TFunction;
|
||||||
|
status: DictationStatus;
|
||||||
|
errorText?: string;
|
||||||
|
emptyReason?: DictationEmptyReason | null;
|
||||||
|
}): string | null {
|
||||||
|
const { t, status, errorText, emptyReason } = input;
|
||||||
|
if (status === "empty") {
|
||||||
|
return t(EMPTY_REASON_MESSAGE_KEYS[emptyReason ?? "no-transcript"]);
|
||||||
|
}
|
||||||
|
if (status !== "failed") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return errorText
|
||||||
|
? t("message.dictation.failed", { error: errorText })
|
||||||
|
: t("message.dictation.failedRetry");
|
||||||
|
}
|
||||||
|
|
||||||
function formatDuration(seconds: number): string {
|
function formatDuration(seconds: number): string {
|
||||||
const mins = Math.floor(seconds / 60);
|
const mins = Math.floor(seconds / 60);
|
||||||
const secs = seconds % 60;
|
const secs = seconds % 60;
|
||||||
@@ -147,18 +173,26 @@ export function DictationOverlay({
|
|||||||
isProcessing,
|
isProcessing,
|
||||||
status,
|
status,
|
||||||
errorText,
|
errorText,
|
||||||
|
emptyReason,
|
||||||
onCancel,
|
onCancel,
|
||||||
onAccept,
|
onAccept,
|
||||||
onAcceptAndSend,
|
onAcceptAndSend,
|
||||||
onRetry,
|
onRetry,
|
||||||
onDiscard,
|
onDiscard,
|
||||||
}: Omit<DictationControlsProps, "onStart" | "disabled" | "transcript"> & { errorText?: string }) {
|
}: Omit<DictationControlsProps, "onStart" | "disabled" | "transcript"> & {
|
||||||
|
errorText?: string;
|
||||||
|
emptyReason?: DictationEmptyReason | null;
|
||||||
|
}) {
|
||||||
const { theme } = useUnistyles();
|
const { theme } = useUnistyles();
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const isFailed = status === "failed";
|
const isEmpty = status === "empty";
|
||||||
|
// An empty result is a finished attempt the user has to acknowledge, like a failure.
|
||||||
|
const isFailed = status === "failed" || isEmpty;
|
||||||
|
const canRetry = !isEmpty || emptyReason !== "no-audio";
|
||||||
const showActiveState = isRecording || isProcessing || isFailed;
|
const showActiveState = isRecording || isProcessing || isFailed;
|
||||||
const actionsDisabled = isProcessing;
|
const actionsDisabled = isProcessing;
|
||||||
const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
|
const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
|
||||||
|
const resultText = resolveResultText({ t, status, errorText, emptyReason });
|
||||||
|
|
||||||
const containerStyle = useMemo(
|
const containerStyle = useMemo(
|
||||||
() => [overlayStyles.container, { backgroundColor: theme.colors.accent }],
|
() => [overlayStyles.container, { backgroundColor: theme.colors.accent }],
|
||||||
@@ -202,26 +236,26 @@ export function DictationOverlay({
|
|||||||
</Pressable>
|
</Pressable>
|
||||||
|
|
||||||
<View style={overlayStyles.centerContainer}>
|
<View style={overlayStyles.centerContainer}>
|
||||||
<View style={overlayStyles.meterRow}>
|
{!resultText ? (
|
||||||
<VolumeMeter
|
<View style={overlayStyles.meterRow}>
|
||||||
volume={volume}
|
<VolumeMeter
|
||||||
isMuted={false}
|
volume={volume}
|
||||||
isSpeaking={false}
|
isMuted={false}
|
||||||
orientation="horizontal"
|
isSpeaking={false}
|
||||||
color={theme.colors.accentForeground}
|
orientation="horizontal"
|
||||||
/>
|
color={theme.colors.accentForeground}
|
||||||
<Text style={overlayTimerTextStyle}>{formatDuration(duration)}</Text>
|
/>
|
||||||
</View>
|
<Text style={overlayTimerTextStyle}>{formatDuration(duration)}</Text>
|
||||||
|
</View>
|
||||||
|
) : null}
|
||||||
{status === "paused" ? (
|
{status === "paused" ? (
|
||||||
<Text numberOfLines={2} style={overlayTranscriptTextStyle}>
|
<Text numberOfLines={2} style={overlayTranscriptTextStyle}>
|
||||||
{t("message.dictation.paused")}
|
{t("message.dictation.paused")}
|
||||||
</Text>
|
</Text>
|
||||||
) : null}
|
) : null}
|
||||||
{isFailed ? (
|
{resultText ? (
|
||||||
<Text numberOfLines={2} style={overlayTranscriptTextStyle}>
|
<Text numberOfLines={2} style={overlayTranscriptTextStyle}>
|
||||||
{errorText
|
{resultText}
|
||||||
? t("message.dictation.failed", { error: errorText })
|
|
||||||
: t("message.dictation.failedRetry")}
|
|
||||||
</Text>
|
</Text>
|
||||||
) : null}
|
) : null}
|
||||||
</View>
|
</View>
|
||||||
@@ -232,7 +266,7 @@ export function DictationOverlay({
|
|||||||
<LoadingSpinner size="small" color={theme.colors.accentForeground} />
|
<LoadingSpinner size="small" color={theme.colors.accentForeground} />
|
||||||
</View>
|
</View>
|
||||||
) : null}
|
) : null}
|
||||||
{!actionsDisabled && isFailed ? (
|
{!actionsDisabled && isFailed && canRetry ? (
|
||||||
<Pressable
|
<Pressable
|
||||||
onPress={onRetry}
|
onPress={onRetry}
|
||||||
accessibilityRole="button"
|
accessibilityRole="button"
|
||||||
|
|||||||
@@ -562,6 +562,7 @@ function MessageInputOverlay({
|
|||||||
isDictationProcessing,
|
isDictationProcessing,
|
||||||
dictationStatus,
|
dictationStatus,
|
||||||
dictationError,
|
dictationError,
|
||||||
|
dictationEmptyReason,
|
||||||
onCancelRecording,
|
onCancelRecording,
|
||||||
onAcceptRecording,
|
onAcceptRecording,
|
||||||
onAcceptAndSendRecording,
|
onAcceptAndSendRecording,
|
||||||
@@ -585,6 +586,7 @@ function MessageInputOverlay({
|
|||||||
isDictationProcessing: boolean;
|
isDictationProcessing: boolean;
|
||||||
dictationStatus: React.ComponentProps<typeof DictationOverlay>["status"];
|
dictationStatus: React.ComponentProps<typeof DictationOverlay>["status"];
|
||||||
dictationError: string | null;
|
dictationError: string | null;
|
||||||
|
dictationEmptyReason: React.ComponentProps<typeof DictationOverlay>["emptyReason"];
|
||||||
onCancelRecording: () => Promise<void>;
|
onCancelRecording: () => Promise<void>;
|
||||||
onAcceptRecording: () => Promise<void>;
|
onAcceptRecording: () => Promise<void>;
|
||||||
onAcceptAndSendRecording: () => Promise<void>;
|
onAcceptAndSendRecording: () => Promise<void>;
|
||||||
@@ -593,6 +595,7 @@ function MessageInputOverlay({
|
|||||||
onRealtimeVoiceStop: () => void;
|
onRealtimeVoiceStop: () => void;
|
||||||
}) {
|
}) {
|
||||||
if (showDictationOverlay) {
|
if (showDictationOverlay) {
|
||||||
|
const awaitsAcknowledgement = dictationStatus === "failed" || dictationStatus === "empty";
|
||||||
return (
|
return (
|
||||||
<DictationOverlay
|
<DictationOverlay
|
||||||
volume={dictationVolume}
|
volume={dictationVolume}
|
||||||
@@ -601,11 +604,12 @@ function MessageInputOverlay({
|
|||||||
isProcessing={isDictationProcessing}
|
isProcessing={isDictationProcessing}
|
||||||
status={dictationStatus}
|
status={dictationStatus}
|
||||||
errorText={dictationStatus === "failed" ? (dictationError ?? undefined) : undefined}
|
errorText={dictationStatus === "failed" ? (dictationError ?? undefined) : undefined}
|
||||||
|
emptyReason={dictationEmptyReason}
|
||||||
onCancel={onCancelRecording}
|
onCancel={onCancelRecording}
|
||||||
onAccept={onAcceptRecording}
|
onAccept={onAcceptRecording}
|
||||||
onAcceptAndSend={onAcceptAndSendRecording}
|
onAcceptAndSend={onAcceptAndSendRecording}
|
||||||
onRetry={dictationStatus === "failed" ? onRetryFailedRecording : undefined}
|
onRetry={awaitsAcknowledgement ? onRetryFailedRecording : undefined}
|
||||||
onDiscard={dictationStatus === "failed" ? onDiscardFailedRecording : undefined}
|
onDiscard={awaitsAcknowledgement ? onDiscardFailedRecording : undefined}
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1032,7 +1036,12 @@ function computeShouldShowDictationOverlay(
|
|||||||
isDictationProcessing: boolean,
|
isDictationProcessing: boolean,
|
||||||
dictationStatus: string,
|
dictationStatus: string,
|
||||||
): boolean {
|
): boolean {
|
||||||
return isDictating || isDictationProcessing || dictationStatus === "failed";
|
return (
|
||||||
|
isDictating ||
|
||||||
|
isDictationProcessing ||
|
||||||
|
dictationStatus === "failed" ||
|
||||||
|
dictationStatus === "empty"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function computeIsDictationStartEnabled(
|
function computeIsDictationStartEnabled(
|
||||||
@@ -1533,6 +1542,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
duration: dictationDuration,
|
duration: dictationDuration,
|
||||||
error: dictationError,
|
error: dictationError,
|
||||||
status: dictationStatus,
|
status: dictationStatus,
|
||||||
|
emptyReason: dictationEmptyReason,
|
||||||
startDictation,
|
startDictation,
|
||||||
cancelDictation,
|
cancelDictation,
|
||||||
confirmDictation,
|
confirmDictation,
|
||||||
@@ -2180,6 +2190,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
isDictationProcessing={isDictationProcessing}
|
isDictationProcessing={isDictationProcessing}
|
||||||
dictationStatus={dictationStatus}
|
dictationStatus={dictationStatus}
|
||||||
dictationError={dictationError}
|
dictationError={dictationError}
|
||||||
|
dictationEmptyReason={dictationEmptyReason}
|
||||||
onCancelRecording={handleCancelRecording}
|
onCancelRecording={handleCancelRecording}
|
||||||
onAcceptRecording={handleAcceptRecording}
|
onAcceptRecording={handleAcceptRecording}
|
||||||
onAcceptAndSendRecording={handleAcceptAndSendRecording}
|
onAcceptAndSendRecording={handleAcceptAndSendRecording}
|
||||||
@@ -2201,6 +2212,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
|||||||
isDictationProcessing,
|
isDictationProcessing,
|
||||||
dictationStatus,
|
dictationStatus,
|
||||||
dictationError,
|
dictationError,
|
||||||
|
dictationEmptyReason,
|
||||||
handleCancelRecording,
|
handleCancelRecording,
|
||||||
handleAcceptRecording,
|
handleAcceptRecording,
|
||||||
handleAcceptAndSendRecording,
|
handleAcceptAndSendRecording,
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { completeDictation, type DictationRecording } from "./result";
|
||||||
|
|
||||||
|
class RecordedAudio implements DictationRecording {
|
||||||
|
constructor(private segments: string[]) {}
|
||||||
|
getSegmentCount() {
|
||||||
|
return this.segments.length;
|
||||||
|
}
|
||||||
|
clearAll() {
|
||||||
|
this.segments = [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("completing dictation", () => {
|
||||||
|
it("keeps a recording through repeated empty results until a retry returns text", () => {
|
||||||
|
const recording = new RecordedAudio(["first PCM chunk", "second PCM chunk"]);
|
||||||
|
const empty = { finalText: " ", partialText: "", recording };
|
||||||
|
const result = completeDictation(empty);
|
||||||
|
expect(recording.getSegmentCount()).toBe(2);
|
||||||
|
expect(result).toEqual({ status: "empty", reason: "no-transcript" });
|
||||||
|
expect(completeDictation(empty)).toEqual({ status: "empty", reason: "no-transcript" });
|
||||||
|
expect(recording.getSegmentCount()).toBe(2);
|
||||||
|
expect(completeDictation({ ...empty, finalText: " recovered " })).toEqual({
|
||||||
|
status: "transcribed",
|
||||||
|
text: "recovered",
|
||||||
|
});
|
||||||
|
expect(recording.getSegmentCount()).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps usable partial text when the final response is empty", () => {
|
||||||
|
const recording = new RecordedAudio(["PCM chunk"]);
|
||||||
|
expect(
|
||||||
|
completeDictation({ finalText: "", partialText: " recognized words ", recording }),
|
||||||
|
).toEqual({
|
||||||
|
status: "transcribed",
|
||||||
|
text: "recognized words",
|
||||||
|
});
|
||||||
|
expect(recording.getSegmentCount()).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a missing recording without inventing a reason for the empty result", () => {
|
||||||
|
expect(
|
||||||
|
completeDictation({ finalText: "", partialText: "", recording: new RecordedAudio([]) }),
|
||||||
|
).toEqual({
|
||||||
|
status: "empty",
|
||||||
|
reason: "no-audio",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
export type DictationEmptyReason = "no-audio" | "no-transcript";
|
||||||
|
|
||||||
|
export interface DictationRecording {
|
||||||
|
getSegmentCount(): number;
|
||||||
|
clearAll(): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DictationResult =
|
||||||
|
| { status: "empty"; reason: DictationEmptyReason }
|
||||||
|
| { status: "transcribed"; text: string };
|
||||||
|
|
||||||
|
export function completeDictation(input: {
|
||||||
|
finalText: string;
|
||||||
|
partialText: string;
|
||||||
|
recording: DictationRecording;
|
||||||
|
}): DictationResult {
|
||||||
|
const text = input.finalText.trim() || input.partialText.trim();
|
||||||
|
if (!text) {
|
||||||
|
return {
|
||||||
|
status: "empty",
|
||||||
|
reason: input.recording.getSegmentCount() === 0 ? "no-audio" : "no-transcript",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
input.recording.clearAll();
|
||||||
|
return { status: "transcribed", text };
|
||||||
|
}
|
||||||
@@ -1,6 +1,14 @@
|
|||||||
|
import type { DictationEmptyReason } from "@/dictation/result";
|
||||||
import { i18n } from "@/i18n/i18next";
|
import { i18n } from "@/i18n/i18next";
|
||||||
|
|
||||||
export type DictationStatus = "idle" | "recording" | "paused" | "uploading" | "failed";
|
export type DictationStatus =
|
||||||
|
| "idle"
|
||||||
|
| "recording"
|
||||||
|
| "paused"
|
||||||
|
| "uploading"
|
||||||
|
| "failed"
|
||||||
|
// Finished without text; captured audio remains available for retry.
|
||||||
|
| "empty";
|
||||||
|
|
||||||
export interface UseDictationOptions {
|
export interface UseDictationOptions {
|
||||||
client: import("@getpaseo/client/internal/daemon-client").DaemonClient | null;
|
client: import("@getpaseo/client/internal/daemon-client").DaemonClient | null;
|
||||||
@@ -22,6 +30,7 @@ export interface UseDictationResult {
|
|||||||
duration: number;
|
duration: number;
|
||||||
error: string | null;
|
error: string | null;
|
||||||
status: DictationStatus;
|
status: DictationStatus;
|
||||||
|
emptyReason: DictationEmptyReason | null;
|
||||||
startDictation: () => Promise<void>;
|
startDictation: () => Promise<void>;
|
||||||
cancelDictation: () => Promise<void>;
|
cancelDictation: () => Promise<void>;
|
||||||
confirmDictation: () => Promise<void>;
|
confirmDictation: () => Promise<void>;
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
|
|||||||
import { useTranslation } from "react-i18next";
|
import { useTranslation } from "react-i18next";
|
||||||
|
|
||||||
import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
|
import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
|
||||||
|
import { completeDictation, type DictationEmptyReason } from "@/dictation/result";
|
||||||
import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source";
|
import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source";
|
||||||
import { generateMessageId } from "@/types/stream";
|
import { generateMessageId } from "@/types/stream";
|
||||||
import { AttemptGuard } from "@/utils/attempt-guard";
|
import { AttemptGuard } from "@/utils/attempt-guard";
|
||||||
@@ -33,6 +34,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
const [duration, setDuration] = useState(0);
|
const [duration, setDuration] = useState(0);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const [status, setStatus] = useState<DictationStatus>("idle");
|
const [status, setStatus] = useState<DictationStatus>("idle");
|
||||||
|
const [emptyReason, setEmptyReason] = useState<DictationEmptyReason | null>(null);
|
||||||
const latestPartialTranscriptRef = useRef("");
|
const latestPartialTranscriptRef = useRef("");
|
||||||
|
|
||||||
const onTranscriptRef = useRef(onTranscript);
|
const onTranscriptRef = useRef(onTranscript);
|
||||||
@@ -131,12 +133,17 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
[setError],
|
[setError],
|
||||||
);
|
);
|
||||||
|
|
||||||
const clearStreamingState = useCallback(() => {
|
const clearTranscriptState = useCallback(() => {
|
||||||
senderRef.current?.clearAll();
|
|
||||||
latestPartialTranscriptRef.current = "";
|
latestPartialTranscriptRef.current = "";
|
||||||
setPartialTranscript("");
|
setPartialTranscript("");
|
||||||
|
setEmptyReason(null);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const clearStreamingState = useCallback(() => {
|
||||||
|
senderRef.current?.clearAll();
|
||||||
|
clearTranscriptState();
|
||||||
|
}, [clearTranscriptState]);
|
||||||
|
|
||||||
const startNewStream = useCallback(async (reason: string) => {
|
const startNewStream = useCallback(async (reason: string) => {
|
||||||
await senderRef.current?.restartStream(reason);
|
await senderRef.current?.restartStream(reason);
|
||||||
}, []);
|
}, []);
|
||||||
@@ -192,16 +199,20 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
setDuration(0);
|
setDuration(0);
|
||||||
setStatus("idle");
|
setStatus("idle");
|
||||||
|
|
||||||
const transcriptText =
|
const result = completeDictation({
|
||||||
text.trim().length > 0 ? text.trim() : latestPartialTranscriptRef.current.trim();
|
finalText: text,
|
||||||
clearStreamingState();
|
partialText: latestPartialTranscriptRef.current,
|
||||||
|
recording: senderRef.current!,
|
||||||
if (!transcriptText) {
|
});
|
||||||
|
if (result.status === "empty") {
|
||||||
|
setStatus("empty");
|
||||||
|
setEmptyReason(result.reason);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
onTranscriptRef.current?.(transcriptText, { requestId });
|
clearTranscriptState();
|
||||||
|
onTranscriptRef.current?.(result.text, { requestId });
|
||||||
},
|
},
|
||||||
[clearStreamingState],
|
[clearTranscriptState],
|
||||||
);
|
);
|
||||||
|
|
||||||
const handleDictationFailure = useCallback(
|
const handleDictationFailure = useCallback(
|
||||||
@@ -395,6 +406,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
setError(null);
|
setError(null);
|
||||||
|
setEmptyReason(null);
|
||||||
setStatus("uploading");
|
setStatus("uploading");
|
||||||
setIsProcessing(true);
|
setIsProcessing(true);
|
||||||
isProcessingRef.current = true;
|
isProcessingRef.current = true;
|
||||||
@@ -462,6 +474,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
|||||||
duration,
|
duration,
|
||||||
error,
|
error,
|
||||||
status: status === "recording" && audio.isPaused ? "paused" : status,
|
status: status === "recording" && audio.isPaused ? "paused" : status,
|
||||||
|
emptyReason,
|
||||||
startDictation,
|
startDictation,
|
||||||
cancelDictation,
|
cancelDictation,
|
||||||
confirmDictation,
|
confirmDictation,
|
||||||
|
|||||||
@@ -371,6 +371,10 @@ export const ar: TranslationResources = {
|
|||||||
insertAndSend: "أدخل النسخ وأرسل",
|
insertAndSend: "أدخل النسخ وأرسل",
|
||||||
failed: "فشل الإملاء:{{error}}",
|
failed: "فشل الإملاء:{{error}}",
|
||||||
failedRetry: "فشل الإملاء. اضغط على إعادة المحاولة.",
|
failedRetry: "فشل الإملاء. اضغط على إعادة المحاولة.",
|
||||||
|
empty: {
|
||||||
|
noAudio: "لم يُسجَّل أي صوت. هل يستخدم تطبيق آخر الميكروفون؟",
|
||||||
|
noTranscript: "لم يتم إرجاع نص. اضغط على إعادة المحاولة.",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "تعذر إرسال ردك. حاول مرة أخرى.",
|
responseFailed: "تعذر إرسال ردك. حاول مرة أخرى.",
|
||||||
|
|||||||
@@ -368,6 +368,10 @@ export const en = {
|
|||||||
insertAndSend: "Insert transcription and send",
|
insertAndSend: "Insert transcription and send",
|
||||||
failed: "Dictation failed: {{error}}",
|
failed: "Dictation failed: {{error}}",
|
||||||
failedRetry: "Dictation failed. Tap retry.",
|
failedRetry: "Dictation failed. Tap retry.",
|
||||||
|
empty: {
|
||||||
|
noAudio: "No audio was recorded. Is another app using the microphone?",
|
||||||
|
noTranscript: "No transcription was returned. Tap retry to try again.",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "Could not send your response. Try again.",
|
responseFailed: "Could not send your response. Try again.",
|
||||||
|
|||||||
@@ -376,6 +376,10 @@ export const es: TranslationResources = {
|
|||||||
insertAndSend: "Insertar transcripción y enviar",
|
insertAndSend: "Insertar transcripción y enviar",
|
||||||
failed: "Fallo en el dictado:{{error}}",
|
failed: "Fallo en el dictado:{{error}}",
|
||||||
failedRetry: "El dictado falló. Toca reintentar.",
|
failedRetry: "El dictado falló. Toca reintentar.",
|
||||||
|
empty: {
|
||||||
|
noAudio: "No se grabó audio. ¿Otra app está usando el micrófono?",
|
||||||
|
noTranscript: "No se devolvió ninguna transcripción. Toca reintentar.",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "No se pudo enviar tu respuesta. Inténtalo de nuevo.",
|
responseFailed: "No se pudo enviar tu respuesta. Inténtalo de nuevo.",
|
||||||
|
|||||||
@@ -376,6 +376,10 @@ export const fr: TranslationResources = {
|
|||||||
insertAndSend: "Insérer la transcription et envoyer",
|
insertAndSend: "Insérer la transcription et envoyer",
|
||||||
failed: "Échec de la dictée : {{error}}",
|
failed: "Échec de la dictée : {{error}}",
|
||||||
failedRetry: "La dictée a échoué. Appuyez sur Réessayer.",
|
failedRetry: "La dictée a échoué. Appuyez sur Réessayer.",
|
||||||
|
empty: {
|
||||||
|
noAudio: "Aucun son n'a été enregistré. Une autre app utilise-t-elle le micro ?",
|
||||||
|
noTranscript: "Aucune transcription reçue. Appuyez sur Réessayer.",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "Impossible d’envoyer votre réponse. Réessayez.",
|
responseFailed: "Impossible d’envoyer votre réponse. Réessayez.",
|
||||||
|
|||||||
@@ -376,6 +376,10 @@ export const ja: TranslationResources = {
|
|||||||
insertAndSend: "文字起こしを挿入して送信",
|
insertAndSend: "文字起こしを挿入して送信",
|
||||||
failed: "音声入力に失敗しました: {{error}}",
|
failed: "音声入力に失敗しました: {{error}}",
|
||||||
failedRetry: "音声入力に失敗しました。再試行をタップしてください。",
|
failedRetry: "音声入力に失敗しました。再試行をタップしてください。",
|
||||||
|
empty: {
|
||||||
|
noAudio: "音声が録音されませんでした。別のアプリがマイクを使用していませんか?",
|
||||||
|
noTranscript: "文字起こしが返されませんでした。再試行をタップしてください。",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "回答を送信できませんでした。もう一度お試しください。",
|
responseFailed: "回答を送信できませんでした。もう一度お試しください。",
|
||||||
|
|||||||
@@ -372,6 +372,10 @@ export const ko: TranslationResources = {
|
|||||||
insertAndSend: "전사 삽입 후 보내기",
|
insertAndSend: "전사 삽입 후 보내기",
|
||||||
failed: "받아쓰기 실패: {{error}}",
|
failed: "받아쓰기 실패: {{error}}",
|
||||||
failedRetry: "받아쓰기에 실패했습니다. 다시 시도하려면 누르세요.",
|
failedRetry: "받아쓰기에 실패했습니다. 다시 시도하려면 누르세요.",
|
||||||
|
empty: {
|
||||||
|
noAudio: "녹음된 오디오가 없습니다. 다른 앱이 마이크를 사용 중인가요?",
|
||||||
|
noTranscript: "전사 결과가 없습니다. 다시 시도하려면 누르세요.",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "응답을 보내지 못했습니다. 다시 시도해 주세요.",
|
responseFailed: "응답을 보내지 못했습니다. 다시 시도해 주세요.",
|
||||||
|
|||||||
@@ -375,6 +375,10 @@ export const ptBR: TranslationResources = {
|
|||||||
insertAndSend: "Inserir transcrição e enviar",
|
insertAndSend: "Inserir transcrição e enviar",
|
||||||
failed: "Falha no ditado: {{error}}",
|
failed: "Falha no ditado: {{error}}",
|
||||||
failedRetry: "Falha no ditado. Toque para tentar novamente.",
|
failedRetry: "Falha no ditado. Toque para tentar novamente.",
|
||||||
|
empty: {
|
||||||
|
noAudio: "Nenhum áudio foi gravado. Outro app está usando o microfone?",
|
||||||
|
noTranscript: "Nenhuma transcrição foi retornada. Toque para tentar novamente.",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "Não foi possível enviar sua resposta. Tente novamente.",
|
responseFailed: "Não foi possível enviar sua resposta. Tente novamente.",
|
||||||
|
|||||||
@@ -375,6 +375,10 @@ export const ru: TranslationResources = {
|
|||||||
insertAndSend: "Вставить транскрипцию и отправить",
|
insertAndSend: "Вставить транскрипцию и отправить",
|
||||||
failed: "Диктовка не удалась: {{error}}",
|
failed: "Диктовка не удалась: {{error}}",
|
||||||
failedRetry: "Диктовка не удалась. Нажмите «Повторить».",
|
failedRetry: "Диктовка не удалась. Нажмите «Повторить».",
|
||||||
|
empty: {
|
||||||
|
noAudio: "Звук не записан. Возможно, микрофон занят другим приложением?",
|
||||||
|
noTranscript: "Текст не получен. Нажмите «Повторить».",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "Не удалось отправить ответ. Попробуйте ещё раз.",
|
responseFailed: "Не удалось отправить ответ. Попробуйте ещё раз.",
|
||||||
|
|||||||
@@ -371,6 +371,10 @@ export const zhCN: TranslationResources = {
|
|||||||
insertAndSend: "插入转写并发送",
|
insertAndSend: "插入转写并发送",
|
||||||
failed: "听写失败:{{error}}",
|
failed: "听写失败:{{error}}",
|
||||||
failedRetry: "听写失败。点按重试。",
|
failedRetry: "听写失败。点按重试。",
|
||||||
|
empty: {
|
||||||
|
noAudio: "未录到音频。是否有其他应用正在使用麦克风?",
|
||||||
|
noTranscript: "未返回转写文本。点按重试。",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
question: {
|
question: {
|
||||||
responseFailed: "未能发送你的回复,请重试。",
|
responseFailed: "未能发送你的回复,请重试。",
|
||||||
|
|||||||
Reference in New Issue
Block a user