mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Stream dictation via OpenAI Realtime and simplify retry UI
This commit is contained in:
Mohamed Boudra
@@ -18,7 +18,6 @@ interface DictationControlsProps {
|
||||
onRetry?: () => void;
|
||||
onDiscard?: () => void;
|
||||
disabled?: boolean;
|
||||
retryStatusText?: string;
|
||||
}
|
||||
|
||||
function formatDuration(seconds: number): string {
|
||||
@@ -42,13 +41,11 @@ export function DictationControls({
|
||||
onRetry,
|
||||
onDiscard,
|
||||
disabled = false,
|
||||
retryStatusText,
|
||||
}: DictationControlsProps) {
|
||||
const { theme } = useUnistyles();
|
||||
const isRetrying = status === "retrying";
|
||||
const isFailed = status === "failed";
|
||||
const showActiveState = isRecording || isProcessing || isRetrying || isFailed;
|
||||
const actionsDisabled = isProcessing || isRetrying;
|
||||
const showActiveState = isRecording || isProcessing || isFailed;
|
||||
const actionsDisabled = isProcessing;
|
||||
const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
|
||||
|
||||
if (!showActiveState) {
|
||||
@@ -123,13 +120,6 @@ export function DictationControls({
|
||||
</>
|
||||
)}
|
||||
</View>
|
||||
{retryStatusText && (isRetrying || isFailed) && (
|
||||
<Text
|
||||
style={[styles.statusLabel, { color: theme.colors.mutedForeground }]}
|
||||
>
|
||||
{retryStatusText}
|
||||
</Text>
|
||||
)}
|
||||
</View>
|
||||
);
|
||||
}
|
||||
@@ -149,12 +139,11 @@ export function DictationOverlay({
|
||||
onAcceptAndSend,
|
||||
onRetry,
|
||||
onDiscard,
|
||||
}: Omit<DictationControlsProps, "onStart" | "disabled" | "retryStatusText">) {
|
||||
}: Omit<DictationControlsProps, "onStart" | "disabled">) {
|
||||
const { theme } = useUnistyles();
|
||||
const isRetrying = status === "retrying";
|
||||
const isFailed = status === "failed";
|
||||
const showActiveState = isRecording || isProcessing || isRetrying || isFailed;
|
||||
const actionsDisabled = isProcessing || isRetrying;
|
||||
const showActiveState = isRecording || isProcessing || isFailed;
|
||||
const actionsDisabled = isProcessing;
|
||||
const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
|
||||
|
||||
if (!showActiveState) {
|
||||
|
||||
@@ -210,7 +210,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (isDictating || isDictationProcessing || dictationStatus === "retrying") {
|
||||
if (isDictating || isDictationProcessing) {
|
||||
return;
|
||||
}
|
||||
sendAfterTranscriptRef.current = false;
|
||||
@@ -246,7 +246,6 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(
|
||||
const showOverlay =
|
||||
isDictating ||
|
||||
isDictationProcessing ||
|
||||
dictationStatus === "retrying" ||
|
||||
dictationStatus === "failed";
|
||||
overlayTransition.value = withTiming(showOverlay ? 1 : 0, {
|
||||
duration: 200,
|
||||
|
||||
@@ -88,7 +88,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
|
||||
return;
|
||||
}
|
||||
|
||||
if (volume > 0.01) {
|
||||
if (volume > 0.001) {
|
||||
// Active volume - animate heights based on volume
|
||||
const target1 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.2);
|
||||
const target2 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.05);
|
||||
@@ -136,7 +136,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
|
||||
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
|
||||
const volumeBoost = isMuted ? 0 : volume * 0.3;
|
||||
return {
|
||||
height: line1Height.value * (isMuted || volume > 0.01 ? 1 : line1Pulse.value),
|
||||
height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value),
|
||||
opacity: baseOpacity + volumeBoost,
|
||||
};
|
||||
});
|
||||
@@ -146,7 +146,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
|
||||
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
|
||||
const volumeBoost = isMuted ? 0 : volume * 0.3;
|
||||
return {
|
||||
height: line2Height.value * (isMuted || volume > 0.01 ? 1 : line2Pulse.value),
|
||||
height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value),
|
||||
opacity: baseOpacity + volumeBoost,
|
||||
};
|
||||
});
|
||||
@@ -156,7 +156,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
|
||||
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
|
||||
const volumeBoost = isMuted ? 0 : volume * 0.3;
|
||||
return {
|
||||
height: line3Height.value * (isMuted || volume > 0.01 ? 1 : line3Pulse.value),
|
||||
height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value),
|
||||
opacity: baseOpacity + volumeBoost,
|
||||
};
|
||||
});
|
||||
|
||||
126
packages/app/src/dictation/dictation-stream-sender.test.ts
Normal file
126
packages/app/src/dictation/dictation-stream-sender.test.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
|
||||
|
||||
type FakeFinish = { dictationId: string; finalSeq: number };
|
||||
type FakeStart = { dictationId: string; format: string };
|
||||
type FakeChunk = { dictationId: string; seq: number; audio: string; format: string };
|
||||
|
||||
class FakeDaemonClient {
|
||||
isConnected = true;
|
||||
starts: FakeStart[] = [];
|
||||
chunks: FakeChunk[] = [];
|
||||
finishes: FakeFinish[] = [];
|
||||
cancels: string[] = [];
|
||||
|
||||
async startDictationStream(dictationId: string, format: string): Promise<void> {
|
||||
this.starts.push({ dictationId, format });
|
||||
}
|
||||
|
||||
sendDictationStreamChunk(dictationId: string, seq: number, audio: string, format: string): void {
|
||||
this.chunks.push({ dictationId, seq, audio, format });
|
||||
}
|
||||
|
||||
async finishDictationStream(dictationId: string, finalSeq: number): Promise<{ dictationId: string; text: string }> {
|
||||
this.finishes.push({ dictationId, finalSeq });
|
||||
return { dictationId, text: "ok" };
|
||||
}
|
||||
|
||||
cancelDictationStream(dictationId: string): void {
|
||||
this.cancels.push(dictationId);
|
||||
}
|
||||
}
|
||||
|
||||
const tick = async () => {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
};
|
||||
|
||||
describe("DictationStreamSender", () => {
|
||||
it("enqueues segments and sends them after stream start", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
const ids = ["d1"];
|
||||
const sender = new DictationStreamSender({
|
||||
client: client as any,
|
||||
format: "audio/pcm;rate=16000;bits=16",
|
||||
createDictationId: () => ids.shift() ?? "dX",
|
||||
});
|
||||
|
||||
sender.enqueueSegment("seg0");
|
||||
sender.enqueueSegment("seg1");
|
||||
|
||||
await tick();
|
||||
|
||||
expect(client.starts).toEqual([{ dictationId: "d1", format: "audio/pcm;rate=16000;bits=16" }]);
|
||||
expect(client.chunks).toEqual([
|
||||
{ dictationId: "d1", seq: 0, audio: "seg0", format: "audio/pcm;rate=16000;bits=16" },
|
||||
{ dictationId: "d1", seq: 1, audio: "seg1", format: "audio/pcm;rate=16000;bits=16" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("restarts stream and resends from seq=0 on reconnect", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
const ids = ["d1", "d2"];
|
||||
const sender = new DictationStreamSender({
|
||||
client: client as any,
|
||||
format: "audio/pcm;rate=16000;bits=16",
|
||||
createDictationId: () => ids.shift() ?? "dX",
|
||||
});
|
||||
|
||||
sender.enqueueSegment("seg0");
|
||||
sender.enqueueSegment("seg1");
|
||||
await tick();
|
||||
|
||||
await sender.restartStream("reconnect");
|
||||
|
||||
expect(client.starts.map((s) => s.dictationId)).toEqual(["d1", "d2"]);
|
||||
const d2Chunks = client.chunks.filter((c) => c.dictationId === "d2");
|
||||
expect(d2Chunks.map((c) => [c.seq, c.audio])).toEqual([
|
||||
[0, "seg0"],
|
||||
[1, "seg1"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("finish flushes all queued segments and sends finish with finalSeq", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
const ids = ["d1"];
|
||||
const sender = new DictationStreamSender({
|
||||
client: client as any,
|
||||
format: "audio/pcm;rate=16000;bits=16",
|
||||
createDictationId: () => ids.shift() ?? "dX",
|
||||
});
|
||||
|
||||
sender.enqueueSegment("seg0");
|
||||
sender.enqueueSegment("seg1");
|
||||
|
||||
const finalSeq = sender.getFinalSeq();
|
||||
const result = await sender.finish(finalSeq);
|
||||
|
||||
expect(result.text).toBe("ok");
|
||||
expect(client.chunks.map((c) => c.seq)).toEqual([0, 1]);
|
||||
expect(client.finishes).toEqual([{ dictationId: "d1", finalSeq: 1 }]);
|
||||
});
|
||||
|
||||
it("keeps segments while disconnected and sends them after restart when reconnected", async () => {
|
||||
const client = new FakeDaemonClient();
|
||||
client.isConnected = false;
|
||||
const ids = ["d1"];
|
||||
const sender = new DictationStreamSender({
|
||||
client: client as any,
|
||||
format: "audio/pcm;rate=16000;bits=16",
|
||||
createDictationId: () => ids.shift() ?? "dX",
|
||||
});
|
||||
|
||||
sender.enqueueSegment("seg0");
|
||||
sender.enqueueSegment("seg1");
|
||||
|
||||
expect(client.starts).toHaveLength(0);
|
||||
expect(client.chunks).toHaveLength(0);
|
||||
|
||||
client.isConnected = true;
|
||||
await sender.restartStream("reconnect");
|
||||
|
||||
expect(client.chunks.map((c) => c.seq)).toEqual([0, 1]);
|
||||
});
|
||||
});
|
||||
|
||||
189
packages/app/src/dictation/dictation-stream-sender.ts
Normal file
189
packages/app/src/dictation/dictation-stream-sender.ts
Normal file
@@ -0,0 +1,189 @@
|
||||
import { generateMessageId } from "@/types/stream";
|
||||
import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
|
||||
|
||||
export type DictationStreamSenderParams = {
|
||||
client: DaemonClientV2 | null;
|
||||
format: string;
|
||||
createDictationId?: () => string;
|
||||
};
|
||||
|
||||
type DictationFinishResult = { dictationId: string; text: string };
|
||||
|
||||
/**
|
||||
* Small, non-React state machine for dictation streaming.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Maintain an ordered buffer of base64 PCM segments
|
||||
* - Start/restart a dictation stream (dictationId)
|
||||
* - Send missing segments (seq) when connected
|
||||
* - Finish/cancel the stream
|
||||
*
|
||||
* This class intentionally keeps sending synchronous (no internal async mutex),
|
||||
* so enqueues can't "miss" a flush due to in-flight await/coalescing bugs.
|
||||
*/
|
||||
export class DictationStreamSender {
|
||||
private client: DaemonClientV2 | null;
|
||||
private readonly format: string;
|
||||
private readonly createDictationId: () => string;
|
||||
|
||||
private dictationId: string | null = null;
|
||||
private sendSeq = 0;
|
||||
private segments: string[] = [];
|
||||
private streamReady = false;
|
||||
|
||||
private startGeneration = 0;
|
||||
private startPromise: Promise<void> | null = null;
|
||||
|
||||
constructor(params: DictationStreamSenderParams) {
|
||||
this.client = params.client;
|
||||
this.format = params.format;
|
||||
this.createDictationId = params.createDictationId ?? generateMessageId;
|
||||
}
|
||||
|
||||
setClient(client: DaemonClientV2 | null): void {
|
||||
this.client = client;
|
||||
}
|
||||
|
||||
getDictationId(): string | null {
|
||||
return this.dictationId;
|
||||
}
|
||||
|
||||
getSegmentCount(): number {
|
||||
return this.segments.length;
|
||||
}
|
||||
|
||||
getFinalSeq(): number {
|
||||
return this.segments.length - 1;
|
||||
}
|
||||
|
||||
hasSegments(): boolean {
|
||||
return this.segments.length > 0;
|
||||
}
|
||||
|
||||
clearAll(): void {
|
||||
this.dictationId = null;
|
||||
this.sendSeq = 0;
|
||||
this.segments = [];
|
||||
this.streamReady = false;
|
||||
this.startPromise = null;
|
||||
this.startGeneration += 1;
|
||||
}
|
||||
|
||||
resetStreamForReplay(): void {
|
||||
this.dictationId = null;
|
||||
this.sendSeq = 0;
|
||||
this.streamReady = false;
|
||||
this.startPromise = null;
|
||||
this.startGeneration += 1;
|
||||
}
|
||||
|
||||
enqueueSegment(base64Pcm: string): void {
|
||||
this.segments.push(base64Pcm);
|
||||
|
||||
const client = this.client;
|
||||
if (!client?.isConnected) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.dictationId) {
|
||||
void this.restartStream("enqueue");
|
||||
return;
|
||||
}
|
||||
|
||||
this.flush();
|
||||
}
|
||||
|
||||
flush(): number {
|
||||
const client = this.client;
|
||||
const dictationId = this.dictationId;
|
||||
if (!client?.isConnected || !dictationId || !this.streamReady) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let sent = 0;
|
||||
while (this.sendSeq < this.segments.length) {
|
||||
const seq = this.sendSeq;
|
||||
const audio = this.segments[seq]!;
|
||||
client.sendDictationStreamChunk(dictationId, seq, audio, this.format);
|
||||
this.sendSeq = seq + 1;
|
||||
sent += 1;
|
||||
}
|
||||
return sent;
|
||||
}
|
||||
|
||||
async restartStream(reason: string): Promise<void> {
|
||||
const client = this.client;
|
||||
if (!client?.isConnected) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.startGeneration += 1;
|
||||
const generation = this.startGeneration;
|
||||
|
||||
const dictationId = this.createDictationId();
|
||||
this.dictationId = dictationId;
|
||||
this.sendSeq = 0;
|
||||
this.streamReady = false;
|
||||
|
||||
const start = (async () => {
|
||||
await client.startDictationStream(dictationId, this.format);
|
||||
if (this.startGeneration !== generation) {
|
||||
return;
|
||||
}
|
||||
if (this.dictationId !== dictationId) {
|
||||
return;
|
||||
}
|
||||
this.streamReady = true;
|
||||
this.flush();
|
||||
})().catch((error) => {
|
||||
// If starting failed, keep the segments for retry but clear the stream so finish can error cleanly.
|
||||
if (this.startGeneration === generation && this.dictationId === dictationId) {
|
||||
this.dictationId = null;
|
||||
this.streamReady = false;
|
||||
}
|
||||
throw error;
|
||||
}).finally(() => {
|
||||
if (this.startPromise === start) {
|
||||
this.startPromise = null;
|
||||
}
|
||||
});
|
||||
|
||||
this.startPromise = start;
|
||||
await start;
|
||||
void reason;
|
||||
}
|
||||
|
||||
async finish(finalSeq: number): Promise<DictationFinishResult> {
|
||||
const client = this.client;
|
||||
if (!client) {
|
||||
throw new Error("Daemon client unavailable");
|
||||
}
|
||||
if (!client.isConnected) {
|
||||
throw new Error("Daemon client is disconnected");
|
||||
}
|
||||
|
||||
if (!this.dictationId) {
|
||||
await this.restartStream("finalize");
|
||||
}
|
||||
if (this.startPromise) {
|
||||
await this.startPromise;
|
||||
}
|
||||
|
||||
const dictationId = this.dictationId;
|
||||
if (!dictationId || !this.streamReady) {
|
||||
throw new Error("Failed to start dictation stream");
|
||||
}
|
||||
|
||||
this.flush();
|
||||
return client.finishDictationStream(dictationId, finalSeq);
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
const client = this.client;
|
||||
const dictationId = this.dictationId;
|
||||
if (client?.isConnected && dictationId) {
|
||||
client.cancelDictationStream(dictationId);
|
||||
}
|
||||
this.resetStreamForReplay();
|
||||
}
|
||||
}
|
||||
@@ -251,6 +251,10 @@ export function useAgentFormState(
|
||||
model: "",
|
||||
workingDir: "",
|
||||
}));
|
||||
const formStateRef = useRef(formState);
|
||||
useEffect(() => {
|
||||
formStateRef.current = formState;
|
||||
}, [formState]);
|
||||
|
||||
// Track if we've done initial resolution (to avoid flickering)
|
||||
const hasResolvedRef = useRef(false);
|
||||
@@ -299,16 +303,16 @@ export function useAgentFormState(
|
||||
preferences,
|
||||
availableModels,
|
||||
userModified,
|
||||
formState
|
||||
formStateRef.current
|
||||
);
|
||||
|
||||
// Only update if something changed
|
||||
if (
|
||||
resolved.serverId !== formState.serverId ||
|
||||
resolved.provider !== formState.provider ||
|
||||
resolved.modeId !== formState.modeId ||
|
||||
resolved.model !== formState.model ||
|
||||
resolved.workingDir !== formState.workingDir
|
||||
resolved.serverId !== formStateRef.current.serverId ||
|
||||
resolved.provider !== formStateRef.current.provider ||
|
||||
resolved.modeId !== formStateRef.current.modeId ||
|
||||
resolved.model !== formStateRef.current.model ||
|
||||
resolved.workingDir !== formStateRef.current.workingDir
|
||||
) {
|
||||
setFormState(resolved);
|
||||
}
|
||||
@@ -322,7 +326,6 @@ export function useAgentFormState(
|
||||
preferences,
|
||||
availableModels,
|
||||
userModified,
|
||||
formState,
|
||||
]);
|
||||
|
||||
// Persist inferred serverId so reloads keep the selection (e.g. URL serverId or first-time load).
|
||||
|
||||
2
packages/app/src/hooks/use-dictation-audio-source.d.ts
vendored
Normal file
2
packages/app/src/hooks/use-dictation-audio-source.d.ts
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
export * from "./use-dictation-audio-source.native";
|
||||
|
||||
44
packages/app/src/hooks/use-dictation-audio-source.native.ts
Normal file
44
packages/app/src/hooks/use-dictation-audio-source.native.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import { useCallback, useEffect, useRef } from "react";
|
||||
|
||||
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
|
||||
|
||||
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||
const onErrorRef = useRef(config.onError);
|
||||
|
||||
useEffect(() => {
|
||||
onPcmSegmentRef.current = config.onPcmSegment;
|
||||
onErrorRef.current = config.onError;
|
||||
}, [config.onPcmSegment, config.onError]);
|
||||
|
||||
const speechmatics = useSpeechmaticsAudio({
|
||||
enableContinuousStreaming: true,
|
||||
onAudioSegment: ({ audioData }) => {
|
||||
onPcmSegmentRef.current(audioData);
|
||||
},
|
||||
onError: (err) => {
|
||||
onErrorRef.current?.(err);
|
||||
},
|
||||
volumeThreshold: 0.3,
|
||||
silenceDuration: 2000,
|
||||
speechConfirmationDuration: 300,
|
||||
detectionGracePeriod: 200,
|
||||
});
|
||||
|
||||
const start = useCallback(async () => {
|
||||
await speechmatics.start();
|
||||
}, [speechmatics]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
await speechmatics.stop();
|
||||
}, [speechmatics]);
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
volume: speechmatics.volume,
|
||||
};
|
||||
}
|
||||
|
||||
11
packages/app/src/hooks/use-dictation-audio-source.types.ts
Normal file
11
packages/app/src/hooks/use-dictation-audio-source.types.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
export type DictationAudioSourceConfig = {
|
||||
onPcmSegment: (pcm16Base64: string) => void;
|
||||
onError?: (error: Error) => void;
|
||||
};
|
||||
|
||||
export type DictationAudioSource = {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
volume: number;
|
||||
};
|
||||
|
||||
247
packages/app/src/hooks/use-dictation-audio-source.web.ts
Normal file
247
packages/app/src/hooks/use-dictation-audio-source.web.ts
Normal file
@@ -0,0 +1,247 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
|
||||
|
||||
const getAudioContextCtor = (): (typeof AudioContext) | null => {
|
||||
if (typeof window === "undefined") {
|
||||
return null;
|
||||
}
|
||||
const ctor =
|
||||
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
|
||||
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
|
||||
return ctor ?? null;
|
||||
};
|
||||
|
||||
const floatToInt16 = (sample: number): number => {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
||||
};
|
||||
|
||||
const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
|
||||
if (input.length === 0) {
|
||||
return new Int16Array(0);
|
||||
}
|
||||
if (inputRate === outputRate) {
|
||||
const out = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
out[i] = floatToInt16(input[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const ratio = inputRate / outputRate;
|
||||
const outputLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const out = new Int16Array(outputLength);
|
||||
for (let i = 0; i < outputLength; i++) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const i1 = Math.min(input.length - 1, i0 + 1);
|
||||
const frac = sourceIndex - i0;
|
||||
const sample = input[i0] * (1 - frac) + input[i1] * frac;
|
||||
out[i] = floatToInt16(sample);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
const concatInt16 = (a: Int16Array, b: Int16Array): Int16Array => {
|
||||
if (a.length === 0) {
|
||||
return b;
|
||||
}
|
||||
if (b.length === 0) {
|
||||
return a;
|
||||
}
|
||||
const out = new Int16Array(a.length + b.length);
|
||||
out.set(a, 0);
|
||||
out.set(b, a.length);
|
||||
return out;
|
||||
};
|
||||
|
||||
const int16ToBase64 = (pcm: Int16Array): string => {
|
||||
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
};
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const [volume, setVolume] = useState(0);
|
||||
|
||||
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||
const onErrorRef = useRef(config.onError);
|
||||
|
||||
useEffect(() => {
|
||||
onPcmSegmentRef.current = config.onPcmSegment;
|
||||
onErrorRef.current = config.onError;
|
||||
}, [config.onPcmSegment, config.onError]);
|
||||
|
||||
const refs = useRef<{
|
||||
stream: MediaStream | null;
|
||||
context: AudioContext | null;
|
||||
source: MediaStreamAudioSourceNode | null;
|
||||
processor: ScriptProcessorNode | null;
|
||||
gain: GainNode | null;
|
||||
pending: Int16Array;
|
||||
started: boolean;
|
||||
}>({ stream: null, context: null, source: null, processor: null, gain: null, pending: new Int16Array(0), started: false });
|
||||
|
||||
const start = useCallback(async () => {
|
||||
const missingNavigator =
|
||||
typeof navigator === "undefined" ||
|
||||
!navigator.mediaDevices ||
|
||||
typeof navigator.mediaDevices.getUserMedia !== "function";
|
||||
|
||||
const secureContext =
|
||||
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
|
||||
? window.isSecureContext
|
||||
: true;
|
||||
const currentOrigin =
|
||||
typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
|
||||
|
||||
if (missingNavigator) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
if (!secureContext) {
|
||||
throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const processor = context.createScriptProcessor(4096, 1, 1);
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 0;
|
||||
|
||||
const outputRate = 16000;
|
||||
const chunkSamples = outputRate; // ~1s
|
||||
|
||||
refs.current.started = true;
|
||||
|
||||
processor.onaudioprocess = (event) => {
|
||||
if (!refs.current.started) {
|
||||
return;
|
||||
}
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const sample = input[i];
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
const normalized = Math.min(1, Math.max(0, rms * 2));
|
||||
setVolume(normalized);
|
||||
|
||||
const next = resampleToPcm16(input, context.sampleRate, outputRate);
|
||||
refs.current.pending = concatInt16(refs.current.pending, next);
|
||||
|
||||
while (refs.current.pending.length >= chunkSamples) {
|
||||
const chunk = refs.current.pending.slice(0, chunkSamples);
|
||||
refs.current.pending = refs.current.pending.slice(chunkSamples);
|
||||
onPcmSegmentRef.current(int16ToBase64(chunk));
|
||||
}
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
|
||||
refs.current = { stream, context, source, processor, gain, pending: new Int16Array(0), started: true };
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
refs.current.started = false;
|
||||
setVolume(0);
|
||||
|
||||
const { processor, source, gain, context, stream, pending } = refs.current;
|
||||
|
||||
if (processor) {
|
||||
try {
|
||||
processor.onaudioprocess = null;
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
try {
|
||||
processor.disconnect();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
if (source) {
|
||||
try {
|
||||
source.disconnect();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
if (gain) {
|
||||
try {
|
||||
gain.disconnect();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
if (stream) {
|
||||
stream.getTracks().forEach((track) => {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
});
|
||||
}
|
||||
if (context) {
|
||||
try {
|
||||
await context.close();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
|
||||
if (pending.length > 0) {
|
||||
onPcmSegmentRef.current(int16ToBase64(pending));
|
||||
}
|
||||
|
||||
refs.current = { stream: null, context: null, source: null, processor: null, gain: null, pending: new Int16Array(0), started: false };
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
void stop().catch((err) => {
|
||||
onErrorRef.current?.(err instanceof Error ? err : new Error(String(err)));
|
||||
});
|
||||
};
|
||||
}, [stop]);
|
||||
|
||||
return useMemo(
|
||||
() => ({
|
||||
start: async () => {
|
||||
try {
|
||||
await start();
|
||||
} catch (err) {
|
||||
const normalized = err instanceof Error ? err : new Error(String(err));
|
||||
onErrorRef.current?.(normalized);
|
||||
throw normalized;
|
||||
}
|
||||
},
|
||||
stop,
|
||||
volume,
|
||||
}),
|
||||
[start, stop, volume]
|
||||
);
|
||||
}
|
||||
|
||||
1
packages/app/src/hooks/use-dictation.d.ts
vendored
Normal file
1
packages/app/src/hooks/use-dictation.d.ts
vendored
Normal file
@@ -0,0 +1 @@
|
||||
export * from "./use-dictation";
|
||||
40
packages/app/src/hooks/use-dictation.shared.ts
Normal file
40
packages/app/src/hooks/use-dictation.shared.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
export type DictationStatus = "idle" | "recording" | "uploading" | "failed";
|
||||
|
||||
export type UseDictationOptions = {
|
||||
client: import("@server/client/daemon-client-v2").DaemonClientV2 | null;
|
||||
onTranscript: (text: string, meta: { requestId: string }) => void;
|
||||
onError?: (error: Error) => void;
|
||||
onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
|
||||
canStart?: () => boolean;
|
||||
canConfirm?: () => boolean;
|
||||
autoStopWhenHidden?: { isVisible: boolean };
|
||||
enableDuration?: boolean;
|
||||
};
|
||||
|
||||
export type UseDictationResult = {
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
volume: number;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
status: DictationStatus;
|
||||
startDictation: () => Promise<void>;
|
||||
cancelDictation: () => Promise<void>;
|
||||
confirmDictation: () => Promise<void>;
|
||||
retryFailedDictation: () => Promise<void>;
|
||||
discardFailedDictation: () => void;
|
||||
reset: () => void;
|
||||
};
|
||||
|
||||
export const DURATION_TICK_MS = 1000;
|
||||
export const PCM_DICTATION_FORMAT = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
export const toError = (error: unknown): Error => {
|
||||
if (error instanceof Error) {
|
||||
return error;
|
||||
}
|
||||
if (typeof error === "string" && error.trim().length > 0) {
|
||||
return new Error(error);
|
||||
}
|
||||
return new Error("An unexpected error occurred while handling dictation.");
|
||||
};
|
||||
@@ -1,160 +1,23 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
|
||||
import { useAudioRecorder } from "@/hooks/use-audio-recorder";
|
||||
import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
|
||||
import type { TranscriptionResultMessage } from "@server/shared/messages";
|
||||
import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
|
||||
import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source";
|
||||
import { generateMessageId } from "@/types/stream";
|
||||
import { AttemptGuard } from "@/utils/attempt-guard";
|
||||
|
||||
export type DictationStatus = "idle" | "recording" | "uploading" | "retrying" | "failed";
|
||||
|
||||
type DictationRetryReason = "dispatch" | "timeout" | "response" | "disconnected";
|
||||
|
||||
export type DictationRetryInfo = {
|
||||
attempt: number;
|
||||
maxAttempts: number;
|
||||
reason: DictationRetryReason;
|
||||
errorMessage: string;
|
||||
nextRetryMs: number;
|
||||
};
|
||||
|
||||
export type FailedDictationRecording = {
|
||||
requestId: string;
|
||||
durationSeconds: number;
|
||||
sizeBytes: number;
|
||||
format: string;
|
||||
recordedAt: number;
|
||||
errorMessage: string;
|
||||
};
|
||||
|
||||
export type DictationOutcome =
|
||||
| { type: "success"; requestId: string; timestamp: number }
|
||||
| { type: "failure"; requestId: string; errorMessage: string; timestamp: number };
|
||||
|
||||
export type UseDictationOptions = {
|
||||
client: DaemonClientV2 | null;
|
||||
onTranscript: (text: string, meta: { requestId: string }) => void;
|
||||
onError?: (error: Error) => void;
|
||||
onRetryAttempt?: (info: DictationRetryInfo) => void;
|
||||
onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
|
||||
canStart?: () => boolean;
|
||||
canConfirm?: () => boolean;
|
||||
autoStopWhenHidden?: { isVisible: boolean };
|
||||
enableDuration?: boolean;
|
||||
};
|
||||
|
||||
export type UseDictationResult = {
|
||||
isRecording: boolean;
|
||||
isProcessing: boolean;
|
||||
volume: number;
|
||||
duration: number;
|
||||
pendingRequestId: string | null;
|
||||
error: string | null;
|
||||
status: DictationStatus;
|
||||
retryAttempt: number;
|
||||
maxRetryAttempts: number;
|
||||
retryInfo: DictationRetryInfo | null;
|
||||
failedRecording: FailedDictationRecording | null;
|
||||
lastOutcome: DictationOutcome | null;
|
||||
startDictation: () => Promise<void>;
|
||||
cancelDictation: () => Promise<void>;
|
||||
confirmDictation: () => Promise<void>;
|
||||
retryFailedDictation: () => Promise<void>;
|
||||
discardFailedDictation: () => void;
|
||||
reset: () => void;
|
||||
};
|
||||
|
||||
const DURATION_TICK_MS = 1000;
|
||||
const MAX_AUTO_RETRY_ATTEMPTS = 5;
|
||||
const RETRY_BASE_DELAY_MS = 2000;
|
||||
const RETRY_MAX_DELAY_MS = 12000;
|
||||
const RETRY_BACKOFF_FACTOR = 1.8;
|
||||
const RETRY_JITTER_MS = 400;
|
||||
const TRANSCRIPTION_TIMEOUT_MS = 120000;
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
const computeRetryDelayMs = (attempt: number): number => {
|
||||
const exponential = RETRY_BASE_DELAY_MS * RETRY_BACKOFF_FACTOR ** Math.max(0, attempt - 1);
|
||||
const capped = Math.min(RETRY_MAX_DELAY_MS, exponential);
|
||||
if (RETRY_JITTER_MS <= 0) {
|
||||
return capped;
|
||||
}
|
||||
return capped + Math.floor(Math.random() * RETRY_JITTER_MS);
|
||||
};
|
||||
|
||||
const isTimeoutError = (error: Error): boolean =>
|
||||
/timed out|timeout/i.test(error.message);
|
||||
|
||||
class DictationAttemptError extends Error {
|
||||
public readonly reason: DictationRetryReason;
|
||||
|
||||
constructor(reason: DictationRetryReason, error: Error) {
|
||||
super(error.message);
|
||||
this.name = "DictationAttemptError";
|
||||
this.reason = reason;
|
||||
(this as Error & { cause?: Error }).cause = error;
|
||||
}
|
||||
}
|
||||
|
||||
const toError = (error: unknown): Error => {
|
||||
if (error instanceof Error) {
|
||||
return error;
|
||||
}
|
||||
if (typeof error === "string" && error.trim().length > 0) {
|
||||
return new Error(error);
|
||||
}
|
||||
return new Error("An unexpected error occurred while handling dictation.");
|
||||
};
|
||||
|
||||
type CapturedAudioPayload = {
|
||||
blob: Blob;
|
||||
format: string;
|
||||
sizeBytes: number;
|
||||
durationSeconds: number;
|
||||
recordedAt: number;
|
||||
};
|
||||
|
||||
type TranscriptionPayload = TranscriptionResultMessage["payload"];
|
||||
|
||||
const blobToBase64 = async (blob: Blob): Promise<string> => {
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
const bytes = new Uint8Array(arrayBuffer);
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
};
|
||||
|
||||
const deriveFormatFromMime = (mimeType?: string): string => {
|
||||
if (!mimeType || mimeType.length === 0) {
|
||||
return "webm";
|
||||
}
|
||||
const slashIndex = mimeType.indexOf("/");
|
||||
let formatPart = slashIndex >= 0 ? mimeType.slice(slashIndex + 1) : mimeType;
|
||||
const semicolonIndex = formatPart.indexOf(";");
|
||||
if (semicolonIndex >= 0) {
|
||||
formatPart = formatPart.slice(0, semicolonIndex);
|
||||
}
|
||||
return formatPart.trim().length > 0 ? formatPart.trim() : "webm";
|
||||
};
|
||||
|
||||
const buildCapturedAudioPayload = (blob: Blob, durationSeconds: number): CapturedAudioPayload => ({
|
||||
blob,
|
||||
format: deriveFormatFromMime(blob.type),
|
||||
sizeBytes: typeof blob.size === "number" ? blob.size : 0,
|
||||
durationSeconds,
|
||||
recordedAt: Date.now(),
|
||||
});
|
||||
import {
|
||||
DURATION_TICK_MS,
|
||||
PCM_DICTATION_FORMAT,
|
||||
toError,
|
||||
type DictationStatus,
|
||||
type UseDictationOptions,
|
||||
type UseDictationResult,
|
||||
} from "./use-dictation.shared";
|
||||
|
||||
export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
const {
|
||||
client,
|
||||
onTranscript,
|
||||
onError,
|
||||
onRetryAttempt,
|
||||
onPermanentFailure,
|
||||
canStart,
|
||||
canConfirm,
|
||||
@@ -164,127 +27,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const [duration, setDuration] = useState(0);
|
||||
const [pendingRequestId, setPendingRequestId] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [status, setStatus] = useState<DictationStatus>("idle");
|
||||
const [retryAttempt, setRetryAttempt] = useState(0);
|
||||
const [retryInfo, setRetryInfo] = useState<DictationRetryInfo | null>(null);
|
||||
const [failedRecording, setFailedRecording] = useState<FailedDictationRecording | null>(null);
|
||||
const [lastOutcome, setLastOutcome] = useState<DictationOutcome | null>(null);
|
||||
const maxRetryAttempts = MAX_AUTO_RETRY_ATTEMPTS;
|
||||
|
||||
const transcriptionMutation = useMutation({
|
||||
mutationFn: async (): Promise<TranscriptionPayload> => {
|
||||
const capturedAudio = pendingAudioRef.current;
|
||||
if (!capturedAudio) {
|
||||
throw new Error("No recorded audio available for transcription");
|
||||
}
|
||||
|
||||
setRetryAttempt(1);
|
||||
setRetryInfo(null);
|
||||
|
||||
let attempt = 1;
|
||||
|
||||
while (attempt <= MAX_AUTO_RETRY_ATTEMPTS) {
|
||||
try {
|
||||
if (!client) {
|
||||
throw new DictationAttemptError(
|
||||
"disconnected",
|
||||
new Error("Daemon client unavailable")
|
||||
);
|
||||
}
|
||||
if (!client.isConnected) {
|
||||
throw new DictationAttemptError(
|
||||
"disconnected",
|
||||
new Error("Daemon client is disconnected")
|
||||
);
|
||||
}
|
||||
|
||||
console.info("[useDictation] sending transcription request", {
|
||||
attempt,
|
||||
size: capturedAudio.sizeBytes,
|
||||
durationSeconds: capturedAudio.durationSeconds,
|
||||
});
|
||||
|
||||
setStatus("uploading");
|
||||
setRetryAttempt(attempt);
|
||||
|
||||
try {
|
||||
const base64Audio = await blobToBase64(capturedAudio.blob);
|
||||
return await client.transcribeAudio({
|
||||
audio: base64Audio,
|
||||
format: capturedAudio.format,
|
||||
timeout: TRANSCRIPTION_TIMEOUT_MS,
|
||||
});
|
||||
} catch (error) {
|
||||
const normalized = toError(error);
|
||||
const reason: DictationRetryReason = isTimeoutError(normalized)
|
||||
? "timeout"
|
||||
: "response";
|
||||
throw new DictationAttemptError(reason, normalized);
|
||||
}
|
||||
} catch (error) {
|
||||
const attemptError =
|
||||
error instanceof DictationAttemptError
|
||||
? error
|
||||
: new DictationAttemptError("response", toError(error));
|
||||
|
||||
if (attempt >= MAX_AUTO_RETRY_ATTEMPTS) {
|
||||
throw attemptError;
|
||||
}
|
||||
|
||||
const nextAttempt = attempt + 1;
|
||||
const delayMs = computeRetryDelayMs(nextAttempt);
|
||||
const info: DictationRetryInfo = {
|
||||
attempt: nextAttempt,
|
||||
maxAttempts: MAX_AUTO_RETRY_ATTEMPTS,
|
||||
reason: attemptError.reason,
|
||||
errorMessage: attemptError.message,
|
||||
nextRetryMs: delayMs,
|
||||
};
|
||||
|
||||
setStatus("retrying");
|
||||
setRetryAttempt(nextAttempt);
|
||||
setRetryInfo(info);
|
||||
onRetryAttemptRef.current?.(info);
|
||||
console.warn("[useDictation] retry scheduled", {
|
||||
attempt: nextAttempt,
|
||||
maxAttempts: MAX_AUTO_RETRY_ATTEMPTS,
|
||||
reason: attemptError.reason,
|
||||
error: attemptError.message,
|
||||
nextRetryMs: delayMs,
|
||||
});
|
||||
|
||||
await sleep(delayMs);
|
||||
attempt = nextAttempt;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Failed to complete transcription");
|
||||
},
|
||||
});
|
||||
const {
|
||||
mutateAsync: runTranscription,
|
||||
reset: resetTranscriptionMutation,
|
||||
} = transcriptionMutation;
|
||||
|
||||
const pendingAudioRef = useRef<CapturedAudioPayload | null>(null);
|
||||
const handleAudioLevel = useCallback((level: number) => {
|
||||
setVolume(level);
|
||||
}, []);
|
||||
|
||||
const durationRef = useRef(0);
|
||||
useEffect(() => {
|
||||
durationRef.current = duration;
|
||||
}, [duration]);
|
||||
|
||||
const recorder = useAudioRecorder({ onAudioLevel: handleAudioLevel });
|
||||
const recorderRef = useRef(recorder);
|
||||
useEffect(() => {
|
||||
recorderRef.current = recorder;
|
||||
}, [recorder]);
|
||||
|
||||
const onTranscriptRef = useRef(onTranscript);
|
||||
useEffect(() => {
|
||||
@@ -296,11 +41,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
onErrorRef.current = onError;
|
||||
}, [onError]);
|
||||
|
||||
const onRetryAttemptRef = useRef(onRetryAttempt);
|
||||
useEffect(() => {
|
||||
onRetryAttemptRef.current = onRetryAttempt;
|
||||
}, [onRetryAttempt]);
|
||||
|
||||
const onPermanentFailureRef = useRef(onPermanentFailure);
|
||||
useEffect(() => {
|
||||
onPermanentFailureRef.current = onPermanentFailure;
|
||||
@@ -316,10 +56,27 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
isProcessingRef.current = isProcessing;
|
||||
}, [isProcessing]);
|
||||
|
||||
const pendingRequestIdRef = useRef<string | null>(null);
|
||||
const activeStopPromiseRef = useRef<Promise<Blob> | null>(null);
|
||||
// duration is used for UI only; no need to mirror into a ref.
|
||||
|
||||
const durationIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const attemptGuardRef = useRef(new AttemptGuard());
|
||||
const actionGateRef = useRef<{ starting: boolean; confirming: boolean; cancelling: boolean }>({
|
||||
starting: false,
|
||||
confirming: false,
|
||||
cancelling: false,
|
||||
});
|
||||
|
||||
const senderRef = useRef<DictationStreamSender | null>(null);
|
||||
if (!senderRef.current) {
|
||||
senderRef.current = new DictationStreamSender({
|
||||
client,
|
||||
format: PCM_DICTATION_FORMAT,
|
||||
createDictationId: generateMessageId,
|
||||
});
|
||||
}
|
||||
useEffect(() => {
|
||||
senderRef.current?.setClient(client);
|
||||
}, [client]);
|
||||
|
||||
const stopDurationTracking = useCallback(() => {
|
||||
if (durationIntervalRef.current) {
|
||||
@@ -330,7 +87,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
|
||||
const startDurationTracking = useCallback(() => {
|
||||
if (!enableDuration) {
|
||||
console.log("[useDictation] startDictation blocked by canStart()");
|
||||
return;
|
||||
}
|
||||
if (durationIntervalRef.current) {
|
||||
@@ -351,6 +107,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
const reportError = useCallback(
|
||||
(err: unknown, context?: string) => {
|
||||
const normalized = toError(err);
|
||||
if (normalized.name === "AttemptCancelledError") {
|
||||
return;
|
||||
}
|
||||
if (context) {
|
||||
console.error(`[useDictation] ${context}`, normalized);
|
||||
} else {
|
||||
@@ -362,136 +121,114 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
[setError]
|
||||
);
|
||||
|
||||
const stopRecorder = useCallback(async (): Promise<Blob> => {
|
||||
if (activeStopPromiseRef.current) {
|
||||
return activeStopPromiseRef.current;
|
||||
}
|
||||
const recorderInstance = recorderRef.current;
|
||||
if (!recorderInstance) {
|
||||
throw new Error("Recorder unavailable");
|
||||
}
|
||||
const stopPromise = (async () => {
|
||||
try {
|
||||
return await recorderInstance.stop();
|
||||
} finally {
|
||||
activeStopPromiseRef.current = null;
|
||||
}
|
||||
})();
|
||||
activeStopPromiseRef.current = stopPromise;
|
||||
return stopPromise;
|
||||
const clearStreamingState = useCallback(() => {
|
||||
senderRef.current?.clearAll();
|
||||
}, []);
|
||||
|
||||
const transmitDictation = useCallback(
|
||||
async () => runTranscription(),
|
||||
[runTranscription]
|
||||
const startNewStream = useCallback(
|
||||
async (reason: string) => {
|
||||
await senderRef.current?.restartStream(reason);
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
const handleTranscriptionSuccess = useCallback(
|
||||
(transcription: TranscriptionPayload) => {
|
||||
const requestId = transcription.requestId;
|
||||
pendingRequestIdRef.current = null;
|
||||
setPendingRequestId(null);
|
||||
const ensureFinalTranscript = useCallback(
|
||||
async (finalSeq: number): Promise<string> => {
|
||||
const result = await senderRef.current!.finish(finalSeq);
|
||||
return result.text;
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
return client.subscribeConnectionStatus((next) => {
|
||||
if (next.status !== "connected") {
|
||||
return;
|
||||
}
|
||||
if (!isRecordingRef.current) {
|
||||
return;
|
||||
}
|
||||
void startNewStream("reconnect");
|
||||
});
|
||||
}, [client, startNewStream]);
|
||||
|
||||
const audio = useDictationAudioSource({
|
||||
onPcmSegment: (audioData) => {
|
||||
senderRef.current?.enqueueSegment(audioData);
|
||||
},
|
||||
onError: (err) => {
|
||||
onErrorRef.current?.(err);
|
||||
},
|
||||
});
|
||||
const audioStopRef = useRef(audio.stop);
|
||||
useEffect(() => {
|
||||
audioStopRef.current = audio.stop;
|
||||
}, [audio.stop]);
|
||||
|
||||
const handleStreamingTranscriptionSuccess = useCallback(
|
||||
(text: string, requestId: string) => {
|
||||
setIsProcessing(false);
|
||||
setDuration(0);
|
||||
setStatus("idle");
|
||||
setRetryAttempt(0);
|
||||
setRetryInfo(null);
|
||||
setFailedRecording(null);
|
||||
pendingAudioRef.current = null;
|
||||
setLastOutcome({ type: "success", requestId, timestamp: Date.now() });
|
||||
|
||||
const transcriptText = transcription.text?.trim();
|
||||
clearStreamingState();
|
||||
|
||||
const transcriptText = text.trim();
|
||||
if (!transcriptText) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("[useDictation] transcription_result received", {
|
||||
requestId,
|
||||
textLength: transcriptText.length,
|
||||
});
|
||||
onTranscriptRef.current?.(transcriptText, { requestId });
|
||||
},
|
||||
[onTranscriptRef]
|
||||
[clearStreamingState]
|
||||
);
|
||||
|
||||
const handleDictationFailure = useCallback(
|
||||
(failure: unknown) => {
|
||||
const normalized = toError(failure);
|
||||
const failureId = generateMessageId();
|
||||
pendingRequestIdRef.current = null;
|
||||
setPendingRequestId(null);
|
||||
setIsProcessing(false);
|
||||
isRecordingRef.current = false;
|
||||
setIsRecording(false);
|
||||
setVolume(0);
|
||||
setRetryInfo(null);
|
||||
|
||||
const capturedAudio = pendingAudioRef.current;
|
||||
if (capturedAudio) {
|
||||
if (senderRef.current?.hasSegments()) {
|
||||
setStatus("failed");
|
||||
setFailedRecording({
|
||||
requestId: failureId,
|
||||
durationSeconds: capturedAudio.durationSeconds,
|
||||
sizeBytes: capturedAudio.sizeBytes,
|
||||
format: capturedAudio.format,
|
||||
recordedAt: capturedAudio.recordedAt,
|
||||
errorMessage: normalized.message,
|
||||
});
|
||||
onPermanentFailureRef.current?.(normalized, { requestId: failureId });
|
||||
} else {
|
||||
setStatus("idle");
|
||||
}
|
||||
|
||||
setRetryAttempt(0);
|
||||
setLastOutcome({
|
||||
type: "failure",
|
||||
requestId: failureId,
|
||||
errorMessage: normalized.message,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
reportError(normalized, "Failed to complete dictation");
|
||||
},
|
||||
[onPermanentFailureRef, reportError]
|
||||
[reportError]
|
||||
);
|
||||
|
||||
const startDictation = useCallback(async () => {
|
||||
console.log("[useDictation] startDictation requested", {
|
||||
isRecording: isRecordingRef.current,
|
||||
isProcessing,
|
||||
});
|
||||
if (isRecordingRef.current || isProcessing) {
|
||||
console.log("[useDictation] startDictation aborted: already recording/processing", {
|
||||
isRecording: isRecordingRef.current,
|
||||
isProcessing,
|
||||
});
|
||||
if (actionGateRef.current.starting || actionGateRef.current.confirming || actionGateRef.current.cancelling) {
|
||||
return;
|
||||
}
|
||||
if (isRecordingRef.current || isProcessingRef.current) {
|
||||
return;
|
||||
}
|
||||
const startAllowed = canStart ? canStart() : true;
|
||||
if (!startAllowed) {
|
||||
console.log("[useDictation] startDictation blocked by canStart()");
|
||||
return;
|
||||
}
|
||||
|
||||
actionGateRef.current.starting = true;
|
||||
setError(null);
|
||||
setVolume(0);
|
||||
setDuration(0);
|
||||
setIsProcessing(false);
|
||||
setStatus("recording");
|
||||
setRetryAttempt(0);
|
||||
setRetryInfo(null);
|
||||
setFailedRecording(null);
|
||||
pendingAudioRef.current = null;
|
||||
setLastOutcome(null);
|
||||
pendingRequestIdRef.current = null;
|
||||
setPendingRequestId(null);
|
||||
clearStreamingState();
|
||||
|
||||
try {
|
||||
const recorderInstance = recorderRef.current;
|
||||
if (!recorderInstance) {
|
||||
throw new Error("Recorder unavailable");
|
||||
await audio.start();
|
||||
if (client?.isConnected) {
|
||||
void startNewStream("start");
|
||||
}
|
||||
await recorderInstance.start();
|
||||
console.log("[useDictation] recorder.start succeeded");
|
||||
isRecordingRef.current = true;
|
||||
setIsRecording(true);
|
||||
if (enableDuration) {
|
||||
@@ -502,160 +239,155 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
isRecordingRef.current = false;
|
||||
setIsRecording(false);
|
||||
reportError(err, "Failed to start dictation");
|
||||
} finally {
|
||||
actionGateRef.current.starting = false;
|
||||
}
|
||||
}, [
|
||||
audio,
|
||||
canStart,
|
||||
clearStreamingState,
|
||||
client,
|
||||
enableDuration,
|
||||
isProcessing,
|
||||
reportError,
|
||||
startDurationTracking,
|
||||
startNewStream,
|
||||
stopDurationTracking,
|
||||
]);
|
||||
|
||||
const cancelDictation = useCallback(async () => {
|
||||
console.log("[useDictation] cancelDictation requested", {
|
||||
isRecording: isRecordingRef.current,
|
||||
hasActiveStop: Boolean(activeStopPromiseRef.current),
|
||||
});
|
||||
attemptGuardRef.current.cancel();
|
||||
if (!isRecordingRef.current && !activeStopPromiseRef.current) {
|
||||
console.log("[useDictation] cancelDictation ignored: nothing to cancel");
|
||||
if (actionGateRef.current.cancelling) {
|
||||
return;
|
||||
}
|
||||
if (!isRecordingRef.current && !isProcessingRef.current) {
|
||||
return;
|
||||
}
|
||||
actionGateRef.current.cancelling = true;
|
||||
stopDurationTracking();
|
||||
setDuration(0);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
await stopRecorder();
|
||||
try {
|
||||
senderRef.current?.cancel();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
await audio.stop();
|
||||
} catch (err) {
|
||||
reportError(err, "Failed to cancel dictation");
|
||||
} finally {
|
||||
isRecordingRef.current = false;
|
||||
setIsRecording(false);
|
||||
setIsProcessing(false);
|
||||
setVolume(0);
|
||||
isProcessingRef.current = false;
|
||||
setStatus("idle");
|
||||
setRetryAttempt(0);
|
||||
setRetryInfo(null);
|
||||
pendingAudioRef.current = null;
|
||||
setFailedRecording(null);
|
||||
setLastOutcome(null);
|
||||
clearStreamingState();
|
||||
actionGateRef.current.cancelling = false;
|
||||
}
|
||||
}, [reportError, stopDurationTracking, stopRecorder]);
|
||||
}, [audio, clearStreamingState, client, reportError, stopDurationTracking]);
|
||||
|
||||
const confirmDictation = useCallback(async () => {
|
||||
console.log("[useDictation] confirmDictation requested", {
|
||||
isRecording: isRecordingRef.current,
|
||||
isProcessing,
|
||||
});
|
||||
if (!isRecordingRef.current || isProcessing) {
|
||||
console.log("[useDictation] confirmDictation ignored: recording flag", {
|
||||
isRecording: isRecordingRef.current,
|
||||
isProcessing,
|
||||
});
|
||||
if (actionGateRef.current.confirming) {
|
||||
return;
|
||||
}
|
||||
if (!isRecordingRef.current || isProcessingRef.current) {
|
||||
return;
|
||||
}
|
||||
const confirmAllowed = canConfirm ? canConfirm() : true;
|
||||
if (!confirmAllowed) {
|
||||
console.log("[useDictation] confirmDictation blocked by canConfirm()");
|
||||
return;
|
||||
}
|
||||
|
||||
actionGateRef.current.confirming = true;
|
||||
setError(null);
|
||||
stopDurationTracking();
|
||||
setIsProcessing(true);
|
||||
setRetryInfo(null);
|
||||
setRetryAttempt(0);
|
||||
setLastOutcome(null);
|
||||
isProcessingRef.current = true;
|
||||
|
||||
const attemptId = attemptGuardRef.current.next();
|
||||
|
||||
try {
|
||||
const audioData = await stopRecorder();
|
||||
await audio.stop();
|
||||
attemptGuardRef.current.assertCurrent(attemptId);
|
||||
const recordedDurationSeconds = durationRef.current;
|
||||
pendingAudioRef.current = buildCapturedAudioPayload(audioData, recordedDurationSeconds);
|
||||
|
||||
setStatus("uploading");
|
||||
isRecordingRef.current = false;
|
||||
setIsRecording(false);
|
||||
setVolume(0);
|
||||
|
||||
const transcription = await transmitDictation();
|
||||
const finalSeq = senderRef.current?.getFinalSeq() ?? -1;
|
||||
if (finalSeq < 0) {
|
||||
handleStreamingTranscriptionSuccess("", generateMessageId());
|
||||
return;
|
||||
}
|
||||
|
||||
const transcriptText = await ensureFinalTranscript(finalSeq);
|
||||
attemptGuardRef.current.assertCurrent(attemptId);
|
||||
handleTranscriptionSuccess(transcription);
|
||||
handleStreamingTranscriptionSuccess(transcriptText, generateMessageId());
|
||||
} catch (err) {
|
||||
resetTranscriptionMutation();
|
||||
if (err instanceof Error && err.name === "AttemptCancelledError") {
|
||||
return;
|
||||
}
|
||||
handleDictationFailure(err);
|
||||
} finally {
|
||||
actionGateRef.current.confirming = false;
|
||||
}
|
||||
}, [
|
||||
audio,
|
||||
canConfirm,
|
||||
isProcessing,
|
||||
handleDictationFailure,
|
||||
handleTranscriptionSuccess,
|
||||
resetTranscriptionMutation,
|
||||
handleStreamingTranscriptionSuccess,
|
||||
stopDurationTracking,
|
||||
stopRecorder,
|
||||
transmitDictation,
|
||||
ensureFinalTranscript,
|
||||
]);
|
||||
|
||||
const retryFailedDictation = useCallback(async () => {
|
||||
if (!pendingAudioRef.current) {
|
||||
if (!senderRef.current?.hasSegments()) {
|
||||
return;
|
||||
}
|
||||
setError(null);
|
||||
setRetryInfo(null);
|
||||
setRetryAttempt(0);
|
||||
setStatus("uploading");
|
||||
setIsProcessing(true);
|
||||
setLastOutcome(null);
|
||||
isProcessingRef.current = true;
|
||||
|
||||
try {
|
||||
const transcription = await transmitDictation();
|
||||
handleTranscriptionSuccess(transcription);
|
||||
if (!client?.isConnected) {
|
||||
throw new Error("Daemon client is disconnected");
|
||||
}
|
||||
senderRef.current.resetStreamForReplay();
|
||||
const finalSeq = senderRef.current.getFinalSeq();
|
||||
const text = await ensureFinalTranscript(finalSeq);
|
||||
handleStreamingTranscriptionSuccess(text, generateMessageId());
|
||||
} catch (err) {
|
||||
resetTranscriptionMutation();
|
||||
if (err instanceof Error && err.name === "AttemptCancelledError") {
|
||||
return;
|
||||
}
|
||||
handleDictationFailure(err);
|
||||
}
|
||||
}, [
|
||||
handleDictationFailure,
|
||||
handleTranscriptionSuccess,
|
||||
resetTranscriptionMutation,
|
||||
transmitDictation,
|
||||
]);
|
||||
}, [client, ensureFinalTranscript, handleDictationFailure, handleStreamingTranscriptionSuccess]);
|
||||
|
||||
const discardFailedDictation = useCallback(() => {
|
||||
pendingAudioRef.current = null;
|
||||
pendingRequestIdRef.current = null;
|
||||
setPendingRequestId(null);
|
||||
setIsProcessing(false);
|
||||
isProcessingRef.current = false;
|
||||
setDuration(0);
|
||||
setFailedRecording(null);
|
||||
setStatus("idle");
|
||||
setRetryAttempt(0);
|
||||
setRetryInfo(null);
|
||||
setError(null);
|
||||
setLastOutcome(null);
|
||||
}, []);
|
||||
clearStreamingState();
|
||||
}, [clearStreamingState]);
|
||||
|
||||
const reset = useCallback(() => {
|
||||
pendingRequestIdRef.current = null;
|
||||
setPendingRequestId(null);
|
||||
setIsRecording(false);
|
||||
isRecordingRef.current = false;
|
||||
setIsProcessing(false);
|
||||
isProcessingRef.current = false;
|
||||
stopDurationTracking();
|
||||
setDuration(0);
|
||||
setVolume(0);
|
||||
setError(null);
|
||||
setStatus("idle");
|
||||
setRetryAttempt(0);
|
||||
setRetryInfo(null);
|
||||
setFailedRecording(null);
|
||||
pendingAudioRef.current = null;
|
||||
setLastOutcome(null);
|
||||
resetTranscriptionMutation();
|
||||
}, [resetTranscriptionMutation, stopDurationTracking]);
|
||||
clearStreamingState();
|
||||
}, [clearStreamingState, stopDurationTracking]);
|
||||
|
||||
const cancelRef = useRef<(() => void) | null>(null);
|
||||
useEffect(() => {
|
||||
@@ -668,8 +400,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null
|
||||
);
|
||||
useEffect(() => {
|
||||
const nextVisible =
|
||||
typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null;
|
||||
const nextVisible = typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null;
|
||||
const prevVisible = visibilityRef.current;
|
||||
visibilityRef.current = nextVisible;
|
||||
|
||||
@@ -685,48 +416,28 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
stopDurationTracking();
|
||||
setDuration(0);
|
||||
setIsProcessing(false);
|
||||
setVolume(0);
|
||||
setError(null);
|
||||
setStatus("idle");
|
||||
setRetryAttempt(0);
|
||||
setRetryInfo(null);
|
||||
pendingAudioRef.current = null;
|
||||
setFailedRecording(null);
|
||||
setLastOutcome(null);
|
||||
clearStreamingState();
|
||||
}
|
||||
}
|
||||
}, [autoStopWhenHidden?.isVisible, stopDurationTracking]);
|
||||
|
||||
}, [autoStopWhenHidden?.isVisible, clearStreamingState, stopDurationTracking]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
attemptGuardRef.current.cancel();
|
||||
stopDurationTracking();
|
||||
const activeStop = activeStopPromiseRef.current;
|
||||
if (activeStop) {
|
||||
void activeStop.catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
const recorderInstance = recorderRef.current;
|
||||
if (recorderInstance?.isRecording?.()) {
|
||||
void recorderInstance.stop().catch(() => undefined);
|
||||
}
|
||||
void audioStopRef.current().catch(() => undefined);
|
||||
};
|
||||
}, [stopDurationTracking]);
|
||||
|
||||
return {
|
||||
isRecording,
|
||||
isProcessing,
|
||||
volume,
|
||||
volume: audio.volume,
|
||||
duration,
|
||||
pendingRequestId,
|
||||
error,
|
||||
status,
|
||||
retryAttempt,
|
||||
maxRetryAttempts,
|
||||
retryInfo,
|
||||
failedRecording,
|
||||
lastOutcome,
|
||||
startDictation,
|
||||
cancelDictation,
|
||||
confirmDictation,
|
||||
@@ -735,3 +446,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
|
||||
reset,
|
||||
};
|
||||
}
|
||||
|
||||
export type {
|
||||
DictationStatus,
|
||||
UseDictationOptions,
|
||||
UseDictationResult,
|
||||
} from "./use-dictation.shared";
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useState, useEffect, useRef, useCallback } from "react";
|
||||
import { Buffer } from "buffer";
|
||||
import {
|
||||
initialize,
|
||||
useMicrophonePermissions,
|
||||
@@ -14,6 +15,8 @@ export interface SpeechmaticsAudioConfig {
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
/** When true, stream microphone PCM continuously without VAD gating. */
|
||||
enableContinuousStreaming?: boolean;
|
||||
volumeThreshold: number; // Volume threshold for speech detection (0-1)
|
||||
silenceDuration: number; // ms of silence before ending segment
|
||||
speechConfirmationDuration: number; // ms of sustained speech before confirming
|
||||
@@ -33,11 +36,17 @@ export interface SpeechmaticsAudio {
|
||||
}
|
||||
|
||||
function uint8ArrayToBase64(bytes: Uint8Array): string {
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
// NOTE: This is performance-sensitive during continuous streaming.
|
||||
// Buffer-backed base64 is significantly faster than manual string building.
|
||||
try {
|
||||
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
|
||||
} catch {
|
||||
let binary = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
|
||||
@@ -67,6 +76,8 @@ export function useSpeechmaticsAudio(
|
||||
const [isMuted, setIsMuted] = useState(false);
|
||||
const [segmentDuration, setSegmentDuration] = useState(0);
|
||||
|
||||
const enableContinuousStreaming = config.enableContinuousStreaming === true;
|
||||
|
||||
const audioBufferRef = useRef<Uint8Array[]>([]);
|
||||
const silenceStartRef = useRef<number | null>(null);
|
||||
const isSpeakingRef = useRef(false);
|
||||
@@ -136,6 +147,15 @@ export function useSpeechmaticsAudio(
|
||||
|
||||
const pcmData: Uint8Array = event.data;
|
||||
|
||||
if (enableContinuousStreaming) {
|
||||
audioBufferRef.current.push(pcmData);
|
||||
bufferedBytesRef.current += pcmData.length;
|
||||
if (bufferedBytesRef.current >= MIN_CHUNK_BYTES) {
|
||||
flushBufferedAudio(false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Buffer the audio chunk if we're detecting or speaking
|
||||
// Start buffering from first spike to capture beginning of speech
|
||||
if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
|
||||
@@ -150,7 +170,7 @@ export function useSpeechmaticsAudio(
|
||||
}
|
||||
}
|
||||
},
|
||||
[isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
|
||||
[enableContinuousStreaming, isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
|
||||
)
|
||||
);
|
||||
|
||||
@@ -165,6 +185,7 @@ export function useSpeechmaticsAudio(
|
||||
setVolume(volumeLevel);
|
||||
|
||||
if (isMuted) return;
|
||||
if (enableContinuousStreaming) return;
|
||||
|
||||
const speechDetected = volumeLevel > VOLUME_THRESHOLD;
|
||||
|
||||
@@ -270,6 +291,7 @@ export function useSpeechmaticsAudio(
|
||||
}
|
||||
},
|
||||
[
|
||||
enableContinuousStreaming,
|
||||
isActive,
|
||||
isMuted,
|
||||
VOLUME_THRESHOLD,
|
||||
@@ -341,6 +363,10 @@ export function useSpeechmaticsAudio(
|
||||
toggleRecording(false);
|
||||
}
|
||||
|
||||
if (enableContinuousStreaming) {
|
||||
flushBufferedAudio(true);
|
||||
}
|
||||
|
||||
// Tear down audio session
|
||||
if (audioInitialized) {
|
||||
tearDown();
|
||||
|
||||
@@ -5,6 +5,7 @@ export interface SpeechmaticsAudioConfig {
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
enableContinuousStreaming?: boolean;
|
||||
volumeThreshold: number;
|
||||
silenceDuration: number;
|
||||
speechConfirmationDuration: number;
|
||||
|
||||
@@ -42,7 +42,6 @@ import type {
|
||||
SendAgentMessage,
|
||||
SessionInboundMessage,
|
||||
SessionOutboundMessage,
|
||||
TranscriptionResultMessage,
|
||||
} from "../shared/messages.js";
|
||||
import type {
|
||||
AgentPermissionRequest,
|
||||
@@ -150,12 +149,6 @@ export type DaemonClientV2Config = {
|
||||
|
||||
export type SendMessageOptions = Pick<SendAgentMessage, "messageId" | "images">;
|
||||
|
||||
export type TranscribeAudioOptions = {
|
||||
audio: string; // base64 encoded
|
||||
format: string;
|
||||
timeout?: number;
|
||||
};
|
||||
|
||||
type AgentConfigOverrides = Partial<Omit<AgentSessionConfig, "provider" | "cwd">>;
|
||||
|
||||
export type CreateAgentRequestOptions = {
|
||||
@@ -188,7 +181,6 @@ type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"];
|
||||
type ListProviderModelsPayload = ListProviderModelsResponseMessage["payload"];
|
||||
type ListCommandsPayload = ListCommandsResponse["payload"];
|
||||
type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
|
||||
type TranscriptionResultPayload = TranscriptionResultMessage["payload"];
|
||||
type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
|
||||
type ListTerminalsPayload = ListTerminalsResponse["payload"];
|
||||
type CreateTerminalPayload = CreateTerminalResponse["payload"];
|
||||
@@ -472,7 +464,26 @@ export class DaemonClientV2 {
|
||||
throw new Error("Transport not connected");
|
||||
}
|
||||
const payload = SessionInboundMessageSchema.parse(message);
|
||||
this.transport.send(JSON.stringify({ type: "session", message: payload }));
|
||||
try {
|
||||
this.transport.send(JSON.stringify({ type: "session", message: payload }));
|
||||
} catch (error) {
|
||||
if (this.config.suppressSendErrors) {
|
||||
return;
|
||||
}
|
||||
throw error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
}
|
||||
|
||||
private sendSessionMessageStrict(message: SessionInboundMessage): void {
|
||||
if (!this.transport || this.connectionState.status !== "connected") {
|
||||
throw new Error("Transport not connected");
|
||||
}
|
||||
const payload = SessionInboundMessageSchema.parse(message);
|
||||
try {
|
||||
this.transport.send(JSON.stringify({ type: "session", message: payload }));
|
||||
} catch (error) {
|
||||
throw error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
}
|
||||
|
||||
sendUserMessage(text: string): void {
|
||||
@@ -839,37 +850,6 @@ export class DaemonClientV2 {
|
||||
await this.sendAgentMessage(agentId, text, options);
|
||||
}
|
||||
|
||||
async transcribeAudio(
|
||||
options: TranscribeAudioOptions
|
||||
): Promise<TranscriptionResultPayload> {
|
||||
const requestId = this.createRequestId();
|
||||
const timeout = options.timeout ?? 120000;
|
||||
|
||||
const responsePromise = this.waitFor(
|
||||
(msg) => {
|
||||
if (msg.type !== "transcription_result") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.requestId !== requestId) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
timeout,
|
||||
{ skipQueue: true }
|
||||
);
|
||||
|
||||
const message = SessionInboundMessageSchema.parse({
|
||||
type: "transcribe_audio_request",
|
||||
audio: options.audio,
|
||||
format: options.format,
|
||||
requestId,
|
||||
});
|
||||
this.sendSessionMessage(message);
|
||||
|
||||
return responsePromise;
|
||||
}
|
||||
|
||||
async cancelAgent(agentId: string): Promise<void> {
|
||||
this.sendSessionMessage({ type: "cancel_agent_request", agentId });
|
||||
}
|
||||
@@ -929,6 +909,87 @@ export class DaemonClientV2 {
|
||||
this.sendSessionMessage({ type: "realtime_audio_chunk", audio, format, isLast });
|
||||
}
|
||||
|
||||
startDictationStream(dictationId: string, format: string): Promise<void> {
|
||||
const ackPromise = this.waitFor(
|
||||
(msg) => {
|
||||
if (msg.type !== "dictation_stream_ack") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.dictationId !== dictationId) {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.ackSeq !== -1) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
30000,
|
||||
{ skipQueue: true }
|
||||
).then(() => undefined);
|
||||
|
||||
const errorPromise = this.waitFor(
|
||||
(msg) => {
|
||||
if (msg.type !== "dictation_stream_error") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.dictationId !== dictationId) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
30000,
|
||||
{ skipQueue: true }
|
||||
).then((payload) => {
|
||||
throw new Error(payload.error);
|
||||
});
|
||||
|
||||
this.sendSessionMessageStrict({ type: "dictation_stream_start", dictationId, format });
|
||||
return Promise.race([ackPromise, errorPromise]);
|
||||
}
|
||||
|
||||
sendDictationStreamChunk(dictationId: string, seq: number, audio: string, format: string): void {
|
||||
this.sendSessionMessageStrict({ type: "dictation_stream_chunk", dictationId, seq, audio, format });
|
||||
}
|
||||
|
||||
finishDictationStream(dictationId: string, finalSeq: number): Promise<{ dictationId: string; text: string }> {
|
||||
const finalPromise = this.waitFor(
|
||||
(msg) => {
|
||||
if (msg.type !== "dictation_stream_final") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.dictationId !== dictationId) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
120000,
|
||||
{ skipQueue: true }
|
||||
);
|
||||
|
||||
const errorPromise = this.waitFor(
|
||||
(msg) => {
|
||||
if (msg.type !== "dictation_stream_error") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.dictationId !== dictationId) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
120000,
|
||||
{ skipQueue: true }
|
||||
).then((payload) => {
|
||||
throw new Error(payload.error);
|
||||
});
|
||||
|
||||
this.sendSessionMessageStrict({ type: "dictation_stream_finish", dictationId, finalSeq });
|
||||
return Promise.race([finalPromise, errorPromise]);
|
||||
}
|
||||
|
||||
cancelDictationStream(dictationId: string): void {
|
||||
this.sendSessionMessageStrict({ type: "dictation_stream_cancel", dictationId });
|
||||
}
|
||||
|
||||
async abortRequest(): Promise<void> {
|
||||
this.sendSessionMessage({ type: "abort_request" });
|
||||
}
|
||||
@@ -2097,7 +2158,12 @@ function createWebSocketTransportFactory(
|
||||
return ({ url, headers }) => {
|
||||
const ws = factory(url, { headers });
|
||||
return {
|
||||
send: (data) => ws.send(data),
|
||||
send: (data) => {
|
||||
if (typeof ws.readyState === "number" && ws.readyState !== 1) {
|
||||
throw new Error(`WebSocket not open (readyState=${ws.readyState})`);
|
||||
}
|
||||
ws.send(data);
|
||||
},
|
||||
close: (code?: number, reason?: string) => ws.close(code, reason),
|
||||
onOpen: (handler) => bindWsHandler(ws, "open", handler),
|
||||
onClose: (handler) => bindWsHandler(ws, "close", handler),
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import type pino from "pino";
|
||||
import WebSocket from "ws";
|
||||
import { EventEmitter } from "node:events";
|
||||
|
||||
type OpenAIClientEvent =
|
||||
| {
|
||||
type: "session.update";
|
||||
session: {
|
||||
type: "transcription";
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm"; rate: 24000 };
|
||||
transcription: {
|
||||
model: string;
|
||||
language?: string;
|
||||
prompt?: string;
|
||||
};
|
||||
turn_detection: null;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
| { type: "input_audio_buffer.append"; audio: string }
|
||||
| { type: "input_audio_buffer.commit" }
|
||||
| { type: "input_audio_buffer.clear" };
|
||||
|
||||
type OpenAIServerEvent =
|
||||
| { type: "session.created" | "session.updated" }
|
||||
| {
|
||||
type: "input_audio_buffer.committed";
|
||||
item_id: string;
|
||||
previous_item_id: string | null;
|
||||
}
|
||||
| {
|
||||
type: "conversation.item.input_audio_transcription.completed";
|
||||
item_id: string;
|
||||
transcript: string;
|
||||
}
|
||||
| { type: "error"; error?: { message?: string } };
|
||||
|
||||
export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
|
||||
private readonly apiKey: string;
|
||||
private readonly logger: pino.Logger;
|
||||
private readonly transcriptionModel: string;
|
||||
private readonly language?: string;
|
||||
|
||||
private ws: WebSocket | null = null;
|
||||
private ready: Promise<void> | null = null;
|
||||
private closing = false;
|
||||
|
||||
constructor(params: {
|
||||
apiKey: string;
|
||||
logger: pino.Logger;
|
||||
transcriptionModel: string;
|
||||
language?: string;
|
||||
}) {
|
||||
super();
|
||||
this.apiKey = params.apiKey;
|
||||
this.logger = params.logger.child({ provider: "openai", component: "realtime-transcription" });
|
||||
this.transcriptionModel = params.transcriptionModel;
|
||||
this.language = params.language;
|
||||
}
|
||||
|
||||
public async connect(): Promise<void> {
|
||||
if (this.ready) {
|
||||
return this.ready;
|
||||
}
|
||||
|
||||
this.closing = false;
|
||||
this.ready = new Promise<void>((resolve, reject) => {
|
||||
const url = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||
const ws = new WebSocket(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
},
|
||||
});
|
||||
this.ws = ws;
|
||||
|
||||
let resolved = false;
|
||||
|
||||
const fail = (error: Error) => {
|
||||
if (resolved) {
|
||||
this.emit("error", error);
|
||||
return;
|
||||
}
|
||||
resolved = true;
|
||||
reject(error);
|
||||
};
|
||||
|
||||
ws.on("open", () => {
|
||||
this.logger.debug("OpenAI realtime transcription websocket connected");
|
||||
const update: OpenAIClientEvent = {
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "transcription",
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm", rate: 24000 },
|
||||
transcription: {
|
||||
model: this.transcriptionModel,
|
||||
...(this.language ? { language: this.language } : {}),
|
||||
},
|
||||
// We commit periodically ourselves; no server-side VAD for dictation.
|
||||
turn_detection: null,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
ws.send(JSON.stringify(update));
|
||||
});
|
||||
|
||||
ws.on("message", (data) => {
|
||||
const text = typeof data === "string" ? data : data.toString("utf-8");
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
const event = parsed as OpenAIServerEvent;
|
||||
if (
|
||||
event.type === "session.created" ||
|
||||
event.type === "session.updated"
|
||||
) {
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
resolve();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type === "input_audio_buffer.committed") {
|
||||
this.emit("committed", {
|
||||
itemId: event.item_id,
|
||||
previousItemId: event.previous_item_id,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type === "conversation.item.input_audio_transcription.completed") {
|
||||
this.emit("transcript", { itemId: event.item_id, transcript: event.transcript });
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type === "error") {
|
||||
const message = event.error?.message ?? "OpenAI realtime error";
|
||||
fail(new Error(message));
|
||||
}
|
||||
});
|
||||
|
||||
ws.on("error", (err) => {
|
||||
fail(err instanceof Error ? err : new Error(String(err)));
|
||||
});
|
||||
|
||||
ws.on("close", () => {
|
||||
this.logger.debug("OpenAI realtime websocket closed");
|
||||
if (this.closing) {
|
||||
return;
|
||||
}
|
||||
if (!resolved) {
|
||||
fail(new Error("OpenAI realtime websocket closed before ready"));
|
||||
return;
|
||||
}
|
||||
fail(new Error("OpenAI realtime websocket closed"));
|
||||
});
|
||||
});
|
||||
|
||||
return this.ready;
|
||||
}
|
||||
|
||||
public appendPcm16Base64(base64Audio: string): void {
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
||||
throw new Error("OpenAI realtime websocket not connected");
|
||||
}
|
||||
const event: OpenAIClientEvent = { type: "input_audio_buffer.append", audio: base64Audio };
|
||||
this.ws.send(JSON.stringify(event));
|
||||
}
|
||||
|
||||
public commit(): void {
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
||||
throw new Error("OpenAI realtime websocket not connected");
|
||||
}
|
||||
const event: OpenAIClientEvent = { type: "input_audio_buffer.commit" };
|
||||
this.ws.send(JSON.stringify(event));
|
||||
}
|
||||
|
||||
public clear(): void {
|
||||
if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
|
||||
return;
|
||||
}
|
||||
const event: OpenAIClientEvent = { type: "input_audio_buffer.clear" };
|
||||
this.ws.send(JSON.stringify(event));
|
||||
}
|
||||
|
||||
public close(): void {
|
||||
try {
|
||||
this.closing = true;
|
||||
this.ws?.close();
|
||||
} catch {
|
||||
// no-op
|
||||
} finally {
|
||||
this.ws = null;
|
||||
this.ready = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
83
packages/server/src/server/agent/pcm16-resampler.ts
Normal file
83
packages/server/src/server/agent/pcm16-resampler.ts
Normal file
@@ -0,0 +1,83 @@
|
||||
export class Pcm16MonoResampler {
|
||||
private readonly inputRate: number;
|
||||
private readonly outputRate: number;
|
||||
private readonly step: number;
|
||||
private pos: number;
|
||||
private carrySample: number | null;
|
||||
|
||||
constructor(params: { inputRate: number; outputRate: number }) {
|
||||
this.inputRate = params.inputRate;
|
||||
this.outputRate = params.outputRate;
|
||||
this.step = this.inputRate / this.outputRate;
|
||||
this.pos = 0;
|
||||
this.carrySample = null;
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this.pos = 0;
|
||||
this.carrySample = null;
|
||||
}
|
||||
|
||||
public processChunk(pcm16le: Buffer): Buffer {
|
||||
if (pcm16le.length === 0) {
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
if (pcm16le.length % 2 !== 0) {
|
||||
throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
|
||||
}
|
||||
|
||||
const srcChunk = new Int16Array(
|
||||
pcm16le.buffer,
|
||||
pcm16le.byteOffset,
|
||||
pcm16le.byteLength / 2
|
||||
);
|
||||
|
||||
const hasCarry = this.carrySample !== null;
|
||||
const srcLen = srcChunk.length + (hasCarry ? 1 : 0);
|
||||
if (srcLen < 2) {
|
||||
this.carrySample = srcChunk.length ? srcChunk[srcChunk.length - 1] : this.carrySample;
|
||||
return Buffer.alloc(0);
|
||||
}
|
||||
|
||||
const src = new Float32Array(srcLen);
|
||||
let offset = 0;
|
||||
if (hasCarry) {
|
||||
src[0] = (this.carrySample as number) / 32768;
|
||||
offset = 1;
|
||||
}
|
||||
for (let i = 0; i < srcChunk.length; i += 1) {
|
||||
src[offset + i] = srcChunk[i] / 32768;
|
||||
}
|
||||
|
||||
const out: number[] = [];
|
||||
const maxPos = src.length - 1;
|
||||
|
||||
while (this.pos < maxPos) {
|
||||
const i = Math.floor(this.pos);
|
||||
const frac = this.pos - i;
|
||||
const s0 = src[i]!;
|
||||
const s1 = src[i + 1]!;
|
||||
const sample = s0 + (s1 - s0) * frac;
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
const int16 = Math.round(clamped * 32767);
|
||||
out.push(int16);
|
||||
this.pos += this.step;
|
||||
}
|
||||
|
||||
// Keep the last input sample as carry for the next chunk.
|
||||
const lastInput = srcChunk[srcChunk.length - 1]!;
|
||||
this.carrySample = lastInput;
|
||||
|
||||
// Shift position so next chunk (which will include carry sample) continues smoothly.
|
||||
const shift = src.length - 1;
|
||||
this.pos = this.pos - shift;
|
||||
if (this.pos < 0) {
|
||||
// Guard against floating point drift.
|
||||
this.pos = 0;
|
||||
}
|
||||
|
||||
const outArr = Int16Array.from(out);
|
||||
return Buffer.from(outArr.buffer, outArr.byteOffset, outArr.byteLength);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,9 @@ if (envPath) {
|
||||
dotenv.config({ path: envPath });
|
||||
}
|
||||
|
||||
// Make dictation streaming commit frequently in tests so we exercise multi-commit assembly logic.
|
||||
process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ??= "1000";
|
||||
|
||||
function tmpCwd(): string {
|
||||
return mkdtempSync(path.join(tmpdir(), "daemon-client-v2-"));
|
||||
}
|
||||
@@ -630,6 +633,216 @@ describe("daemon client v2 E2E", () => {
|
||||
180000
|
||||
);
|
||||
|
||||
test(
|
||||
"streams dictation PCM and returns final transcript via OpenAI Realtime transcription",
|
||||
async () => {
|
||||
if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
|
||||
// Requires OpenAI realtime transcription and is inherently network-flaky.
|
||||
return;
|
||||
}
|
||||
if (!process.env.OPENAI_API_KEY) {
|
||||
return;
|
||||
}
|
||||
|
||||
const fixturePath = path.resolve(
|
||||
process.cwd(),
|
||||
"..",
|
||||
"app",
|
||||
"e2e",
|
||||
"fixtures",
|
||||
"recording.wav"
|
||||
);
|
||||
const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
|
||||
|
||||
const parsePcm16MonoWav = (buffer: Buffer): Buffer => {
|
||||
if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
|
||||
throw new Error("Invalid WAV header");
|
||||
}
|
||||
let offset = 12;
|
||||
let fmt: { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } | null = null;
|
||||
let dataChunk: Buffer | null = null;
|
||||
|
||||
while (offset + 8 <= buffer.length) {
|
||||
const id = buffer.toString("ascii", offset, offset + 4);
|
||||
const size = buffer.readUInt32LE(offset + 4);
|
||||
const payloadStart = offset + 8;
|
||||
const payloadEnd = payloadStart + size;
|
||||
if (payloadEnd > buffer.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (id === "fmt ") {
|
||||
const audioFormat = buffer.readUInt16LE(payloadStart);
|
||||
const channels = buffer.readUInt16LE(payloadStart + 2);
|
||||
const sampleRate = buffer.readUInt32LE(payloadStart + 4);
|
||||
const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
|
||||
fmt = { audioFormat, channels, sampleRate, bitsPerSample };
|
||||
} else if (id === "data") {
|
||||
dataChunk = buffer.subarray(payloadStart, payloadEnd);
|
||||
}
|
||||
|
||||
offset = payloadEnd + (size % 2);
|
||||
}
|
||||
|
||||
if (!fmt || !dataChunk) {
|
||||
throw new Error("Missing WAV fmt/data chunks");
|
||||
}
|
||||
if (fmt.audioFormat !== 1) {
|
||||
throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
|
||||
}
|
||||
if (fmt.channels !== 1 || fmt.sampleRate !== 16000 || fmt.bitsPerSample !== 16) {
|
||||
throw new Error(
|
||||
`Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
|
||||
);
|
||||
}
|
||||
if (dataChunk.length % 2 !== 0) {
|
||||
throw new Error("WAV PCM16 data length must be even");
|
||||
}
|
||||
return dataChunk;
|
||||
};
|
||||
|
||||
const pcm16 = parsePcm16MonoWav(wav);
|
||||
const dictationId = `dict-${Date.now()}`;
|
||||
const format = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
await ctx.client.startDictationStream(dictationId, format);
|
||||
|
||||
const chunkBytes = 3200; // ~100ms @ 16kHz mono PCM16 (1600 samples * 2 bytes)
|
||||
let seq = 0;
|
||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||
ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
|
||||
seq += 1;
|
||||
}
|
||||
|
||||
const finalSeq = seq - 1;
|
||||
const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
|
||||
|
||||
expect(result.dictationId).toBe(dictationId);
|
||||
expect(result.text.toLowerCase()).toContain("voice note");
|
||||
},
|
||||
180000
|
||||
);
|
||||
|
||||
test(
|
||||
"does not commit an empty OpenAI realtime audio buffer on finish",
|
||||
async () => {
|
||||
if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
|
||||
return;
|
||||
}
|
||||
if (!process.env.OPENAI_API_KEY) {
|
||||
return;
|
||||
}
|
||||
|
||||
const fixturePath = path.resolve(
|
||||
process.cwd(),
|
||||
"..",
|
||||
"app",
|
||||
"e2e",
|
||||
"fixtures",
|
||||
"recording.wav"
|
||||
);
|
||||
const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
|
||||
|
||||
const parsePcm16MonoWav = (buffer: Buffer): Buffer => {
|
||||
if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
|
||||
throw new Error("Invalid WAV header");
|
||||
}
|
||||
let offset = 12;
|
||||
let fmt: { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } | null = null;
|
||||
let dataChunk: Buffer | null = null;
|
||||
|
||||
while (offset + 8 <= buffer.length) {
|
||||
const id = buffer.toString("ascii", offset, offset + 4);
|
||||
const size = buffer.readUInt32LE(offset + 4);
|
||||
const payloadStart = offset + 8;
|
||||
const payloadEnd = payloadStart + size;
|
||||
if (payloadEnd > buffer.length) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (id === "fmt ") {
|
||||
const audioFormat = buffer.readUInt16LE(payloadStart);
|
||||
const channels = buffer.readUInt16LE(payloadStart + 2);
|
||||
const sampleRate = buffer.readUInt32LE(payloadStart + 4);
|
||||
const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
|
||||
fmt = { audioFormat, channels, sampleRate, bitsPerSample };
|
||||
} else if (id === "data") {
|
||||
dataChunk = buffer.subarray(payloadStart, payloadEnd);
|
||||
}
|
||||
|
||||
offset = payloadEnd + (size % 2);
|
||||
}
|
||||
|
||||
if (!fmt || !dataChunk) {
|
||||
throw new Error("Missing WAV fmt/data chunks");
|
||||
}
|
||||
if (fmt.audioFormat !== 1) {
|
||||
throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
|
||||
}
|
||||
if (fmt.channels !== 1 || fmt.sampleRate !== 16000 || fmt.bitsPerSample !== 16) {
|
||||
throw new Error(
|
||||
`Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
|
||||
);
|
||||
}
|
||||
if (dataChunk.length % 2 !== 0) {
|
||||
throw new Error("WAV PCM16 data length must be even");
|
||||
}
|
||||
return dataChunk;
|
||||
};
|
||||
|
||||
const pcm16 = parsePcm16MonoWav(wav);
|
||||
const dictationId = `dict-empty-commit-${Date.now()}`;
|
||||
const format = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
await ctx.client.startDictationStream(dictationId, format);
|
||||
|
||||
// Send exactly 10x 100ms chunks. With OPENAI_REALTIME_DICTATION_COMMIT_MS=1000 in this test file,
|
||||
// the server will auto-commit at the 1s boundary. Finishing immediately after that should not
|
||||
// attempt an additional empty commit.
|
||||
const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
|
||||
const targetChunks = 10;
|
||||
let seq = 0;
|
||||
for (let i = 0; i < targetChunks; i += 1) {
|
||||
const start = i * chunkBytes;
|
||||
const end = Math.min(pcm16.length, start + chunkBytes);
|
||||
const chunk = pcm16.subarray(start, end);
|
||||
ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
|
||||
seq += 1;
|
||||
}
|
||||
|
||||
const finalSeq = seq - 1;
|
||||
const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
|
||||
|
||||
expect(result.dictationId).toBe(dictationId);
|
||||
expect(typeof result.text).toBe("string");
|
||||
},
|
||||
180000
|
||||
);
|
||||
|
||||
test(
|
||||
"fails fast if dictation finishes without sending required chunks",
|
||||
async () => {
|
||||
if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
|
||||
return;
|
||||
}
|
||||
if (!process.env.OPENAI_API_KEY) {
|
||||
return;
|
||||
}
|
||||
|
||||
const dictationId = `dict-missing-chunks-${Date.now()}`;
|
||||
const format = "audio/pcm;rate=16000;bits=16";
|
||||
|
||||
await ctx.client.startDictationStream(dictationId, format);
|
||||
|
||||
// Claim that we sent chunk 0, but actually send no chunks.
|
||||
await expect(ctx.client.finishDictationStream(dictationId, 0)).rejects.toThrow(
|
||||
/no audio chunks were received/i
|
||||
);
|
||||
},
|
||||
180000
|
||||
);
|
||||
|
||||
test(
|
||||
"supports git and file operations",
|
||||
async () => {
|
||||
|
||||
@@ -35,6 +35,8 @@ import { TTSManager } from "./agent/tts-manager.js";
|
||||
import { STTManager } from "./agent/stt-manager.js";
|
||||
import type { OpenAISTT } from "./agent/stt-openai.js";
|
||||
import type { OpenAITTS } from "./agent/tts-openai.js";
|
||||
import { OpenAIRealtimeTranscriptionSession } from "./agent/openai-realtime-transcription.js";
|
||||
import { Pcm16MonoResampler } from "./agent/pcm16-resampler.js";
|
||||
import type { VoiceConversationStore } from "./voice-conversation-store.js";
|
||||
import {
|
||||
buildConfigOverrides,
|
||||
@@ -135,6 +137,21 @@ const MIN_STREAMING_SEGMENT_DURATION_MS = 1000;
|
||||
const MIN_STREAMING_SEGMENT_BYTES = Math.round(
|
||||
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
||||
);
|
||||
|
||||
const DICTATION_PCM_INPUT_RATE = 16000;
|
||||
const DICTATION_PCM_OUTPUT_RATE = 24000;
|
||||
const DICTATION_COMMIT_INTERVAL_MS = Number.parseInt(
|
||||
process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ?? "5000",
|
||||
10
|
||||
);
|
||||
const DICTATION_COMMIT_TARGET_BYTES = Math.round(
|
||||
(DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_COMMIT_INTERVAL_MS) / 1000
|
||||
);
|
||||
const DICTATION_MIN_COMMIT_MS = 100;
|
||||
const DICTATION_MIN_COMMIT_BYTES = Math.round(
|
||||
(DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_MIN_COMMIT_MS) / 1000
|
||||
);
|
||||
const DICTATION_FINAL_TIMEOUT_MS = 120000;
|
||||
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
|
||||
|
||||
/**
|
||||
@@ -249,6 +266,27 @@ export class Session {
|
||||
private speechInProgress = false;
|
||||
private voiceConversationId: string | null = null;
|
||||
|
||||
private dictationStreams = new Map<
|
||||
string,
|
||||
{
|
||||
dictationId: string;
|
||||
inputFormat: string;
|
||||
openai: OpenAIRealtimeTranscriptionSession;
|
||||
resampler: Pcm16MonoResampler;
|
||||
receivedChunks: Map<number, Buffer>;
|
||||
nextSeqToForward: number;
|
||||
ackSeq: number;
|
||||
bytesSinceCommit: number;
|
||||
expectedCommits: number;
|
||||
committedCount: number;
|
||||
committedItemIds: string[];
|
||||
transcriptsByItemId: Map<string, string>;
|
||||
finishRequested: boolean;
|
||||
finalSeq: number | null;
|
||||
finalTimeout: ReturnType<typeof setTimeout> | null;
|
||||
}
|
||||
>();
|
||||
|
||||
// Audio buffering for interruption handling
|
||||
private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
|
||||
private bufferTimeout: NodeJS.Timeout | null = null;
|
||||
@@ -735,8 +773,20 @@ export class Session {
|
||||
);
|
||||
break;
|
||||
|
||||
case "transcribe_audio_request":
|
||||
await this.handleTranscribeAudio(msg);
|
||||
case "dictation_stream_start":
|
||||
await this.handleDictationStreamStart(msg);
|
||||
break;
|
||||
|
||||
case "dictation_stream_chunk":
|
||||
await this.handleDictationStreamChunk(msg);
|
||||
break;
|
||||
|
||||
case "dictation_stream_finish":
|
||||
await this.handleDictationStreamFinish(msg);
|
||||
break;
|
||||
|
||||
case "dictation_stream_cancel":
|
||||
await this.handleDictationStreamCancel(msg);
|
||||
break;
|
||||
|
||||
case "create_agent_request":
|
||||
@@ -1177,56 +1227,279 @@ export class Session {
|
||||
this.startAgentStream(agentId, prompt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle audio transcription request
|
||||
*/
|
||||
private async handleTranscribeAudio(
|
||||
msg: Extract<SessionInboundMessage, { type: "transcribe_audio_request" }>
|
||||
): Promise<void> {
|
||||
const { audio, format, requestId } = msg;
|
||||
private emitDictationAck(dictationId: string, ackSeq: number): void {
|
||||
this.emit({
|
||||
type: "dictation_stream_ack",
|
||||
payload: { dictationId, ackSeq },
|
||||
});
|
||||
}
|
||||
|
||||
const audioBuffer = Buffer.from(audio, "base64");
|
||||
private failDictationStream(dictationId: string, error: string, retryable: boolean): void {
|
||||
this.emit({
|
||||
type: "dictation_stream_error",
|
||||
payload: { dictationId, error, retryable },
|
||||
});
|
||||
}
|
||||
|
||||
this.sessionLogger.debug({ requestId }, "Transcribing audio");
|
||||
|
||||
try {
|
||||
const result = await this.sttManager.transcribe(audioBuffer, format, {
|
||||
requestId,
|
||||
label: "dictation",
|
||||
});
|
||||
|
||||
this.sessionLogger.info(
|
||||
{ requestId, textLength: result.text.length },
|
||||
"Transcription complete"
|
||||
);
|
||||
|
||||
this.emit({
|
||||
type: "transcription_result",
|
||||
payload: {
|
||||
text: result.text,
|
||||
language: result.language,
|
||||
duration: result.duration,
|
||||
requestId,
|
||||
avgLogprob: result.avgLogprob,
|
||||
isLowConfidence: result.isLowConfidence,
|
||||
byteLength: result.byteLength,
|
||||
format: result.format,
|
||||
debugRecordingPath: result.debugRecordingPath,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
this.sessionLogger.error({ err: error, requestId }, "Transcription failed");
|
||||
this.emit({
|
||||
type: "activity_log",
|
||||
payload: {
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: "error",
|
||||
content: `Transcription failed: ${error.message}`,
|
||||
},
|
||||
});
|
||||
throw error;
|
||||
private cleanupDictationStream(dictationId: string): void {
|
||||
const state = this.dictationStreams.get(dictationId) ?? null;
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
if (state.finalTimeout) {
|
||||
clearTimeout(state.finalTimeout);
|
||||
}
|
||||
try {
|
||||
state.openai.close();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
this.dictationStreams.delete(dictationId);
|
||||
}
|
||||
|
||||
private maybeFinalizeDictationStream(dictationId: string): void {
|
||||
const state = this.dictationStreams.get(dictationId);
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!state.finishRequested || state.finalSeq === null) {
|
||||
return;
|
||||
}
|
||||
if (state.ackSeq < state.finalSeq) {
|
||||
return;
|
||||
}
|
||||
if (state.committedCount < state.expectedCommits) {
|
||||
return;
|
||||
}
|
||||
|
||||
const allTranscriptsReady = state.committedItemIds.every((itemId) =>
|
||||
state.transcriptsByItemId.has(itemId)
|
||||
);
|
||||
if (!allTranscriptsReady) {
|
||||
return;
|
||||
}
|
||||
|
||||
const orderedText = state.committedItemIds
|
||||
.map((itemId) => state.transcriptsByItemId.get(itemId) ?? "")
|
||||
.join(" ")
|
||||
.trim();
|
||||
|
||||
this.emit({
|
||||
type: "dictation_stream_final",
|
||||
payload: { dictationId, text: orderedText },
|
||||
});
|
||||
|
||||
this.cleanupDictationStream(dictationId);
|
||||
}
|
||||
|
||||
private async handleDictationStreamStart(
|
||||
msg: Extract<SessionInboundMessage, { type: "dictation_stream_start" }>
|
||||
): Promise<void> {
|
||||
const dictationId = msg.dictationId;
|
||||
this.cleanupDictationStream(dictationId);
|
||||
|
||||
const apiKey = process.env.OPENAI_API_KEY;
|
||||
if (!apiKey) {
|
||||
this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false);
|
||||
return;
|
||||
}
|
||||
|
||||
const transcriptionModel =
|
||||
process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
|
||||
|
||||
const openai = new OpenAIRealtimeTranscriptionSession({
|
||||
apiKey,
|
||||
logger: this.sessionLogger.child({ dictationId }),
|
||||
transcriptionModel,
|
||||
language: "en",
|
||||
});
|
||||
|
||||
openai.on("committed", ({ itemId }: { itemId: string }) => {
|
||||
const state = this.dictationStreams.get(dictationId);
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
state.committedCount += 1;
|
||||
state.committedItemIds.push(itemId);
|
||||
this.maybeFinalizeDictationStream(dictationId);
|
||||
});
|
||||
|
||||
openai.on("transcript", ({ itemId, transcript }: { itemId: string; transcript: string }) => {
|
||||
const state = this.dictationStreams.get(dictationId);
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
state.transcriptsByItemId.set(itemId, transcript);
|
||||
this.maybeFinalizeDictationStream(dictationId);
|
||||
});
|
||||
|
||||
openai.on("error", (err) => {
|
||||
const error = err instanceof Error ? err.message : String(err);
|
||||
this.failDictationStream(dictationId, error, true);
|
||||
this.cleanupDictationStream(dictationId);
|
||||
});
|
||||
|
||||
await openai.connect();
|
||||
|
||||
this.dictationStreams.set(dictationId, {
|
||||
dictationId,
|
||||
inputFormat: msg.format,
|
||||
openai,
|
||||
resampler: new Pcm16MonoResampler({
|
||||
inputRate: DICTATION_PCM_INPUT_RATE,
|
||||
outputRate: DICTATION_PCM_OUTPUT_RATE,
|
||||
}),
|
||||
receivedChunks: new Map(),
|
||||
nextSeqToForward: 0,
|
||||
ackSeq: -1,
|
||||
bytesSinceCommit: 0,
|
||||
expectedCommits: 0,
|
||||
committedCount: 0,
|
||||
committedItemIds: [],
|
||||
transcriptsByItemId: new Map(),
|
||||
finishRequested: false,
|
||||
finalSeq: null,
|
||||
finalTimeout: null,
|
||||
});
|
||||
|
||||
this.emitDictationAck(dictationId, -1);
|
||||
}
|
||||
|
||||
private async handleDictationStreamChunk(
|
||||
msg: Extract<SessionInboundMessage, { type: "dictation_stream_chunk" }>
|
||||
): Promise<void> {
|
||||
const state = this.dictationStreams.get(msg.dictationId);
|
||||
if (!state) {
|
||||
this.failDictationStream(msg.dictationId, "Dictation stream not started", true);
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.format !== state.inputFormat) {
|
||||
this.failDictationStream(
|
||||
msg.dictationId,
|
||||
`Mismatched dictation stream format: ${msg.format}`,
|
||||
false
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg.seq < state.nextSeqToForward) {
|
||||
this.emitDictationAck(msg.dictationId, state.ackSeq);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!state.receivedChunks.has(msg.seq)) {
|
||||
state.receivedChunks.set(msg.seq, Buffer.from(msg.audio, "base64"));
|
||||
}
|
||||
|
||||
while (state.receivedChunks.has(state.nextSeqToForward)) {
|
||||
const seq = state.nextSeqToForward;
|
||||
const pcm16 = state.receivedChunks.get(seq)!;
|
||||
state.receivedChunks.delete(seq);
|
||||
|
||||
const resampled = state.resampler.processChunk(pcm16);
|
||||
if (resampled.length > 0) {
|
||||
state.openai.appendPcm16Base64(resampled.toString("base64"));
|
||||
state.bytesSinceCommit += resampled.length;
|
||||
}
|
||||
|
||||
state.nextSeqToForward += 1;
|
||||
state.ackSeq = state.nextSeqToForward - 1;
|
||||
|
||||
if (state.bytesSinceCommit >= DICTATION_COMMIT_TARGET_BYTES) {
|
||||
state.expectedCommits += 1;
|
||||
state.openai.commit();
|
||||
state.bytesSinceCommit = 0;
|
||||
}
|
||||
}
|
||||
|
||||
this.emitDictationAck(msg.dictationId, state.ackSeq);
|
||||
this.maybeFinalizeDictationStream(msg.dictationId);
|
||||
}
|
||||
|
||||
private async handleDictationStreamFinish(
|
||||
msg: Extract<SessionInboundMessage, { type: "dictation_stream_finish" }>
|
||||
): Promise<void> {
|
||||
const state = this.dictationStreams.get(msg.dictationId);
|
||||
if (!state) {
|
||||
this.failDictationStream(msg.dictationId, "Dictation stream not started", true);
|
||||
return;
|
||||
}
|
||||
|
||||
state.finishRequested = true;
|
||||
state.finalSeq = msg.finalSeq;
|
||||
|
||||
// If the client claims it sent audio (finalSeq >= 0) but we haven't received any chunks at all,
|
||||
// we should fail fast instead of hanging until DICTATION_FINAL_TIMEOUT_MS.
|
||||
if (msg.finalSeq >= 0 && state.ackSeq < 0 && state.nextSeqToForward === 0 && state.receivedChunks.size === 0) {
|
||||
this.sessionLogger.debug(
|
||||
{
|
||||
dictationId: msg.dictationId,
|
||||
finalSeq: msg.finalSeq,
|
||||
ackSeq: state.ackSeq,
|
||||
nextSeqToForward: state.nextSeqToForward,
|
||||
receivedChunks: state.receivedChunks.size,
|
||||
expectedCommits: state.expectedCommits,
|
||||
committedCount: state.committedCount,
|
||||
},
|
||||
"Dictation finish: no chunks received (failing fast)"
|
||||
);
|
||||
this.failDictationStream(
|
||||
msg.dictationId,
|
||||
`Dictation finished (finalSeq=${msg.finalSeq}) but no audio chunks were received`,
|
||||
true
|
||||
);
|
||||
this.cleanupDictationStream(msg.dictationId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Commit any remaining uncommitted audio.
|
||||
// IMPORTANT: OpenAI requires at least ~100ms of audio in the buffer for a commit.
|
||||
// Also, committing an empty buffer throws "buffer too small ... 0.00ms".
|
||||
if (state.bytesSinceCommit > 0) {
|
||||
this.sessionLogger.debug(
|
||||
{ dictationId: msg.dictationId, bytesSinceCommit: state.bytesSinceCommit },
|
||||
"Dictation finish: committing pending audio"
|
||||
);
|
||||
if (state.bytesSinceCommit < DICTATION_MIN_COMMIT_BYTES) {
|
||||
const padBytes = DICTATION_MIN_COMMIT_BYTES - state.bytesSinceCommit;
|
||||
this.sessionLogger.debug(
|
||||
{ dictationId: msg.dictationId, padBytes },
|
||||
"Dictation finish: padding to minimum commit size"
|
||||
);
|
||||
state.openai.appendPcm16Base64(Buffer.alloc(padBytes).toString("base64"));
|
||||
state.bytesSinceCommit += padBytes;
|
||||
}
|
||||
state.expectedCommits += 1;
|
||||
state.openai.commit();
|
||||
state.bytesSinceCommit = 0;
|
||||
} else {
|
||||
this.sessionLogger.debug(
|
||||
{ dictationId: msg.dictationId },
|
||||
"Dictation finish: no pending audio to commit"
|
||||
);
|
||||
}
|
||||
|
||||
if (state.finalTimeout) {
|
||||
clearTimeout(state.finalTimeout);
|
||||
}
|
||||
state.finalTimeout = setTimeout(() => {
|
||||
this.failDictationStream(
|
||||
msg.dictationId,
|
||||
"Timed out waiting for final transcription",
|
||||
true
|
||||
);
|
||||
this.cleanupDictationStream(msg.dictationId);
|
||||
}, DICTATION_FINAL_TIMEOUT_MS);
|
||||
|
||||
this.maybeFinalizeDictationStream(msg.dictationId);
|
||||
}
|
||||
|
||||
private async handleDictationStreamCancel(
|
||||
msg: Extract<SessionInboundMessage, { type: "dictation_stream_cancel" }>
|
||||
): Promise<void> {
|
||||
this.cleanupDictationStream(msg.dictationId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -4209,6 +4482,10 @@ export class Session {
|
||||
this.ttsManager.cleanup();
|
||||
this.sttManager.cleanup();
|
||||
|
||||
for (const dictationId of this.dictationStreams.keys()) {
|
||||
this.cleanupDictationStream(dictationId);
|
||||
}
|
||||
|
||||
// Close MCP clients
|
||||
if (this.agentMcpClient) {
|
||||
try {
|
||||
|
||||
@@ -57,7 +57,13 @@ export class VoiceAssistantWebSocketServer {
|
||||
path: "/ws",
|
||||
verifyClient: ({ req }, callback) => {
|
||||
const origin = req.headers.origin;
|
||||
if (!origin || allowedOrigins.has(origin)) {
|
||||
const requestHost = typeof req.headers.host === "string" ? req.headers.host : null;
|
||||
const sameOrigin =
|
||||
!!origin &&
|
||||
!!requestHost &&
|
||||
(origin === `http://${requestHost}` || origin === `https://${requestHost}`);
|
||||
|
||||
if (!origin || allowedOrigins.has(origin) || sameOrigin) {
|
||||
callback(true);
|
||||
} else {
|
||||
this.logger.warn({ origin }, "Rejected connection from origin");
|
||||
|
||||
@@ -329,11 +329,33 @@ export const SendAgentMessageSchema = z.object({
|
||||
})).optional(),
|
||||
});
|
||||
|
||||
export const TranscribeAudioRequestSchema = z.object({
|
||||
type: z.literal("transcribe_audio_request"),
|
||||
audio: z.string(), // base64 encoded
|
||||
format: z.string(),
|
||||
requestId: z.string(),
|
||||
// ============================================================================
|
||||
// Dictation Streaming (lossless, resumable)
|
||||
// ============================================================================
|
||||
|
||||
export const DictationStreamStartMessageSchema = z.object({
|
||||
type: z.literal("dictation_stream_start"),
|
||||
dictationId: z.string(),
|
||||
format: z.string(), // e.g. "audio/pcm;rate=16000;bits=16"
|
||||
});
|
||||
|
||||
export const DictationStreamChunkMessageSchema = z.object({
|
||||
type: z.literal("dictation_stream_chunk"),
|
||||
dictationId: z.string(),
|
||||
seq: z.number().int().nonnegative(),
|
||||
audio: z.string(), // base64 encoded chunk
|
||||
format: z.string(), // e.g. "audio/pcm;rate=16000;bits=16"
|
||||
});
|
||||
|
||||
export const DictationStreamFinishMessageSchema = z.object({
|
||||
type: z.literal("dictation_stream_finish"),
|
||||
dictationId: z.string(),
|
||||
finalSeq: z.number().int().nonnegative(),
|
||||
});
|
||||
|
||||
export const DictationStreamCancelMessageSchema = z.object({
|
||||
type: z.literal("dictation_stream_cancel"),
|
||||
dictationId: z.string(),
|
||||
});
|
||||
|
||||
const GitSetupOptionsSchema = z.object({
|
||||
@@ -683,7 +705,10 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
|
||||
DeleteAgentRequestMessageSchema,
|
||||
SetVoiceConversationMessageSchema,
|
||||
SendAgentMessageSchema,
|
||||
TranscribeAudioRequestSchema,
|
||||
DictationStreamStartMessageSchema,
|
||||
DictationStreamChunkMessageSchema,
|
||||
DictationStreamFinishMessageSchema,
|
||||
DictationStreamCancelMessageSchema,
|
||||
CreateAgentRequestMessageSchema,
|
||||
ListProviderModelsRequestMessageSchema,
|
||||
ResumeAgentRequestMessageSchema,
|
||||
@@ -780,6 +805,31 @@ export const TranscriptionResultMessageSchema = z.object({
|
||||
}),
|
||||
});
|
||||
|
||||
export const DictationStreamAckMessageSchema = z.object({
|
||||
type: z.literal("dictation_stream_ack"),
|
||||
payload: z.object({
|
||||
dictationId: z.string(),
|
||||
ackSeq: z.number().int(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const DictationStreamFinalMessageSchema = z.object({
|
||||
type: z.literal("dictation_stream_final"),
|
||||
payload: z.object({
|
||||
dictationId: z.string(),
|
||||
text: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const DictationStreamErrorMessageSchema = z.object({
|
||||
type: z.literal("dictation_stream_error"),
|
||||
payload: z.object({
|
||||
dictationId: z.string(),
|
||||
error: z.string(),
|
||||
retryable: z.boolean(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const StatusMessageSchema = z.object({
|
||||
type: z.literal("status"),
|
||||
payload: z
|
||||
@@ -1244,6 +1294,9 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
||||
AssistantChunkMessageSchema,
|
||||
AudioOutputMessageSchema,
|
||||
TranscriptionResultMessageSchema,
|
||||
DictationStreamAckMessageSchema,
|
||||
DictationStreamFinalMessageSchema,
|
||||
DictationStreamErrorMessageSchema,
|
||||
StatusMessageSchema,
|
||||
InitializeAgentResponseMessageSchema,
|
||||
ArtifactMessageSchema,
|
||||
@@ -1323,7 +1376,10 @@ export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
|
||||
export type UserTextMessage = z.infer<typeof UserTextMessageSchema>;
|
||||
export type RealtimeAudioChunkMessage = z.infer<typeof RealtimeAudioChunkMessageSchema>;
|
||||
export type SendAgentMessage = z.infer<typeof SendAgentMessageSchema>;
|
||||
export type TranscribeAudioRequest = z.infer<typeof TranscribeAudioRequestSchema>;
|
||||
export type DictationStreamStartMessage = z.infer<typeof DictationStreamStartMessageSchema>;
|
||||
export type DictationStreamChunkMessage = z.infer<typeof DictationStreamChunkMessageSchema>;
|
||||
export type DictationStreamFinishMessage = z.infer<typeof DictationStreamFinishMessageSchema>;
|
||||
export type DictationStreamCancelMessage = z.infer<typeof DictationStreamCancelMessageSchema>;
|
||||
export type CreateAgentRequestMessage = z.infer<typeof CreateAgentRequestMessageSchema>;
|
||||
export type ListProviderModelsRequestMessage = z.infer<
|
||||
typeof ListProviderModelsRequestMessageSchema
|
||||
|
||||
Reference in New Issue
Block a user