diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx
index a1faf7338..e514a5244 100644
--- a/packages/app/src/components/dictation-controls.tsx
+++ b/packages/app/src/components/dictation-controls.tsx
@@ -18,7 +18,6 @@ interface DictationControlsProps {
onRetry?: () => void;
onDiscard?: () => void;
disabled?: boolean;
- retryStatusText?: string;
}
function formatDuration(seconds: number): string {
@@ -42,13 +41,11 @@ export function DictationControls({
onRetry,
onDiscard,
disabled = false,
- retryStatusText,
}: DictationControlsProps) {
const { theme } = useUnistyles();
- const isRetrying = status === "retrying";
const isFailed = status === "failed";
- const showActiveState = isRecording || isProcessing || isRetrying || isFailed;
- const actionsDisabled = isProcessing || isRetrying;
+ const showActiveState = isRecording || isProcessing || isFailed;
+ const actionsDisabled = isProcessing;
const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
if (!showActiveState) {
@@ -123,13 +120,6 @@ export function DictationControls({
)}
- {retryStatusText && (isRetrying || isFailed) && (
-
- {retryStatusText}
-
- )}
);
}
@@ -149,12 +139,11 @@ export function DictationOverlay({
onAcceptAndSend,
onRetry,
onDiscard,
-}: Omit) {
+}: Omit) {
const { theme } = useUnistyles();
- const isRetrying = status === "retrying";
const isFailed = status === "failed";
- const showActiveState = isRecording || isProcessing || isRetrying || isFailed;
- const actionsDisabled = isProcessing || isRetrying;
+ const showActiveState = isRecording || isProcessing || isFailed;
+ const actionsDisabled = isProcessing;
const handleCancel = isFailed && onDiscard ? onDiscard : onCancel;
if (!showActiveState) {
diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx
index 750fd072e..5b9a1fc2e 100644
--- a/packages/app/src/components/message-input.tsx
+++ b/packages/app/src/components/message-input.tsx
@@ -210,7 +210,7 @@ export const MessageInput = forwardRef(
});
useEffect(() => {
- if (isDictating || isDictationProcessing || dictationStatus === "retrying") {
+ if (isDictating || isDictationProcessing) {
return;
}
sendAfterTranscriptRef.current = false;
@@ -246,7 +246,6 @@ export const MessageInput = forwardRef(
const showOverlay =
isDictating ||
isDictationProcessing ||
- dictationStatus === "retrying" ||
dictationStatus === "failed";
overlayTransition.value = withTiming(showOverlay ? 1 : 0, {
duration: 200,
diff --git a/packages/app/src/components/volume-meter.tsx b/packages/app/src/components/volume-meter.tsx
index ff5b54d6d..6dab61d88 100644
--- a/packages/app/src/components/volume-meter.tsx
+++ b/packages/app/src/components/volume-meter.tsx
@@ -88,7 +88,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
return;
}
- if (volume > 0.01) {
+ if (volume > 0.001) {
// Active volume - animate heights based on volume
const target1 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.2);
const target2 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.05);
@@ -136,7 +136,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
return {
- height: line1Height.value * (isMuted || volume > 0.01 ? 1 : line1Pulse.value),
+ height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
@@ -146,7 +146,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
return {
- height: line2Height.value * (isMuted || volume > 0.01 ? 1 : line2Pulse.value),
+ height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
@@ -156,7 +156,7 @@ export function VolumeMeter({ volume, isMuted = false, isDetecting = false, isSp
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted ? 0 : volume * 0.3;
return {
- height: line3Height.value * (isMuted || volume > 0.01 ? 1 : line3Pulse.value),
+ height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
diff --git a/packages/app/src/dictation/dictation-stream-sender.test.ts b/packages/app/src/dictation/dictation-stream-sender.test.ts
new file mode 100644
index 000000000..3ebc4f1b1
--- /dev/null
+++ b/packages/app/src/dictation/dictation-stream-sender.test.ts
@@ -0,0 +1,126 @@
+import { describe, expect, it } from "vitest";
+
+import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
+
+type FakeFinish = { dictationId: string; finalSeq: number };
+type FakeStart = { dictationId: string; format: string };
+type FakeChunk = { dictationId: string; seq: number; audio: string; format: string };
+
+class FakeDaemonClient {
+ isConnected = true;
+ starts: FakeStart[] = [];
+ chunks: FakeChunk[] = [];
+ finishes: FakeFinish[] = [];
+ cancels: string[] = [];
+
+ async startDictationStream(dictationId: string, format: string): Promise {
+ this.starts.push({ dictationId, format });
+ }
+
+ sendDictationStreamChunk(dictationId: string, seq: number, audio: string, format: string): void {
+ this.chunks.push({ dictationId, seq, audio, format });
+ }
+
+ async finishDictationStream(dictationId: string, finalSeq: number): Promise<{ dictationId: string; text: string }> {
+ this.finishes.push({ dictationId, finalSeq });
+ return { dictationId, text: "ok" };
+ }
+
+ cancelDictationStream(dictationId: string): void {
+ this.cancels.push(dictationId);
+ }
+}
+
+const tick = async () => {
+ await Promise.resolve();
+ await Promise.resolve();
+};
+
+describe("DictationStreamSender", () => {
+ it("enqueues segments and sends them after stream start", async () => {
+ const client = new FakeDaemonClient();
+ const ids = ["d1"];
+ const sender = new DictationStreamSender({
+ client: client as any,
+ format: "audio/pcm;rate=16000;bits=16",
+ createDictationId: () => ids.shift() ?? "dX",
+ });
+
+ sender.enqueueSegment("seg0");
+ sender.enqueueSegment("seg1");
+
+ await tick();
+
+ expect(client.starts).toEqual([{ dictationId: "d1", format: "audio/pcm;rate=16000;bits=16" }]);
+ expect(client.chunks).toEqual([
+ { dictationId: "d1", seq: 0, audio: "seg0", format: "audio/pcm;rate=16000;bits=16" },
+ { dictationId: "d1", seq: 1, audio: "seg1", format: "audio/pcm;rate=16000;bits=16" },
+ ]);
+ });
+
+ it("restarts stream and resends from seq=0 on reconnect", async () => {
+ const client = new FakeDaemonClient();
+ const ids = ["d1", "d2"];
+ const sender = new DictationStreamSender({
+ client: client as any,
+ format: "audio/pcm;rate=16000;bits=16",
+ createDictationId: () => ids.shift() ?? "dX",
+ });
+
+ sender.enqueueSegment("seg0");
+ sender.enqueueSegment("seg1");
+ await tick();
+
+ await sender.restartStream("reconnect");
+
+ expect(client.starts.map((s) => s.dictationId)).toEqual(["d1", "d2"]);
+ const d2Chunks = client.chunks.filter((c) => c.dictationId === "d2");
+ expect(d2Chunks.map((c) => [c.seq, c.audio])).toEqual([
+ [0, "seg0"],
+ [1, "seg1"],
+ ]);
+ });
+
+ it("finish flushes all queued segments and sends finish with finalSeq", async () => {
+ const client = new FakeDaemonClient();
+ const ids = ["d1"];
+ const sender = new DictationStreamSender({
+ client: client as any,
+ format: "audio/pcm;rate=16000;bits=16",
+ createDictationId: () => ids.shift() ?? "dX",
+ });
+
+ sender.enqueueSegment("seg0");
+ sender.enqueueSegment("seg1");
+
+ const finalSeq = sender.getFinalSeq();
+ const result = await sender.finish(finalSeq);
+
+ expect(result.text).toBe("ok");
+ expect(client.chunks.map((c) => c.seq)).toEqual([0, 1]);
+ expect(client.finishes).toEqual([{ dictationId: "d1", finalSeq: 1 }]);
+ });
+
+ it("keeps segments while disconnected and sends them after restart when reconnected", async () => {
+ const client = new FakeDaemonClient();
+ client.isConnected = false;
+ const ids = ["d1"];
+ const sender = new DictationStreamSender({
+ client: client as any,
+ format: "audio/pcm;rate=16000;bits=16",
+ createDictationId: () => ids.shift() ?? "dX",
+ });
+
+ sender.enqueueSegment("seg0");
+ sender.enqueueSegment("seg1");
+
+ expect(client.starts).toHaveLength(0);
+ expect(client.chunks).toHaveLength(0);
+
+ client.isConnected = true;
+ await sender.restartStream("reconnect");
+
+ expect(client.chunks.map((c) => c.seq)).toEqual([0, 1]);
+ });
+});
+
diff --git a/packages/app/src/dictation/dictation-stream-sender.ts b/packages/app/src/dictation/dictation-stream-sender.ts
new file mode 100644
index 000000000..6819bd6a4
--- /dev/null
+++ b/packages/app/src/dictation/dictation-stream-sender.ts
@@ -0,0 +1,189 @@
+import { generateMessageId } from "@/types/stream";
+import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
+
+export type DictationStreamSenderParams = {
+ client: DaemonClientV2 | null;
+ format: string;
+ createDictationId?: () => string;
+};
+
+type DictationFinishResult = { dictationId: string; text: string };
+
+/**
+ * Small, non-React state machine for dictation streaming.
+ *
+ * Responsibilities:
+ * - Maintain an ordered buffer of base64 PCM segments
+ * - Start/restart a dictation stream (dictationId)
+ * - Send missing segments (seq) when connected
+ * - Finish/cancel the stream
+ *
+ * This class intentionally keeps sending synchronous (no internal async mutex),
+ * so enqueues can't "miss" a flush due to in-flight await/coalescing bugs.
+ */
+export class DictationStreamSender {
+ private client: DaemonClientV2 | null;
+ private readonly format: string;
+ private readonly createDictationId: () => string;
+
+ private dictationId: string | null = null;
+ private sendSeq = 0;
+ private segments: string[] = [];
+ private streamReady = false;
+
+ private startGeneration = 0;
+ private startPromise: Promise | null = null;
+
+ constructor(params: DictationStreamSenderParams) {
+ this.client = params.client;
+ this.format = params.format;
+ this.createDictationId = params.createDictationId ?? generateMessageId;
+ }
+
+ setClient(client: DaemonClientV2 | null): void {
+ this.client = client;
+ }
+
+ getDictationId(): string | null {
+ return this.dictationId;
+ }
+
+ getSegmentCount(): number {
+ return this.segments.length;
+ }
+
+ getFinalSeq(): number {
+ return this.segments.length - 1;
+ }
+
+ hasSegments(): boolean {
+ return this.segments.length > 0;
+ }
+
+ clearAll(): void {
+ this.dictationId = null;
+ this.sendSeq = 0;
+ this.segments = [];
+ this.streamReady = false;
+ this.startPromise = null;
+ this.startGeneration += 1;
+ }
+
+ resetStreamForReplay(): void {
+ this.dictationId = null;
+ this.sendSeq = 0;
+ this.streamReady = false;
+ this.startPromise = null;
+ this.startGeneration += 1;
+ }
+
+ enqueueSegment(base64Pcm: string): void {
+ this.segments.push(base64Pcm);
+
+ const client = this.client;
+ if (!client?.isConnected) {
+ return;
+ }
+
+ if (!this.dictationId) {
+ void this.restartStream("enqueue");
+ return;
+ }
+
+ this.flush();
+ }
+
+ flush(): number {
+ const client = this.client;
+ const dictationId = this.dictationId;
+ if (!client?.isConnected || !dictationId || !this.streamReady) {
+ return 0;
+ }
+
+ let sent = 0;
+ while (this.sendSeq < this.segments.length) {
+ const seq = this.sendSeq;
+ const audio = this.segments[seq]!;
+ client.sendDictationStreamChunk(dictationId, seq, audio, this.format);
+ this.sendSeq = seq + 1;
+ sent += 1;
+ }
+ return sent;
+ }
+
+ async restartStream(reason: string): Promise {
+ const client = this.client;
+ if (!client?.isConnected) {
+ return;
+ }
+
+ this.startGeneration += 1;
+ const generation = this.startGeneration;
+
+ const dictationId = this.createDictationId();
+ this.dictationId = dictationId;
+ this.sendSeq = 0;
+ this.streamReady = false;
+
+ const start = (async () => {
+ await client.startDictationStream(dictationId, this.format);
+ if (this.startGeneration !== generation) {
+ return;
+ }
+ if (this.dictationId !== dictationId) {
+ return;
+ }
+ this.streamReady = true;
+ this.flush();
+ })().catch((error) => {
+ // If starting failed, keep the segments for retry but clear the stream so finish can error cleanly.
+ if (this.startGeneration === generation && this.dictationId === dictationId) {
+ this.dictationId = null;
+ this.streamReady = false;
+ }
+ throw error;
+ }).finally(() => {
+ if (this.startPromise === start) {
+ this.startPromise = null;
+ }
+ });
+
+ this.startPromise = start;
+ await start;
+ void reason;
+ }
+
+ async finish(finalSeq: number): Promise {
+ const client = this.client;
+ if (!client) {
+ throw new Error("Daemon client unavailable");
+ }
+ if (!client.isConnected) {
+ throw new Error("Daemon client is disconnected");
+ }
+
+ if (!this.dictationId) {
+ await this.restartStream("finalize");
+ }
+ if (this.startPromise) {
+ await this.startPromise;
+ }
+
+ const dictationId = this.dictationId;
+ if (!dictationId || !this.streamReady) {
+ throw new Error("Failed to start dictation stream");
+ }
+
+ this.flush();
+ return client.finishDictationStream(dictationId, finalSeq);
+ }
+
+ cancel(): void {
+ const client = this.client;
+ const dictationId = this.dictationId;
+ if (client?.isConnected && dictationId) {
+ client.cancelDictationStream(dictationId);
+ }
+ this.resetStreamForReplay();
+ }
+}
diff --git a/packages/app/src/hooks/use-agent-form-state.ts b/packages/app/src/hooks/use-agent-form-state.ts
index 6edc9b662..a18f37b15 100644
--- a/packages/app/src/hooks/use-agent-form-state.ts
+++ b/packages/app/src/hooks/use-agent-form-state.ts
@@ -251,6 +251,10 @@ export function useAgentFormState(
model: "",
workingDir: "",
}));
+ const formStateRef = useRef(formState);
+ useEffect(() => {
+ formStateRef.current = formState;
+ }, [formState]);
// Track if we've done initial resolution (to avoid flickering)
const hasResolvedRef = useRef(false);
@@ -299,16 +303,16 @@ export function useAgentFormState(
preferences,
availableModels,
userModified,
- formState
+ formStateRef.current
);
// Only update if something changed
if (
- resolved.serverId !== formState.serverId ||
- resolved.provider !== formState.provider ||
- resolved.modeId !== formState.modeId ||
- resolved.model !== formState.model ||
- resolved.workingDir !== formState.workingDir
+ resolved.serverId !== formStateRef.current.serverId ||
+ resolved.provider !== formStateRef.current.provider ||
+ resolved.modeId !== formStateRef.current.modeId ||
+ resolved.model !== formStateRef.current.model ||
+ resolved.workingDir !== formStateRef.current.workingDir
) {
setFormState(resolved);
}
@@ -322,7 +326,6 @@ export function useAgentFormState(
preferences,
availableModels,
userModified,
- formState,
]);
// Persist inferred serverId so reloads keep the selection (e.g. URL serverId or first-time load).
diff --git a/packages/app/src/hooks/use-dictation-audio-source.d.ts b/packages/app/src/hooks/use-dictation-audio-source.d.ts
new file mode 100644
index 000000000..45d7043ea
--- /dev/null
+++ b/packages/app/src/hooks/use-dictation-audio-source.d.ts
@@ -0,0 +1,2 @@
+export * from "./use-dictation-audio-source.native";
+
diff --git a/packages/app/src/hooks/use-dictation-audio-source.native.ts b/packages/app/src/hooks/use-dictation-audio-source.native.ts
new file mode 100644
index 000000000..5d3aff7e1
--- /dev/null
+++ b/packages/app/src/hooks/use-dictation-audio-source.native.ts
@@ -0,0 +1,44 @@
+import { useCallback, useEffect, useRef } from "react";
+
+import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
+
+import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
+
+export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
+ const onPcmSegmentRef = useRef(config.onPcmSegment);
+ const onErrorRef = useRef(config.onError);
+
+ useEffect(() => {
+ onPcmSegmentRef.current = config.onPcmSegment;
+ onErrorRef.current = config.onError;
+ }, [config.onPcmSegment, config.onError]);
+
+ const speechmatics = useSpeechmaticsAudio({
+ enableContinuousStreaming: true,
+ onAudioSegment: ({ audioData }) => {
+ onPcmSegmentRef.current(audioData);
+ },
+ onError: (err) => {
+ onErrorRef.current?.(err);
+ },
+ volumeThreshold: 0.3,
+ silenceDuration: 2000,
+ speechConfirmationDuration: 300,
+ detectionGracePeriod: 200,
+ });
+
+ const start = useCallback(async () => {
+ await speechmatics.start();
+ }, [speechmatics]);
+
+ const stop = useCallback(async () => {
+ await speechmatics.stop();
+ }, [speechmatics]);
+
+ return {
+ start,
+ stop,
+ volume: speechmatics.volume,
+ };
+}
+
diff --git a/packages/app/src/hooks/use-dictation-audio-source.types.ts b/packages/app/src/hooks/use-dictation-audio-source.types.ts
new file mode 100644
index 000000000..fd34966b6
--- /dev/null
+++ b/packages/app/src/hooks/use-dictation-audio-source.types.ts
@@ -0,0 +1,11 @@
+export type DictationAudioSourceConfig = {
+ onPcmSegment: (pcm16Base64: string) => void;
+ onError?: (error: Error) => void;
+};
+
+export type DictationAudioSource = {
+ start: () => Promise;
+ stop: () => Promise;
+ volume: number;
+};
+
diff --git a/packages/app/src/hooks/use-dictation-audio-source.web.ts b/packages/app/src/hooks/use-dictation-audio-source.web.ts
new file mode 100644
index 000000000..cd6cb9ff4
--- /dev/null
+++ b/packages/app/src/hooks/use-dictation-audio-source.web.ts
@@ -0,0 +1,247 @@
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+
+import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
+
+const getAudioContextCtor = (): (typeof AudioContext) | null => {
+ if (typeof window === "undefined") {
+ return null;
+ }
+ const ctor =
+ (window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
+ (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
+ return ctor ?? null;
+};
+
+const floatToInt16 = (sample: number): number => {
+ const clamped = Math.max(-1, Math.min(1, sample));
+ return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
+};
+
+const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
+ if (input.length === 0) {
+ return new Int16Array(0);
+ }
+ if (inputRate === outputRate) {
+ const out = new Int16Array(input.length);
+ for (let i = 0; i < input.length; i++) {
+ out[i] = floatToInt16(input[i]);
+ }
+ return out;
+ }
+
+ const ratio = inputRate / outputRate;
+ const outputLength = Math.max(1, Math.round(input.length / ratio));
+ const out = new Int16Array(outputLength);
+ for (let i = 0; i < outputLength; i++) {
+ const sourceIndex = i * ratio;
+ const i0 = Math.floor(sourceIndex);
+ const i1 = Math.min(input.length - 1, i0 + 1);
+ const frac = sourceIndex - i0;
+ const sample = input[i0] * (1 - frac) + input[i1] * frac;
+ out[i] = floatToInt16(sample);
+ }
+ return out;
+};
+
+const concatInt16 = (a: Int16Array, b: Int16Array): Int16Array => {
+ if (a.length === 0) {
+ return b;
+ }
+ if (b.length === 0) {
+ return a;
+ }
+ const out = new Int16Array(a.length + b.length);
+ out.set(a, 0);
+ out.set(b, a.length);
+ return out;
+};
+
+const int16ToBase64 = (pcm: Int16Array): string => {
+ const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+ let binary = "";
+ for (let i = 0; i < bytes.length; i++) {
+ binary += String.fromCharCode(bytes[i]);
+ }
+ return btoa(binary);
+};
+
+export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
+ const [volume, setVolume] = useState(0);
+
+ const onPcmSegmentRef = useRef(config.onPcmSegment);
+ const onErrorRef = useRef(config.onError);
+
+ useEffect(() => {
+ onPcmSegmentRef.current = config.onPcmSegment;
+ onErrorRef.current = config.onError;
+ }, [config.onPcmSegment, config.onError]);
+
+ const refs = useRef<{
+ stream: MediaStream | null;
+ context: AudioContext | null;
+ source: MediaStreamAudioSourceNode | null;
+ processor: ScriptProcessorNode | null;
+ gain: GainNode | null;
+ pending: Int16Array;
+ started: boolean;
+ }>({ stream: null, context: null, source: null, processor: null, gain: null, pending: new Int16Array(0), started: false });
+
+ const start = useCallback(async () => {
+ const missingNavigator =
+ typeof navigator === "undefined" ||
+ !navigator.mediaDevices ||
+ typeof navigator.mediaDevices.getUserMedia !== "function";
+
+ const secureContext =
+ typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
+ ? window.isSecureContext
+ : true;
+ const currentOrigin =
+ typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
+
+ if (missingNavigator) {
+ throw new Error("Microphone capture is not supported in this environment");
+ }
+ if (!secureContext) {
+ throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
+ }
+
+ const AudioContextCtor = getAudioContextCtor();
+ if (!AudioContextCtor) {
+ throw new Error("AudioContext unavailable");
+ }
+
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: {
+ channelCount: 1,
+ noiseSuppression: true,
+ echoCancellation: true,
+ autoGainControl: true,
+ },
+ });
+
+ const context = new AudioContextCtor();
+ const source = context.createMediaStreamSource(stream);
+ const processor = context.createScriptProcessor(4096, 1, 1);
+ const gain = context.createGain();
+ gain.gain.value = 0;
+
+ const outputRate = 16000;
+ const chunkSamples = outputRate; // ~1s
+
+ refs.current.started = true;
+
+ processor.onaudioprocess = (event) => {
+ if (!refs.current.started) {
+ return;
+ }
+ const input = event.inputBuffer.getChannelData(0);
+
+ let sumSquares = 0;
+ for (let i = 0; i < input.length; i++) {
+ const sample = input[i];
+ sumSquares += sample * sample;
+ }
+ const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
+ const normalized = Math.min(1, Math.max(0, rms * 2));
+ setVolume(normalized);
+
+ const next = resampleToPcm16(input, context.sampleRate, outputRate);
+ refs.current.pending = concatInt16(refs.current.pending, next);
+
+ while (refs.current.pending.length >= chunkSamples) {
+ const chunk = refs.current.pending.slice(0, chunkSamples);
+ refs.current.pending = refs.current.pending.slice(chunkSamples);
+ onPcmSegmentRef.current(int16ToBase64(chunk));
+ }
+ };
+
+ source.connect(processor);
+ processor.connect(gain);
+ gain.connect(context.destination);
+
+ refs.current = { stream, context, source, processor, gain, pending: new Int16Array(0), started: true };
+ }, []);
+
+ const stop = useCallback(async () => {
+ refs.current.started = false;
+ setVolume(0);
+
+ const { processor, source, gain, context, stream, pending } = refs.current;
+
+ if (processor) {
+ try {
+ processor.onaudioprocess = null;
+ } catch {
+ // no-op
+ }
+ try {
+ processor.disconnect();
+ } catch {
+ // no-op
+ }
+ }
+ if (source) {
+ try {
+ source.disconnect();
+ } catch {
+ // no-op
+ }
+ }
+ if (gain) {
+ try {
+ gain.disconnect();
+ } catch {
+ // no-op
+ }
+ }
+ if (stream) {
+ stream.getTracks().forEach((track) => {
+ try {
+ track.stop();
+ } catch {
+ // no-op
+ }
+ });
+ }
+ if (context) {
+ try {
+ await context.close();
+ } catch {
+ // no-op
+ }
+ }
+
+ if (pending.length > 0) {
+ onPcmSegmentRef.current(int16ToBase64(pending));
+ }
+
+ refs.current = { stream: null, context: null, source: null, processor: null, gain: null, pending: new Int16Array(0), started: false };
+ }, []);
+
+ useEffect(() => {
+ return () => {
+ void stop().catch((err) => {
+ onErrorRef.current?.(err instanceof Error ? err : new Error(String(err)));
+ });
+ };
+ }, [stop]);
+
+ return useMemo(
+ () => ({
+ start: async () => {
+ try {
+ await start();
+ } catch (err) {
+ const normalized = err instanceof Error ? err : new Error(String(err));
+ onErrorRef.current?.(normalized);
+ throw normalized;
+ }
+ },
+ stop,
+ volume,
+ }),
+ [start, stop, volume]
+ );
+}
+
diff --git a/packages/app/src/hooks/use-dictation.d.ts b/packages/app/src/hooks/use-dictation.d.ts
new file mode 100644
index 000000000..83efec689
--- /dev/null
+++ b/packages/app/src/hooks/use-dictation.d.ts
@@ -0,0 +1 @@
+export * from "./use-dictation";
diff --git a/packages/app/src/hooks/use-dictation.shared.ts b/packages/app/src/hooks/use-dictation.shared.ts
new file mode 100644
index 000000000..68540a809
--- /dev/null
+++ b/packages/app/src/hooks/use-dictation.shared.ts
@@ -0,0 +1,40 @@
+export type DictationStatus = "idle" | "recording" | "uploading" | "failed";
+
+export type UseDictationOptions = {
+ client: import("@server/client/daemon-client-v2").DaemonClientV2 | null;
+ onTranscript: (text: string, meta: { requestId: string }) => void;
+ onError?: (error: Error) => void;
+ onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
+ canStart?: () => boolean;
+ canConfirm?: () => boolean;
+ autoStopWhenHidden?: { isVisible: boolean };
+ enableDuration?: boolean;
+};
+
+export type UseDictationResult = {
+ isRecording: boolean;
+ isProcessing: boolean;
+ volume: number;
+ duration: number;
+ error: string | null;
+ status: DictationStatus;
+ startDictation: () => Promise;
+ cancelDictation: () => Promise;
+ confirmDictation: () => Promise;
+ retryFailedDictation: () => Promise;
+ discardFailedDictation: () => void;
+ reset: () => void;
+};
+
+export const DURATION_TICK_MS = 1000;
+export const PCM_DICTATION_FORMAT = "audio/pcm;rate=16000;bits=16";
+
+export const toError = (error: unknown): Error => {
+ if (error instanceof Error) {
+ return error;
+ }
+ if (typeof error === "string" && error.trim().length > 0) {
+ return new Error(error);
+ }
+ return new Error("An unexpected error occurred while handling dictation.");
+};
diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts
index e4d59c906..3a13bafdd 100644
--- a/packages/app/src/hooks/use-dictation.ts
+++ b/packages/app/src/hooks/use-dictation.ts
@@ -1,160 +1,23 @@
import { useCallback, useEffect, useRef, useState } from "react";
-import { useMutation } from "@tanstack/react-query";
-import { useAudioRecorder } from "@/hooks/use-audio-recorder";
-import type { DaemonClientV2 } from "@server/client/daemon-client-v2";
-import type { TranscriptionResultMessage } from "@server/shared/messages";
+import { DictationStreamSender } from "@/dictation/dictation-stream-sender";
+import { useDictationAudioSource } from "@/hooks/use-dictation-audio-source";
import { generateMessageId } from "@/types/stream";
import { AttemptGuard } from "@/utils/attempt-guard";
-
-export type DictationStatus = "idle" | "recording" | "uploading" | "retrying" | "failed";
-
-type DictationRetryReason = "dispatch" | "timeout" | "response" | "disconnected";
-
-export type DictationRetryInfo = {
- attempt: number;
- maxAttempts: number;
- reason: DictationRetryReason;
- errorMessage: string;
- nextRetryMs: number;
-};
-
-export type FailedDictationRecording = {
- requestId: string;
- durationSeconds: number;
- sizeBytes: number;
- format: string;
- recordedAt: number;
- errorMessage: string;
-};
-
-export type DictationOutcome =
- | { type: "success"; requestId: string; timestamp: number }
- | { type: "failure"; requestId: string; errorMessage: string; timestamp: number };
-
-export type UseDictationOptions = {
- client: DaemonClientV2 | null;
- onTranscript: (text: string, meta: { requestId: string }) => void;
- onError?: (error: Error) => void;
- onRetryAttempt?: (info: DictationRetryInfo) => void;
- onPermanentFailure?: (error: Error, context: { requestId: string }) => void;
- canStart?: () => boolean;
- canConfirm?: () => boolean;
- autoStopWhenHidden?: { isVisible: boolean };
- enableDuration?: boolean;
-};
-
-export type UseDictationResult = {
- isRecording: boolean;
- isProcessing: boolean;
- volume: number;
- duration: number;
- pendingRequestId: string | null;
- error: string | null;
- status: DictationStatus;
- retryAttempt: number;
- maxRetryAttempts: number;
- retryInfo: DictationRetryInfo | null;
- failedRecording: FailedDictationRecording | null;
- lastOutcome: DictationOutcome | null;
- startDictation: () => Promise;
- cancelDictation: () => Promise;
- confirmDictation: () => Promise;
- retryFailedDictation: () => Promise;
- discardFailedDictation: () => void;
- reset: () => void;
-};
-
-const DURATION_TICK_MS = 1000;
-const MAX_AUTO_RETRY_ATTEMPTS = 5;
-const RETRY_BASE_DELAY_MS = 2000;
-const RETRY_MAX_DELAY_MS = 12000;
-const RETRY_BACKOFF_FACTOR = 1.8;
-const RETRY_JITTER_MS = 400;
-const TRANSCRIPTION_TIMEOUT_MS = 120000;
-
-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-
-const computeRetryDelayMs = (attempt: number): number => {
- const exponential = RETRY_BASE_DELAY_MS * RETRY_BACKOFF_FACTOR ** Math.max(0, attempt - 1);
- const capped = Math.min(RETRY_MAX_DELAY_MS, exponential);
- if (RETRY_JITTER_MS <= 0) {
- return capped;
- }
- return capped + Math.floor(Math.random() * RETRY_JITTER_MS);
-};
-
-const isTimeoutError = (error: Error): boolean =>
- /timed out|timeout/i.test(error.message);
-
-class DictationAttemptError extends Error {
- public readonly reason: DictationRetryReason;
-
- constructor(reason: DictationRetryReason, error: Error) {
- super(error.message);
- this.name = "DictationAttemptError";
- this.reason = reason;
- (this as Error & { cause?: Error }).cause = error;
- }
-}
-
-const toError = (error: unknown): Error => {
- if (error instanceof Error) {
- return error;
- }
- if (typeof error === "string" && error.trim().length > 0) {
- return new Error(error);
- }
- return new Error("An unexpected error occurred while handling dictation.");
-};
-
-type CapturedAudioPayload = {
- blob: Blob;
- format: string;
- sizeBytes: number;
- durationSeconds: number;
- recordedAt: number;
-};
-
-type TranscriptionPayload = TranscriptionResultMessage["payload"];
-
-const blobToBase64 = async (blob: Blob): Promise => {
- const arrayBuffer = await blob.arrayBuffer();
- const bytes = new Uint8Array(arrayBuffer);
- let binary = "";
- for (let i = 0; i < bytes.length; i++) {
- binary += String.fromCharCode(bytes[i]);
- }
- return btoa(binary);
-};
-
-const deriveFormatFromMime = (mimeType?: string): string => {
- if (!mimeType || mimeType.length === 0) {
- return "webm";
- }
- const slashIndex = mimeType.indexOf("/");
- let formatPart = slashIndex >= 0 ? mimeType.slice(slashIndex + 1) : mimeType;
- const semicolonIndex = formatPart.indexOf(";");
- if (semicolonIndex >= 0) {
- formatPart = formatPart.slice(0, semicolonIndex);
- }
- return formatPart.trim().length > 0 ? formatPart.trim() : "webm";
-};
-
-const buildCapturedAudioPayload = (blob: Blob, durationSeconds: number): CapturedAudioPayload => ({
- blob,
- format: deriveFormatFromMime(blob.type),
- sizeBytes: typeof blob.size === "number" ? blob.size : 0,
- durationSeconds,
- recordedAt: Date.now(),
-});
+import {
+ DURATION_TICK_MS,
+ PCM_DICTATION_FORMAT,
+ toError,
+ type DictationStatus,
+ type UseDictationOptions,
+ type UseDictationResult,
+} from "./use-dictation.shared";
export function useDictation(options: UseDictationOptions): UseDictationResult {
const {
client,
onTranscript,
onError,
- onRetryAttempt,
onPermanentFailure,
canStart,
canConfirm,
@@ -164,127 +27,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const [isRecording, setIsRecording] = useState(false);
const [isProcessing, setIsProcessing] = useState(false);
- const [volume, setVolume] = useState(0);
const [duration, setDuration] = useState(0);
- const [pendingRequestId, setPendingRequestId] = useState(null);
const [error, setError] = useState(null);
const [status, setStatus] = useState("idle");
- const [retryAttempt, setRetryAttempt] = useState(0);
- const [retryInfo, setRetryInfo] = useState(null);
- const [failedRecording, setFailedRecording] = useState(null);
- const [lastOutcome, setLastOutcome] = useState(null);
- const maxRetryAttempts = MAX_AUTO_RETRY_ATTEMPTS;
-
- const transcriptionMutation = useMutation({
- mutationFn: async (): Promise => {
- const capturedAudio = pendingAudioRef.current;
- if (!capturedAudio) {
- throw new Error("No recorded audio available for transcription");
- }
-
- setRetryAttempt(1);
- setRetryInfo(null);
-
- let attempt = 1;
-
- while (attempt <= MAX_AUTO_RETRY_ATTEMPTS) {
- try {
- if (!client) {
- throw new DictationAttemptError(
- "disconnected",
- new Error("Daemon client unavailable")
- );
- }
- if (!client.isConnected) {
- throw new DictationAttemptError(
- "disconnected",
- new Error("Daemon client is disconnected")
- );
- }
-
- console.info("[useDictation] sending transcription request", {
- attempt,
- size: capturedAudio.sizeBytes,
- durationSeconds: capturedAudio.durationSeconds,
- });
-
- setStatus("uploading");
- setRetryAttempt(attempt);
-
- try {
- const base64Audio = await blobToBase64(capturedAudio.blob);
- return await client.transcribeAudio({
- audio: base64Audio,
- format: capturedAudio.format,
- timeout: TRANSCRIPTION_TIMEOUT_MS,
- });
- } catch (error) {
- const normalized = toError(error);
- const reason: DictationRetryReason = isTimeoutError(normalized)
- ? "timeout"
- : "response";
- throw new DictationAttemptError(reason, normalized);
- }
- } catch (error) {
- const attemptError =
- error instanceof DictationAttemptError
- ? error
- : new DictationAttemptError("response", toError(error));
-
- if (attempt >= MAX_AUTO_RETRY_ATTEMPTS) {
- throw attemptError;
- }
-
- const nextAttempt = attempt + 1;
- const delayMs = computeRetryDelayMs(nextAttempt);
- const info: DictationRetryInfo = {
- attempt: nextAttempt,
- maxAttempts: MAX_AUTO_RETRY_ATTEMPTS,
- reason: attemptError.reason,
- errorMessage: attemptError.message,
- nextRetryMs: delayMs,
- };
-
- setStatus("retrying");
- setRetryAttempt(nextAttempt);
- setRetryInfo(info);
- onRetryAttemptRef.current?.(info);
- console.warn("[useDictation] retry scheduled", {
- attempt: nextAttempt,
- maxAttempts: MAX_AUTO_RETRY_ATTEMPTS,
- reason: attemptError.reason,
- error: attemptError.message,
- nextRetryMs: delayMs,
- });
-
- await sleep(delayMs);
- attempt = nextAttempt;
- }
- }
-
- throw new Error("Failed to complete transcription");
- },
- });
- const {
- mutateAsync: runTranscription,
- reset: resetTranscriptionMutation,
- } = transcriptionMutation;
-
- const pendingAudioRef = useRef(null);
- const handleAudioLevel = useCallback((level: number) => {
- setVolume(level);
- }, []);
-
- const durationRef = useRef(0);
- useEffect(() => {
- durationRef.current = duration;
- }, [duration]);
-
- const recorder = useAudioRecorder({ onAudioLevel: handleAudioLevel });
- const recorderRef = useRef(recorder);
- useEffect(() => {
- recorderRef.current = recorder;
- }, [recorder]);
const onTranscriptRef = useRef(onTranscript);
useEffect(() => {
@@ -296,11 +41,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
onErrorRef.current = onError;
}, [onError]);
- const onRetryAttemptRef = useRef(onRetryAttempt);
- useEffect(() => {
- onRetryAttemptRef.current = onRetryAttempt;
- }, [onRetryAttempt]);
-
const onPermanentFailureRef = useRef(onPermanentFailure);
useEffect(() => {
onPermanentFailureRef.current = onPermanentFailure;
@@ -316,10 +56,27 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
isProcessingRef.current = isProcessing;
}, [isProcessing]);
- const pendingRequestIdRef = useRef(null);
- const activeStopPromiseRef = useRef | null>(null);
+ // duration is used for UI only; no need to mirror into a ref.
+
const durationIntervalRef = useRef | null>(null);
const attemptGuardRef = useRef(new AttemptGuard());
+ const actionGateRef = useRef<{ starting: boolean; confirming: boolean; cancelling: boolean }>({
+ starting: false,
+ confirming: false,
+ cancelling: false,
+ });
+
+ const senderRef = useRef(null);
+ if (!senderRef.current) {
+ senderRef.current = new DictationStreamSender({
+ client,
+ format: PCM_DICTATION_FORMAT,
+ createDictationId: generateMessageId,
+ });
+ }
+ useEffect(() => {
+ senderRef.current?.setClient(client);
+ }, [client]);
const stopDurationTracking = useCallback(() => {
if (durationIntervalRef.current) {
@@ -330,7 +87,6 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const startDurationTracking = useCallback(() => {
if (!enableDuration) {
- console.log("[useDictation] startDictation blocked by canStart()");
return;
}
if (durationIntervalRef.current) {
@@ -351,6 +107,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
const reportError = useCallback(
(err: unknown, context?: string) => {
const normalized = toError(err);
+ if (normalized.name === "AttemptCancelledError") {
+ return;
+ }
if (context) {
console.error(`[useDictation] ${context}`, normalized);
} else {
@@ -362,136 +121,114 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
[setError]
);
- const stopRecorder = useCallback(async (): Promise => {
- if (activeStopPromiseRef.current) {
- return activeStopPromiseRef.current;
- }
- const recorderInstance = recorderRef.current;
- if (!recorderInstance) {
- throw new Error("Recorder unavailable");
- }
- const stopPromise = (async () => {
- try {
- return await recorderInstance.stop();
- } finally {
- activeStopPromiseRef.current = null;
- }
- })();
- activeStopPromiseRef.current = stopPromise;
- return stopPromise;
+ const clearStreamingState = useCallback(() => {
+ senderRef.current?.clearAll();
}, []);
- const transmitDictation = useCallback(
- async () => runTranscription(),
- [runTranscription]
+ const startNewStream = useCallback(
+ async (reason: string) => {
+ await senderRef.current?.restartStream(reason);
+ },
+ []
);
- const handleTranscriptionSuccess = useCallback(
- (transcription: TranscriptionPayload) => {
- const requestId = transcription.requestId;
- pendingRequestIdRef.current = null;
- setPendingRequestId(null);
+ const ensureFinalTranscript = useCallback(
+ async (finalSeq: number): Promise => {
+ const result = await senderRef.current!.finish(finalSeq);
+ return result.text;
+ },
+ []
+ );
+
+ useEffect(() => {
+ if (!client) {
+ return;
+ }
+ return client.subscribeConnectionStatus((next) => {
+ if (next.status !== "connected") {
+ return;
+ }
+ if (!isRecordingRef.current) {
+ return;
+ }
+ void startNewStream("reconnect");
+ });
+ }, [client, startNewStream]);
+
+ const audio = useDictationAudioSource({
+ onPcmSegment: (audioData) => {
+ senderRef.current?.enqueueSegment(audioData);
+ },
+ onError: (err) => {
+ onErrorRef.current?.(err);
+ },
+ });
+ const audioStopRef = useRef(audio.stop);
+ useEffect(() => {
+ audioStopRef.current = audio.stop;
+ }, [audio.stop]);
+
+ const handleStreamingTranscriptionSuccess = useCallback(
+ (text: string, requestId: string) => {
setIsProcessing(false);
setDuration(0);
setStatus("idle");
- setRetryAttempt(0);
- setRetryInfo(null);
- setFailedRecording(null);
- pendingAudioRef.current = null;
- setLastOutcome({ type: "success", requestId, timestamp: Date.now() });
- const transcriptText = transcription.text?.trim();
+ clearStreamingState();
+
+ const transcriptText = text.trim();
if (!transcriptText) {
return;
}
-
- console.log("[useDictation] transcription_result received", {
- requestId,
- textLength: transcriptText.length,
- });
onTranscriptRef.current?.(transcriptText, { requestId });
},
- [onTranscriptRef]
+ [clearStreamingState]
);
const handleDictationFailure = useCallback(
(failure: unknown) => {
const normalized = toError(failure);
const failureId = generateMessageId();
- pendingRequestIdRef.current = null;
- setPendingRequestId(null);
setIsProcessing(false);
isRecordingRef.current = false;
setIsRecording(false);
- setVolume(0);
- setRetryInfo(null);
- const capturedAudio = pendingAudioRef.current;
- if (capturedAudio) {
+ if (senderRef.current?.hasSegments()) {
setStatus("failed");
- setFailedRecording({
- requestId: failureId,
- durationSeconds: capturedAudio.durationSeconds,
- sizeBytes: capturedAudio.sizeBytes,
- format: capturedAudio.format,
- recordedAt: capturedAudio.recordedAt,
- errorMessage: normalized.message,
- });
onPermanentFailureRef.current?.(normalized, { requestId: failureId });
} else {
setStatus("idle");
}
- setRetryAttempt(0);
- setLastOutcome({
- type: "failure",
- requestId: failureId,
- errorMessage: normalized.message,
- timestamp: Date.now(),
- });
reportError(normalized, "Failed to complete dictation");
},
- [onPermanentFailureRef, reportError]
+ [reportError]
);
const startDictation = useCallback(async () => {
- console.log("[useDictation] startDictation requested", {
- isRecording: isRecordingRef.current,
- isProcessing,
- });
- if (isRecordingRef.current || isProcessing) {
- console.log("[useDictation] startDictation aborted: already recording/processing", {
- isRecording: isRecordingRef.current,
- isProcessing,
- });
+ if (actionGateRef.current.starting || actionGateRef.current.confirming || actionGateRef.current.cancelling) {
+ return;
+ }
+ if (isRecordingRef.current || isProcessingRef.current) {
return;
}
const startAllowed = canStart ? canStart() : true;
if (!startAllowed) {
- console.log("[useDictation] startDictation blocked by canStart()");
return;
}
+ actionGateRef.current.starting = true;
setError(null);
- setVolume(0);
setDuration(0);
setIsProcessing(false);
setStatus("recording");
- setRetryAttempt(0);
- setRetryInfo(null);
- setFailedRecording(null);
- pendingAudioRef.current = null;
- setLastOutcome(null);
- pendingRequestIdRef.current = null;
- setPendingRequestId(null);
+ clearStreamingState();
try {
- const recorderInstance = recorderRef.current;
- if (!recorderInstance) {
- throw new Error("Recorder unavailable");
+ await audio.start();
+ if (client?.isConnected) {
+ void startNewStream("start");
}
- await recorderInstance.start();
- console.log("[useDictation] recorder.start succeeded");
isRecordingRef.current = true;
setIsRecording(true);
if (enableDuration) {
@@ -502,160 +239,155 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
isRecordingRef.current = false;
setIsRecording(false);
reportError(err, "Failed to start dictation");
+ } finally {
+ actionGateRef.current.starting = false;
}
}, [
+ audio,
canStart,
+ clearStreamingState,
+ client,
enableDuration,
isProcessing,
reportError,
startDurationTracking,
+ startNewStream,
stopDurationTracking,
]);
const cancelDictation = useCallback(async () => {
- console.log("[useDictation] cancelDictation requested", {
- isRecording: isRecordingRef.current,
- hasActiveStop: Boolean(activeStopPromiseRef.current),
- });
attemptGuardRef.current.cancel();
- if (!isRecordingRef.current && !activeStopPromiseRef.current) {
- console.log("[useDictation] cancelDictation ignored: nothing to cancel");
+ if (actionGateRef.current.cancelling) {
return;
}
+ if (!isRecordingRef.current && !isProcessingRef.current) {
+ return;
+ }
+ actionGateRef.current.cancelling = true;
stopDurationTracking();
setDuration(0);
setError(null);
try {
- await stopRecorder();
+ try {
+ senderRef.current?.cancel();
+ } catch {
+ // no-op
+ }
+ await audio.stop();
} catch (err) {
reportError(err, "Failed to cancel dictation");
} finally {
isRecordingRef.current = false;
setIsRecording(false);
setIsProcessing(false);
- setVolume(0);
+ isProcessingRef.current = false;
setStatus("idle");
- setRetryAttempt(0);
- setRetryInfo(null);
- pendingAudioRef.current = null;
- setFailedRecording(null);
- setLastOutcome(null);
+ clearStreamingState();
+ actionGateRef.current.cancelling = false;
}
- }, [reportError, stopDurationTracking, stopRecorder]);
+ }, [audio, clearStreamingState, client, reportError, stopDurationTracking]);
const confirmDictation = useCallback(async () => {
- console.log("[useDictation] confirmDictation requested", {
- isRecording: isRecordingRef.current,
- isProcessing,
- });
- if (!isRecordingRef.current || isProcessing) {
- console.log("[useDictation] confirmDictation ignored: recording flag", {
- isRecording: isRecordingRef.current,
- isProcessing,
- });
+ if (actionGateRef.current.confirming) {
+ return;
+ }
+ if (!isRecordingRef.current || isProcessingRef.current) {
return;
}
const confirmAllowed = canConfirm ? canConfirm() : true;
if (!confirmAllowed) {
- console.log("[useDictation] confirmDictation blocked by canConfirm()");
return;
}
+ actionGateRef.current.confirming = true;
setError(null);
stopDurationTracking();
setIsProcessing(true);
- setRetryInfo(null);
- setRetryAttempt(0);
- setLastOutcome(null);
+ isProcessingRef.current = true;
const attemptId = attemptGuardRef.current.next();
try {
- const audioData = await stopRecorder();
+ await audio.stop();
attemptGuardRef.current.assertCurrent(attemptId);
- const recordedDurationSeconds = durationRef.current;
- pendingAudioRef.current = buildCapturedAudioPayload(audioData, recordedDurationSeconds);
+
setStatus("uploading");
isRecordingRef.current = false;
setIsRecording(false);
- setVolume(0);
- const transcription = await transmitDictation();
+ const finalSeq = senderRef.current?.getFinalSeq() ?? -1;
+ if (finalSeq < 0) {
+ handleStreamingTranscriptionSuccess("", generateMessageId());
+ return;
+ }
+
+ const transcriptText = await ensureFinalTranscript(finalSeq);
attemptGuardRef.current.assertCurrent(attemptId);
- handleTranscriptionSuccess(transcription);
+ handleStreamingTranscriptionSuccess(transcriptText, generateMessageId());
} catch (err) {
- resetTranscriptionMutation();
+ if (err instanceof Error && err.name === "AttemptCancelledError") {
+ return;
+ }
handleDictationFailure(err);
+ } finally {
+ actionGateRef.current.confirming = false;
}
}, [
+ audio,
canConfirm,
isProcessing,
handleDictationFailure,
- handleTranscriptionSuccess,
- resetTranscriptionMutation,
+ handleStreamingTranscriptionSuccess,
stopDurationTracking,
- stopRecorder,
- transmitDictation,
+ ensureFinalTranscript,
]);
const retryFailedDictation = useCallback(async () => {
- if (!pendingAudioRef.current) {
+ if (!senderRef.current?.hasSegments()) {
return;
}
setError(null);
- setRetryInfo(null);
- setRetryAttempt(0);
setStatus("uploading");
setIsProcessing(true);
- setLastOutcome(null);
+ isProcessingRef.current = true;
try {
- const transcription = await transmitDictation();
- handleTranscriptionSuccess(transcription);
+ if (!client?.isConnected) {
+ throw new Error("Daemon client is disconnected");
+ }
+ senderRef.current.resetStreamForReplay();
+ const finalSeq = senderRef.current.getFinalSeq();
+ const text = await ensureFinalTranscript(finalSeq);
+ handleStreamingTranscriptionSuccess(text, generateMessageId());
} catch (err) {
- resetTranscriptionMutation();
+ if (err instanceof Error && err.name === "AttemptCancelledError") {
+ return;
+ }
handleDictationFailure(err);
}
- }, [
- handleDictationFailure,
- handleTranscriptionSuccess,
- resetTranscriptionMutation,
- transmitDictation,
- ]);
+ }, [client, ensureFinalTranscript, handleDictationFailure, handleStreamingTranscriptionSuccess]);
const discardFailedDictation = useCallback(() => {
- pendingAudioRef.current = null;
- pendingRequestIdRef.current = null;
- setPendingRequestId(null);
setIsProcessing(false);
+ isProcessingRef.current = false;
setDuration(0);
- setFailedRecording(null);
setStatus("idle");
- setRetryAttempt(0);
- setRetryInfo(null);
setError(null);
- setLastOutcome(null);
- }, []);
+ clearStreamingState();
+ }, [clearStreamingState]);
const reset = useCallback(() => {
- pendingRequestIdRef.current = null;
- setPendingRequestId(null);
setIsRecording(false);
isRecordingRef.current = false;
setIsProcessing(false);
+ isProcessingRef.current = false;
stopDurationTracking();
setDuration(0);
- setVolume(0);
setError(null);
setStatus("idle");
- setRetryAttempt(0);
- setRetryInfo(null);
- setFailedRecording(null);
- pendingAudioRef.current = null;
- setLastOutcome(null);
- resetTranscriptionMutation();
- }, [resetTranscriptionMutation, stopDurationTracking]);
+ clearStreamingState();
+ }, [clearStreamingState, stopDurationTracking]);
const cancelRef = useRef<(() => void) | null>(null);
useEffect(() => {
@@ -668,8 +400,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null
);
useEffect(() => {
- const nextVisible =
- typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null;
+ const nextVisible = typeof autoStopWhenHidden?.isVisible === "boolean" ? autoStopWhenHidden.isVisible : null;
const prevVisible = visibilityRef.current;
visibilityRef.current = nextVisible;
@@ -685,48 +416,28 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
stopDurationTracking();
setDuration(0);
setIsProcessing(false);
- setVolume(0);
setError(null);
setStatus("idle");
- setRetryAttempt(0);
- setRetryInfo(null);
- pendingAudioRef.current = null;
- setFailedRecording(null);
- setLastOutcome(null);
+ clearStreamingState();
}
}
- }, [autoStopWhenHidden?.isVisible, stopDurationTracking]);
-
+ }, [autoStopWhenHidden?.isVisible, clearStreamingState, stopDurationTracking]);
useEffect(() => {
return () => {
attemptGuardRef.current.cancel();
stopDurationTracking();
- const activeStop = activeStopPromiseRef.current;
- if (activeStop) {
- void activeStop.catch(() => undefined);
- return;
- }
- const recorderInstance = recorderRef.current;
- if (recorderInstance?.isRecording?.()) {
- void recorderInstance.stop().catch(() => undefined);
- }
+ void audioStopRef.current().catch(() => undefined);
};
}, [stopDurationTracking]);
return {
isRecording,
isProcessing,
- volume,
+ volume: audio.volume,
duration,
- pendingRequestId,
error,
status,
- retryAttempt,
- maxRetryAttempts,
- retryInfo,
- failedRecording,
- lastOutcome,
startDictation,
cancelDictation,
confirmDictation,
@@ -735,3 +446,9 @@ export function useDictation(options: UseDictationOptions): UseDictationResult {
reset,
};
}
+
+export type {
+ DictationStatus,
+ UseDictationOptions,
+ UseDictationResult,
+} from "./use-dictation.shared";
diff --git a/packages/app/src/hooks/use-speechmatics-audio.native.ts b/packages/app/src/hooks/use-speechmatics-audio.native.ts
index 57afe3589..6bdc9fad6 100644
--- a/packages/app/src/hooks/use-speechmatics-audio.native.ts
+++ b/packages/app/src/hooks/use-speechmatics-audio.native.ts
@@ -1,4 +1,5 @@
import { useState, useEffect, useRef, useCallback } from "react";
+import { Buffer } from "buffer";
import {
initialize,
useMicrophonePermissions,
@@ -14,6 +15,8 @@ export interface SpeechmaticsAudioConfig {
onSpeechStart?: () => void;
onSpeechEnd?: () => void;
onError?: (error: Error) => void;
+ /** When true, stream microphone PCM continuously without VAD gating. */
+ enableContinuousStreaming?: boolean;
volumeThreshold: number; // Volume threshold for speech detection (0-1)
silenceDuration: number; // ms of silence before ending segment
speechConfirmationDuration: number; // ms of sustained speech before confirming
@@ -33,11 +36,17 @@ export interface SpeechmaticsAudio {
}
function uint8ArrayToBase64(bytes: Uint8Array): string {
- let binary = "";
- for (let i = 0; i < bytes.length; i++) {
- binary += String.fromCharCode(bytes[i]);
+ // NOTE: This is performance-sensitive during continuous streaming.
+ // Buffer-backed base64 is significantly faster than manual string building.
+ try {
+ return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
+ } catch {
+ let binary = "";
+ for (let i = 0; i < bytes.length; i++) {
+ binary += String.fromCharCode(bytes[i]);
+ }
+ return btoa(binary);
}
- return btoa(binary);
}
function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
@@ -67,6 +76,8 @@ export function useSpeechmaticsAudio(
const [isMuted, setIsMuted] = useState(false);
const [segmentDuration, setSegmentDuration] = useState(0);
+ const enableContinuousStreaming = config.enableContinuousStreaming === true;
+
const audioBufferRef = useRef([]);
const silenceStartRef = useRef(null);
const isSpeakingRef = useRef(false);
@@ -136,6 +147,15 @@ export function useSpeechmaticsAudio(
const pcmData: Uint8Array = event.data;
+ if (enableContinuousStreaming) {
+ audioBufferRef.current.push(pcmData);
+ bufferedBytesRef.current += pcmData.length;
+ if (bufferedBytesRef.current >= MIN_CHUNK_BYTES) {
+ flushBufferedAudio(false);
+ }
+ return;
+ }
+
// Buffer the audio chunk if we're detecting or speaking
// Start buffering from first spike to capture beginning of speech
if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
@@ -150,7 +170,7 @@ export function useSpeechmaticsAudio(
}
}
},
- [isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
+ [enableContinuousStreaming, isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
)
);
@@ -165,6 +185,7 @@ export function useSpeechmaticsAudio(
setVolume(volumeLevel);
if (isMuted) return;
+ if (enableContinuousStreaming) return;
const speechDetected = volumeLevel > VOLUME_THRESHOLD;
@@ -270,6 +291,7 @@ export function useSpeechmaticsAudio(
}
},
[
+ enableContinuousStreaming,
isActive,
isMuted,
VOLUME_THRESHOLD,
@@ -341,6 +363,10 @@ export function useSpeechmaticsAudio(
toggleRecording(false);
}
+ if (enableContinuousStreaming) {
+ flushBufferedAudio(true);
+ }
+
// Tear down audio session
if (audioInitialized) {
tearDown();
diff --git a/packages/app/src/hooks/use-speechmatics-audio.web.ts b/packages/app/src/hooks/use-speechmatics-audio.web.ts
index 2d62884b7..320e83b66 100644
--- a/packages/app/src/hooks/use-speechmatics-audio.web.ts
+++ b/packages/app/src/hooks/use-speechmatics-audio.web.ts
@@ -5,6 +5,7 @@ export interface SpeechmaticsAudioConfig {
onSpeechStart?: () => void;
onSpeechEnd?: () => void;
onError?: (error: Error) => void;
+ enableContinuousStreaming?: boolean;
volumeThreshold: number;
silenceDuration: number;
speechConfirmationDuration: number;
diff --git a/packages/server/src/client/daemon-client-v2.ts b/packages/server/src/client/daemon-client-v2.ts
index 81916a34e..a4ac5588d 100644
--- a/packages/server/src/client/daemon-client-v2.ts
+++ b/packages/server/src/client/daemon-client-v2.ts
@@ -42,7 +42,6 @@ import type {
SendAgentMessage,
SessionInboundMessage,
SessionOutboundMessage,
- TranscriptionResultMessage,
} from "../shared/messages.js";
import type {
AgentPermissionRequest,
@@ -150,12 +149,6 @@ export type DaemonClientV2Config = {
export type SendMessageOptions = Pick;
-export type TranscribeAudioOptions = {
- audio: string; // base64 encoded
- format: string;
- timeout?: number;
-};
-
type AgentConfigOverrides = Partial>;
export type CreateAgentRequestOptions = {
@@ -188,7 +181,6 @@ type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"];
type ListProviderModelsPayload = ListProviderModelsResponseMessage["payload"];
type ListCommandsPayload = ListCommandsResponse["payload"];
type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
-type TranscriptionResultPayload = TranscriptionResultMessage["payload"];
type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
type ListTerminalsPayload = ListTerminalsResponse["payload"];
type CreateTerminalPayload = CreateTerminalResponse["payload"];
@@ -472,7 +464,26 @@ export class DaemonClientV2 {
throw new Error("Transport not connected");
}
const payload = SessionInboundMessageSchema.parse(message);
- this.transport.send(JSON.stringify({ type: "session", message: payload }));
+ try {
+ this.transport.send(JSON.stringify({ type: "session", message: payload }));
+ } catch (error) {
+ if (this.config.suppressSendErrors) {
+ return;
+ }
+ throw error instanceof Error ? error : new Error(String(error));
+ }
+ }
+
+ private sendSessionMessageStrict(message: SessionInboundMessage): void {
+ if (!this.transport || this.connectionState.status !== "connected") {
+ throw new Error("Transport not connected");
+ }
+ const payload = SessionInboundMessageSchema.parse(message);
+ try {
+ this.transport.send(JSON.stringify({ type: "session", message: payload }));
+ } catch (error) {
+ throw error instanceof Error ? error : new Error(String(error));
+ }
}
sendUserMessage(text: string): void {
@@ -839,37 +850,6 @@ export class DaemonClientV2 {
await this.sendAgentMessage(agentId, text, options);
}
- async transcribeAudio(
- options: TranscribeAudioOptions
- ): Promise {
- const requestId = this.createRequestId();
- const timeout = options.timeout ?? 120000;
-
- const responsePromise = this.waitFor(
- (msg) => {
- if (msg.type !== "transcription_result") {
- return null;
- }
- if (msg.payload.requestId !== requestId) {
- return null;
- }
- return msg.payload;
- },
- timeout,
- { skipQueue: true }
- );
-
- const message = SessionInboundMessageSchema.parse({
- type: "transcribe_audio_request",
- audio: options.audio,
- format: options.format,
- requestId,
- });
- this.sendSessionMessage(message);
-
- return responsePromise;
- }
-
async cancelAgent(agentId: string): Promise {
this.sendSessionMessage({ type: "cancel_agent_request", agentId });
}
@@ -929,6 +909,87 @@ export class DaemonClientV2 {
this.sendSessionMessage({ type: "realtime_audio_chunk", audio, format, isLast });
}
+ startDictationStream(dictationId: string, format: string): Promise {
+ const ackPromise = this.waitFor(
+ (msg) => {
+ if (msg.type !== "dictation_stream_ack") {
+ return null;
+ }
+ if (msg.payload.dictationId !== dictationId) {
+ return null;
+ }
+ if (msg.payload.ackSeq !== -1) {
+ return null;
+ }
+ return msg.payload;
+ },
+ 30000,
+ { skipQueue: true }
+ ).then(() => undefined);
+
+ const errorPromise = this.waitFor(
+ (msg) => {
+ if (msg.type !== "dictation_stream_error") {
+ return null;
+ }
+ if (msg.payload.dictationId !== dictationId) {
+ return null;
+ }
+ return msg.payload;
+ },
+ 30000,
+ { skipQueue: true }
+ ).then((payload) => {
+ throw new Error(payload.error);
+ });
+
+ this.sendSessionMessageStrict({ type: "dictation_stream_start", dictationId, format });
+ return Promise.race([ackPromise, errorPromise]);
+ }
+
+ sendDictationStreamChunk(dictationId: string, seq: number, audio: string, format: string): void {
+ this.sendSessionMessageStrict({ type: "dictation_stream_chunk", dictationId, seq, audio, format });
+ }
+
+ finishDictationStream(dictationId: string, finalSeq: number): Promise<{ dictationId: string; text: string }> {
+ const finalPromise = this.waitFor(
+ (msg) => {
+ if (msg.type !== "dictation_stream_final") {
+ return null;
+ }
+ if (msg.payload.dictationId !== dictationId) {
+ return null;
+ }
+ return msg.payload;
+ },
+ 120000,
+ { skipQueue: true }
+ );
+
+ const errorPromise = this.waitFor(
+ (msg) => {
+ if (msg.type !== "dictation_stream_error") {
+ return null;
+ }
+ if (msg.payload.dictationId !== dictationId) {
+ return null;
+ }
+ return msg.payload;
+ },
+ 120000,
+ { skipQueue: true }
+ ).then((payload) => {
+ throw new Error(payload.error);
+ });
+
+ this.sendSessionMessageStrict({ type: "dictation_stream_finish", dictationId, finalSeq });
+ return Promise.race([finalPromise, errorPromise]);
+ }
+
+ cancelDictationStream(dictationId: string): void {
+ this.sendSessionMessageStrict({ type: "dictation_stream_cancel", dictationId });
+ }
+
async abortRequest(): Promise {
this.sendSessionMessage({ type: "abort_request" });
}
@@ -2097,7 +2158,12 @@ function createWebSocketTransportFactory(
return ({ url, headers }) => {
const ws = factory(url, { headers });
return {
- send: (data) => ws.send(data),
+ send: (data) => {
+ if (typeof ws.readyState === "number" && ws.readyState !== 1) {
+ throw new Error(`WebSocket not open (readyState=${ws.readyState})`);
+ }
+ ws.send(data);
+ },
close: (code?: number, reason?: string) => ws.close(code, reason),
onOpen: (handler) => bindWsHandler(ws, "open", handler),
onClose: (handler) => bindWsHandler(ws, "close", handler),
diff --git a/packages/server/src/server/agent/openai-realtime-transcription.ts b/packages/server/src/server/agent/openai-realtime-transcription.ts
new file mode 100644
index 000000000..9de64ed80
--- /dev/null
+++ b/packages/server/src/server/agent/openai-realtime-transcription.ts
@@ -0,0 +1,207 @@
+import type pino from "pino";
+import WebSocket from "ws";
+import { EventEmitter } from "node:events";
+
+type OpenAIClientEvent =
+ | {
+ type: "session.update";
+ session: {
+ type: "transcription";
+ audio: {
+ input: {
+ format: { type: "audio/pcm"; rate: 24000 };
+ transcription: {
+ model: string;
+ language?: string;
+ prompt?: string;
+ };
+ turn_detection: null;
+ };
+ };
+ };
+ }
+ | { type: "input_audio_buffer.append"; audio: string }
+ | { type: "input_audio_buffer.commit" }
+ | { type: "input_audio_buffer.clear" };
+
+type OpenAIServerEvent =
+ | { type: "session.created" | "session.updated" }
+ | {
+ type: "input_audio_buffer.committed";
+ item_id: string;
+ previous_item_id: string | null;
+ }
+ | {
+ type: "conversation.item.input_audio_transcription.completed";
+ item_id: string;
+ transcript: string;
+ }
+ | { type: "error"; error?: { message?: string } };
+
+export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
+ private readonly apiKey: string;
+ private readonly logger: pino.Logger;
+ private readonly transcriptionModel: string;
+ private readonly language?: string;
+
+ private ws: WebSocket | null = null;
+ private ready: Promise | null = null;
+ private closing = false;
+
+ constructor(params: {
+ apiKey: string;
+ logger: pino.Logger;
+ transcriptionModel: string;
+ language?: string;
+ }) {
+ super();
+ this.apiKey = params.apiKey;
+ this.logger = params.logger.child({ provider: "openai", component: "realtime-transcription" });
+ this.transcriptionModel = params.transcriptionModel;
+ this.language = params.language;
+ }
+
+ public async connect(): Promise {
+ if (this.ready) {
+ return this.ready;
+ }
+
+ this.closing = false;
+ this.ready = new Promise((resolve, reject) => {
+ const url = "wss://api.openai.com/v1/realtime?intent=transcription";
+ const ws = new WebSocket(url, {
+ headers: {
+ Authorization: `Bearer ${this.apiKey}`,
+ },
+ });
+ this.ws = ws;
+
+ let resolved = false;
+
+ const fail = (error: Error) => {
+ if (resolved) {
+ this.emit("error", error);
+ return;
+ }
+ resolved = true;
+ reject(error);
+ };
+
+ ws.on("open", () => {
+ this.logger.debug("OpenAI realtime transcription websocket connected");
+ const update: OpenAIClientEvent = {
+ type: "session.update",
+ session: {
+ type: "transcription",
+ audio: {
+ input: {
+ format: { type: "audio/pcm", rate: 24000 },
+ transcription: {
+ model: this.transcriptionModel,
+ ...(this.language ? { language: this.language } : {}),
+ },
+ // We commit periodically ourselves; no server-side VAD for dictation.
+ turn_detection: null,
+ },
+ },
+ },
+ };
+ ws.send(JSON.stringify(update));
+ });
+
+ ws.on("message", (data) => {
+ const text = typeof data === "string" ? data : data.toString("utf-8");
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(text);
+ } catch {
+ return;
+ }
+
+ const event = parsed as OpenAIServerEvent;
+ if (
+ event.type === "session.created" ||
+ event.type === "session.updated"
+ ) {
+ if (!resolved) {
+ resolved = true;
+ resolve();
+ }
+ return;
+ }
+
+ if (event.type === "input_audio_buffer.committed") {
+ this.emit("committed", {
+ itemId: event.item_id,
+ previousItemId: event.previous_item_id,
+ });
+ return;
+ }
+
+ if (event.type === "conversation.item.input_audio_transcription.completed") {
+ this.emit("transcript", { itemId: event.item_id, transcript: event.transcript });
+ return;
+ }
+
+ if (event.type === "error") {
+ const message = event.error?.message ?? "OpenAI realtime error";
+ fail(new Error(message));
+ }
+ });
+
+ ws.on("error", (err) => {
+ fail(err instanceof Error ? err : new Error(String(err)));
+ });
+
+ ws.on("close", () => {
+ this.logger.debug("OpenAI realtime websocket closed");
+ if (this.closing) {
+ return;
+ }
+ if (!resolved) {
+ fail(new Error("OpenAI realtime websocket closed before ready"));
+ return;
+ }
+ fail(new Error("OpenAI realtime websocket closed"));
+ });
+ });
+
+ return this.ready;
+ }
+
+ public appendPcm16Base64(base64Audio: string): void {
+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+ throw new Error("OpenAI realtime websocket not connected");
+ }
+ const event: OpenAIClientEvent = { type: "input_audio_buffer.append", audio: base64Audio };
+ this.ws.send(JSON.stringify(event));
+ }
+
+ public commit(): void {
+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+ throw new Error("OpenAI realtime websocket not connected");
+ }
+ const event: OpenAIClientEvent = { type: "input_audio_buffer.commit" };
+ this.ws.send(JSON.stringify(event));
+ }
+
+ public clear(): void {
+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+ return;
+ }
+ const event: OpenAIClientEvent = { type: "input_audio_buffer.clear" };
+ this.ws.send(JSON.stringify(event));
+ }
+
+ public close(): void {
+ try {
+ this.closing = true;
+ this.ws?.close();
+ } catch {
+ // no-op
+ } finally {
+ this.ws = null;
+ this.ready = null;
+ }
+ }
+}
diff --git a/packages/server/src/server/agent/pcm16-resampler.ts b/packages/server/src/server/agent/pcm16-resampler.ts
new file mode 100644
index 000000000..24cb10862
--- /dev/null
+++ b/packages/server/src/server/agent/pcm16-resampler.ts
@@ -0,0 +1,83 @@
+export class Pcm16MonoResampler {
+ private readonly inputRate: number;
+ private readonly outputRate: number;
+ private readonly step: number;
+ private pos: number;
+ private carrySample: number | null;
+
+ constructor(params: { inputRate: number; outputRate: number }) {
+ this.inputRate = params.inputRate;
+ this.outputRate = params.outputRate;
+ this.step = this.inputRate / this.outputRate;
+ this.pos = 0;
+ this.carrySample = null;
+ }
+
+ public reset(): void {
+ this.pos = 0;
+ this.carrySample = null;
+ }
+
+ public processChunk(pcm16le: Buffer): Buffer {
+ if (pcm16le.length === 0) {
+ return Buffer.alloc(0);
+ }
+ if (pcm16le.length % 2 !== 0) {
+ throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
+ }
+
+ const srcChunk = new Int16Array(
+ pcm16le.buffer,
+ pcm16le.byteOffset,
+ pcm16le.byteLength / 2
+ );
+
+ const hasCarry = this.carrySample !== null;
+ const srcLen = srcChunk.length + (hasCarry ? 1 : 0);
+ if (srcLen < 2) {
+ this.carrySample = srcChunk.length ? srcChunk[srcChunk.length - 1] : this.carrySample;
+ return Buffer.alloc(0);
+ }
+
+ const src = new Float32Array(srcLen);
+ let offset = 0;
+ if (hasCarry) {
+ src[0] = (this.carrySample as number) / 32768;
+ offset = 1;
+ }
+ for (let i = 0; i < srcChunk.length; i += 1) {
+ src[offset + i] = srcChunk[i] / 32768;
+ }
+
+ const out: number[] = [];
+ const maxPos = src.length - 1;
+
+ while (this.pos < maxPos) {
+ const i = Math.floor(this.pos);
+ const frac = this.pos - i;
+ const s0 = src[i]!;
+ const s1 = src[i + 1]!;
+ const sample = s0 + (s1 - s0) * frac;
+ const clamped = Math.max(-1, Math.min(1, sample));
+ const int16 = Math.round(clamped * 32767);
+ out.push(int16);
+ this.pos += this.step;
+ }
+
+ // Keep the last input sample as carry for the next chunk.
+ const lastInput = srcChunk[srcChunk.length - 1]!;
+ this.carrySample = lastInput;
+
+ // Shift position so next chunk (which will include carry sample) continues smoothly.
+ const shift = src.length - 1;
+ this.pos = this.pos - shift;
+ if (this.pos < 0) {
+ // Guard against floating point drift.
+ this.pos = 0;
+ }
+
+ const outArr = Int16Array.from(out);
+ return Buffer.from(outArr.buffer, outArr.byteOffset, outArr.byteLength);
+ }
+}
+
diff --git a/packages/server/src/server/daemon-client-v2.e2e.test.ts b/packages/server/src/server/daemon-client-v2.e2e.test.ts
index 1545a935c..4dc135f4c 100644
--- a/packages/server/src/server/daemon-client-v2.e2e.test.ts
+++ b/packages/server/src/server/daemon-client-v2.e2e.test.ts
@@ -23,6 +23,9 @@ if (envPath) {
dotenv.config({ path: envPath });
}
+// Make dictation streaming commit frequently in tests so we exercise multi-commit assembly logic.
+process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ??= "1000";
+
function tmpCwd(): string {
return mkdtempSync(path.join(tmpdir(), "daemon-client-v2-"));
}
@@ -630,6 +633,216 @@ describe("daemon client v2 E2E", () => {
180000
);
+ test(
+ "streams dictation PCM and returns final transcript via OpenAI Realtime transcription",
+ async () => {
+ if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+ // Requires OpenAI realtime transcription and is inherently network-flaky.
+ return;
+ }
+ if (!process.env.OPENAI_API_KEY) {
+ return;
+ }
+
+ const fixturePath = path.resolve(
+ process.cwd(),
+ "..",
+ "app",
+ "e2e",
+ "fixtures",
+ "recording.wav"
+ );
+ const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
+
+ const parsePcm16MonoWav = (buffer: Buffer): Buffer => {
+ if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
+ throw new Error("Invalid WAV header");
+ }
+ let offset = 12;
+ let fmt: { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } | null = null;
+ let dataChunk: Buffer | null = null;
+
+ while (offset + 8 <= buffer.length) {
+ const id = buffer.toString("ascii", offset, offset + 4);
+ const size = buffer.readUInt32LE(offset + 4);
+ const payloadStart = offset + 8;
+ const payloadEnd = payloadStart + size;
+ if (payloadEnd > buffer.length) {
+ break;
+ }
+
+ if (id === "fmt ") {
+ const audioFormat = buffer.readUInt16LE(payloadStart);
+ const channels = buffer.readUInt16LE(payloadStart + 2);
+ const sampleRate = buffer.readUInt32LE(payloadStart + 4);
+ const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
+ fmt = { audioFormat, channels, sampleRate, bitsPerSample };
+ } else if (id === "data") {
+ dataChunk = buffer.subarray(payloadStart, payloadEnd);
+ }
+
+ offset = payloadEnd + (size % 2);
+ }
+
+ if (!fmt || !dataChunk) {
+ throw new Error("Missing WAV fmt/data chunks");
+ }
+ if (fmt.audioFormat !== 1) {
+ throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
+ }
+ if (fmt.channels !== 1 || fmt.sampleRate !== 16000 || fmt.bitsPerSample !== 16) {
+ throw new Error(
+ `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
+ );
+ }
+ if (dataChunk.length % 2 !== 0) {
+ throw new Error("WAV PCM16 data length must be even");
+ }
+ return dataChunk;
+ };
+
+ const pcm16 = parsePcm16MonoWav(wav);
+ const dictationId = `dict-${Date.now()}`;
+ const format = "audio/pcm;rate=16000;bits=16";
+
+ await ctx.client.startDictationStream(dictationId, format);
+
+ const chunkBytes = 3200; // ~100ms @ 16kHz mono PCM16 (1600 samples * 2 bytes)
+ let seq = 0;
+ for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
+ const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
+ ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
+ seq += 1;
+ }
+
+ const finalSeq = seq - 1;
+ const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
+
+ expect(result.dictationId).toBe(dictationId);
+ expect(result.text.toLowerCase()).toContain("voice note");
+ },
+ 180000
+ );
+
+ test(
+ "does not commit an empty OpenAI realtime audio buffer on finish",
+ async () => {
+ if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+ return;
+ }
+ if (!process.env.OPENAI_API_KEY) {
+ return;
+ }
+
+ const fixturePath = path.resolve(
+ process.cwd(),
+ "..",
+ "app",
+ "e2e",
+ "fixtures",
+ "recording.wav"
+ );
+ const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
+
+ const parsePcm16MonoWav = (buffer: Buffer): Buffer => {
+ if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
+ throw new Error("Invalid WAV header");
+ }
+ let offset = 12;
+ let fmt: { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } | null = null;
+ let dataChunk: Buffer | null = null;
+
+ while (offset + 8 <= buffer.length) {
+ const id = buffer.toString("ascii", offset, offset + 4);
+ const size = buffer.readUInt32LE(offset + 4);
+ const payloadStart = offset + 8;
+ const payloadEnd = payloadStart + size;
+ if (payloadEnd > buffer.length) {
+ break;
+ }
+
+ if (id === "fmt ") {
+ const audioFormat = buffer.readUInt16LE(payloadStart);
+ const channels = buffer.readUInt16LE(payloadStart + 2);
+ const sampleRate = buffer.readUInt32LE(payloadStart + 4);
+ const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
+ fmt = { audioFormat, channels, sampleRate, bitsPerSample };
+ } else if (id === "data") {
+ dataChunk = buffer.subarray(payloadStart, payloadEnd);
+ }
+
+ offset = payloadEnd + (size % 2);
+ }
+
+ if (!fmt || !dataChunk) {
+ throw new Error("Missing WAV fmt/data chunks");
+ }
+ if (fmt.audioFormat !== 1) {
+ throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
+ }
+ if (fmt.channels !== 1 || fmt.sampleRate !== 16000 || fmt.bitsPerSample !== 16) {
+ throw new Error(
+ `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
+ );
+ }
+ if (dataChunk.length % 2 !== 0) {
+ throw new Error("WAV PCM16 data length must be even");
+ }
+ return dataChunk;
+ };
+
+ const pcm16 = parsePcm16MonoWav(wav);
+ const dictationId = `dict-empty-commit-${Date.now()}`;
+ const format = "audio/pcm;rate=16000;bits=16";
+
+ await ctx.client.startDictationStream(dictationId, format);
+
+ // Send exactly 10x 100ms chunks. With OPENAI_REALTIME_DICTATION_COMMIT_MS=1000 in this test file,
+ // the server will auto-commit at the 1s boundary. Finishing immediately after that should not
+ // attempt an additional empty commit.
+ const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
+ const targetChunks = 10;
+ let seq = 0;
+ for (let i = 0; i < targetChunks; i += 1) {
+ const start = i * chunkBytes;
+ const end = Math.min(pcm16.length, start + chunkBytes);
+ const chunk = pcm16.subarray(start, end);
+ ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
+ seq += 1;
+ }
+
+ const finalSeq = seq - 1;
+ const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
+
+ expect(result.dictationId).toBe(dictationId);
+ expect(typeof result.text).toBe("string");
+ },
+ 180000
+ );
+
+ test(
+ "fails fast if dictation finishes without sending required chunks",
+ async () => {
+ if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+ return;
+ }
+ if (!process.env.OPENAI_API_KEY) {
+ return;
+ }
+
+ const dictationId = `dict-missing-chunks-${Date.now()}`;
+ const format = "audio/pcm;rate=16000;bits=16";
+
+ await ctx.client.startDictationStream(dictationId, format);
+
+ // Claim that we sent chunk 0, but actually send no chunks.
+ await expect(ctx.client.finishDictationStream(dictationId, 0)).rejects.toThrow(
+ /no audio chunks were received/i
+ );
+ },
+ 180000
+ );
+
test(
"supports git and file operations",
async () => {
diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts
index ea7587612..200ef6d9a 100644
--- a/packages/server/src/server/session.ts
+++ b/packages/server/src/server/session.ts
@@ -35,6 +35,8 @@ import { TTSManager } from "./agent/tts-manager.js";
import { STTManager } from "./agent/stt-manager.js";
import type { OpenAISTT } from "./agent/stt-openai.js";
import type { OpenAITTS } from "./agent/tts-openai.js";
+import { OpenAIRealtimeTranscriptionSession } from "./agent/openai-realtime-transcription.js";
+import { Pcm16MonoResampler } from "./agent/pcm16-resampler.js";
import type { VoiceConversationStore } from "./voice-conversation-store.js";
import {
buildConfigOverrides,
@@ -135,6 +137,21 @@ const MIN_STREAMING_SEGMENT_DURATION_MS = 1000;
const MIN_STREAMING_SEGMENT_BYTES = Math.round(
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
);
+
+const DICTATION_PCM_INPUT_RATE = 16000;
+const DICTATION_PCM_OUTPUT_RATE = 24000;
+const DICTATION_COMMIT_INTERVAL_MS = Number.parseInt(
+ process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ?? "5000",
+ 10
+);
+const DICTATION_COMMIT_TARGET_BYTES = Math.round(
+ (DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_COMMIT_INTERVAL_MS) / 1000
+);
+const DICTATION_MIN_COMMIT_MS = 100;
+const DICTATION_MIN_COMMIT_BYTES = Math.round(
+ (DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_MIN_COMMIT_MS) / 1000
+);
+const DICTATION_FINAL_TIMEOUT_MS = 120000;
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
/**
@@ -249,6 +266,27 @@ export class Session {
private speechInProgress = false;
private voiceConversationId: string | null = null;
+ private dictationStreams = new Map<
+ string,
+ {
+ dictationId: string;
+ inputFormat: string;
+ openai: OpenAIRealtimeTranscriptionSession;
+ resampler: Pcm16MonoResampler;
+ receivedChunks: Map;
+ nextSeqToForward: number;
+ ackSeq: number;
+ bytesSinceCommit: number;
+ expectedCommits: number;
+ committedCount: number;
+ committedItemIds: string[];
+ transcriptsByItemId: Map;
+ finishRequested: boolean;
+ finalSeq: number | null;
+ finalTimeout: ReturnType | null;
+ }
+ >();
+
// Audio buffering for interruption handling
private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
private bufferTimeout: NodeJS.Timeout | null = null;
@@ -735,8 +773,20 @@ export class Session {
);
break;
- case "transcribe_audio_request":
- await this.handleTranscribeAudio(msg);
+ case "dictation_stream_start":
+ await this.handleDictationStreamStart(msg);
+ break;
+
+ case "dictation_stream_chunk":
+ await this.handleDictationStreamChunk(msg);
+ break;
+
+ case "dictation_stream_finish":
+ await this.handleDictationStreamFinish(msg);
+ break;
+
+ case "dictation_stream_cancel":
+ await this.handleDictationStreamCancel(msg);
break;
case "create_agent_request":
@@ -1177,56 +1227,279 @@ export class Session {
this.startAgentStream(agentId, prompt);
}
- /**
- * Handle audio transcription request
- */
- private async handleTranscribeAudio(
- msg: Extract
- ): Promise {
- const { audio, format, requestId } = msg;
+ private emitDictationAck(dictationId: string, ackSeq: number): void {
+ this.emit({
+ type: "dictation_stream_ack",
+ payload: { dictationId, ackSeq },
+ });
+ }
- const audioBuffer = Buffer.from(audio, "base64");
+ private failDictationStream(dictationId: string, error: string, retryable: boolean): void {
+ this.emit({
+ type: "dictation_stream_error",
+ payload: { dictationId, error, retryable },
+ });
+ }
- this.sessionLogger.debug({ requestId }, "Transcribing audio");
-
- try {
- const result = await this.sttManager.transcribe(audioBuffer, format, {
- requestId,
- label: "dictation",
- });
-
- this.sessionLogger.info(
- { requestId, textLength: result.text.length },
- "Transcription complete"
- );
-
- this.emit({
- type: "transcription_result",
- payload: {
- text: result.text,
- language: result.language,
- duration: result.duration,
- requestId,
- avgLogprob: result.avgLogprob,
- isLowConfidence: result.isLowConfidence,
- byteLength: result.byteLength,
- format: result.format,
- debugRecordingPath: result.debugRecordingPath,
- },
- });
- } catch (error: any) {
- this.sessionLogger.error({ err: error, requestId }, "Transcription failed");
- this.emit({
- type: "activity_log",
- payload: {
- id: uuidv4(),
- timestamp: new Date(),
- type: "error",
- content: `Transcription failed: ${error.message}`,
- },
- });
- throw error;
+ private cleanupDictationStream(dictationId: string): void {
+ const state = this.dictationStreams.get(dictationId) ?? null;
+ if (!state) {
+ return;
}
+ if (state.finalTimeout) {
+ clearTimeout(state.finalTimeout);
+ }
+ try {
+ state.openai.close();
+ } catch {
+ // no-op
+ }
+ this.dictationStreams.delete(dictationId);
+ }
+
+ private maybeFinalizeDictationStream(dictationId: string): void {
+ const state = this.dictationStreams.get(dictationId);
+ if (!state) {
+ return;
+ }
+
+ if (!state.finishRequested || state.finalSeq === null) {
+ return;
+ }
+ if (state.ackSeq < state.finalSeq) {
+ return;
+ }
+ if (state.committedCount < state.expectedCommits) {
+ return;
+ }
+
+ const allTranscriptsReady = state.committedItemIds.every((itemId) =>
+ state.transcriptsByItemId.has(itemId)
+ );
+ if (!allTranscriptsReady) {
+ return;
+ }
+
+ const orderedText = state.committedItemIds
+ .map((itemId) => state.transcriptsByItemId.get(itemId) ?? "")
+ .join(" ")
+ .trim();
+
+ this.emit({
+ type: "dictation_stream_final",
+ payload: { dictationId, text: orderedText },
+ });
+
+ this.cleanupDictationStream(dictationId);
+ }
+
+ private async handleDictationStreamStart(
+ msg: Extract
+ ): Promise {
+ const dictationId = msg.dictationId;
+ this.cleanupDictationStream(dictationId);
+
+ const apiKey = process.env.OPENAI_API_KEY;
+ if (!apiKey) {
+ this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false);
+ return;
+ }
+
+ const transcriptionModel =
+ process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
+
+ const openai = new OpenAIRealtimeTranscriptionSession({
+ apiKey,
+ logger: this.sessionLogger.child({ dictationId }),
+ transcriptionModel,
+ language: "en",
+ });
+
+ openai.on("committed", ({ itemId }: { itemId: string }) => {
+ const state = this.dictationStreams.get(dictationId);
+ if (!state) {
+ return;
+ }
+ state.committedCount += 1;
+ state.committedItemIds.push(itemId);
+ this.maybeFinalizeDictationStream(dictationId);
+ });
+
+ openai.on("transcript", ({ itemId, transcript }: { itemId: string; transcript: string }) => {
+ const state = this.dictationStreams.get(dictationId);
+ if (!state) {
+ return;
+ }
+ state.transcriptsByItemId.set(itemId, transcript);
+ this.maybeFinalizeDictationStream(dictationId);
+ });
+
+ openai.on("error", (err) => {
+ const error = err instanceof Error ? err.message : String(err);
+ this.failDictationStream(dictationId, error, true);
+ this.cleanupDictationStream(dictationId);
+ });
+
+ await openai.connect();
+
+ this.dictationStreams.set(dictationId, {
+ dictationId,
+ inputFormat: msg.format,
+ openai,
+ resampler: new Pcm16MonoResampler({
+ inputRate: DICTATION_PCM_INPUT_RATE,
+ outputRate: DICTATION_PCM_OUTPUT_RATE,
+ }),
+ receivedChunks: new Map(),
+ nextSeqToForward: 0,
+ ackSeq: -1,
+ bytesSinceCommit: 0,
+ expectedCommits: 0,
+ committedCount: 0,
+ committedItemIds: [],
+ transcriptsByItemId: new Map(),
+ finishRequested: false,
+ finalSeq: null,
+ finalTimeout: null,
+ });
+
+ this.emitDictationAck(dictationId, -1);
+ }
+
+ private async handleDictationStreamChunk(
+ msg: Extract
+ ): Promise {
+ const state = this.dictationStreams.get(msg.dictationId);
+ if (!state) {
+ this.failDictationStream(msg.dictationId, "Dictation stream not started", true);
+ return;
+ }
+
+ if (msg.format !== state.inputFormat) {
+ this.failDictationStream(
+ msg.dictationId,
+ `Mismatched dictation stream format: ${msg.format}`,
+ false
+ );
+ return;
+ }
+
+ if (msg.seq < state.nextSeqToForward) {
+ this.emitDictationAck(msg.dictationId, state.ackSeq);
+ return;
+ }
+
+ if (!state.receivedChunks.has(msg.seq)) {
+ state.receivedChunks.set(msg.seq, Buffer.from(msg.audio, "base64"));
+ }
+
+ while (state.receivedChunks.has(state.nextSeqToForward)) {
+ const seq = state.nextSeqToForward;
+ const pcm16 = state.receivedChunks.get(seq)!;
+ state.receivedChunks.delete(seq);
+
+ const resampled = state.resampler.processChunk(pcm16);
+ if (resampled.length > 0) {
+ state.openai.appendPcm16Base64(resampled.toString("base64"));
+ state.bytesSinceCommit += resampled.length;
+ }
+
+ state.nextSeqToForward += 1;
+ state.ackSeq = state.nextSeqToForward - 1;
+
+ if (state.bytesSinceCommit >= DICTATION_COMMIT_TARGET_BYTES) {
+ state.expectedCommits += 1;
+ state.openai.commit();
+ state.bytesSinceCommit = 0;
+ }
+ }
+
+ this.emitDictationAck(msg.dictationId, state.ackSeq);
+ this.maybeFinalizeDictationStream(msg.dictationId);
+ }
+
+ private async handleDictationStreamFinish(
+ msg: Extract
+ ): Promise {
+ const state = this.dictationStreams.get(msg.dictationId);
+ if (!state) {
+ this.failDictationStream(msg.dictationId, "Dictation stream not started", true);
+ return;
+ }
+
+ state.finishRequested = true;
+ state.finalSeq = msg.finalSeq;
+
+ // If the client claims it sent audio (finalSeq >= 0) but we haven't received any chunks at all,
+ // we should fail fast instead of hanging until DICTATION_FINAL_TIMEOUT_MS.
+ if (msg.finalSeq >= 0 && state.ackSeq < 0 && state.nextSeqToForward === 0 && state.receivedChunks.size === 0) {
+ this.sessionLogger.debug(
+ {
+ dictationId: msg.dictationId,
+ finalSeq: msg.finalSeq,
+ ackSeq: state.ackSeq,
+ nextSeqToForward: state.nextSeqToForward,
+ receivedChunks: state.receivedChunks.size,
+ expectedCommits: state.expectedCommits,
+ committedCount: state.committedCount,
+ },
+ "Dictation finish: no chunks received (failing fast)"
+ );
+ this.failDictationStream(
+ msg.dictationId,
+ `Dictation finished (finalSeq=${msg.finalSeq}) but no audio chunks were received`,
+ true
+ );
+ this.cleanupDictationStream(msg.dictationId);
+ return;
+ }
+
+ // Commit any remaining uncommitted audio.
+ // IMPORTANT: OpenAI requires at least ~100ms of audio in the buffer for a commit.
+ // Also, committing an empty buffer throws "buffer too small ... 0.00ms".
+ if (state.bytesSinceCommit > 0) {
+ this.sessionLogger.debug(
+ { dictationId: msg.dictationId, bytesSinceCommit: state.bytesSinceCommit },
+ "Dictation finish: committing pending audio"
+ );
+ if (state.bytesSinceCommit < DICTATION_MIN_COMMIT_BYTES) {
+ const padBytes = DICTATION_MIN_COMMIT_BYTES - state.bytesSinceCommit;
+ this.sessionLogger.debug(
+ { dictationId: msg.dictationId, padBytes },
+ "Dictation finish: padding to minimum commit size"
+ );
+ state.openai.appendPcm16Base64(Buffer.alloc(padBytes).toString("base64"));
+ state.bytesSinceCommit += padBytes;
+ }
+ state.expectedCommits += 1;
+ state.openai.commit();
+ state.bytesSinceCommit = 0;
+ } else {
+ this.sessionLogger.debug(
+ { dictationId: msg.dictationId },
+ "Dictation finish: no pending audio to commit"
+ );
+ }
+
+ if (state.finalTimeout) {
+ clearTimeout(state.finalTimeout);
+ }
+ state.finalTimeout = setTimeout(() => {
+ this.failDictationStream(
+ msg.dictationId,
+ "Timed out waiting for final transcription",
+ true
+ );
+ this.cleanupDictationStream(msg.dictationId);
+ }, DICTATION_FINAL_TIMEOUT_MS);
+
+ this.maybeFinalizeDictationStream(msg.dictationId);
+ }
+
+ private async handleDictationStreamCancel(
+ msg: Extract
+ ): Promise {
+ this.cleanupDictationStream(msg.dictationId);
}
/**
@@ -4209,6 +4482,10 @@ export class Session {
this.ttsManager.cleanup();
this.sttManager.cleanup();
+ for (const dictationId of this.dictationStreams.keys()) {
+ this.cleanupDictationStream(dictationId);
+ }
+
// Close MCP clients
if (this.agentMcpClient) {
try {
diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts
index 79646f182..17afef7b0 100644
--- a/packages/server/src/server/websocket-server.ts
+++ b/packages/server/src/server/websocket-server.ts
@@ -57,7 +57,13 @@ export class VoiceAssistantWebSocketServer {
path: "/ws",
verifyClient: ({ req }, callback) => {
const origin = req.headers.origin;
- if (!origin || allowedOrigins.has(origin)) {
+ const requestHost = typeof req.headers.host === "string" ? req.headers.host : null;
+ const sameOrigin =
+ !!origin &&
+ !!requestHost &&
+ (origin === `http://${requestHost}` || origin === `https://${requestHost}`);
+
+ if (!origin || allowedOrigins.has(origin) || sameOrigin) {
callback(true);
} else {
this.logger.warn({ origin }, "Rejected connection from origin");
diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts
index ea2f5adbf..ea540d595 100644
--- a/packages/server/src/shared/messages.ts
+++ b/packages/server/src/shared/messages.ts
@@ -329,11 +329,33 @@ export const SendAgentMessageSchema = z.object({
})).optional(),
});
-export const TranscribeAudioRequestSchema = z.object({
- type: z.literal("transcribe_audio_request"),
- audio: z.string(), // base64 encoded
- format: z.string(),
- requestId: z.string(),
+// ============================================================================
+// Dictation Streaming (lossless, resumable)
+// ============================================================================
+
+export const DictationStreamStartMessageSchema = z.object({
+ type: z.literal("dictation_stream_start"),
+ dictationId: z.string(),
+ format: z.string(), // e.g. "audio/pcm;rate=16000;bits=16"
+});
+
+export const DictationStreamChunkMessageSchema = z.object({
+ type: z.literal("dictation_stream_chunk"),
+ dictationId: z.string(),
+ seq: z.number().int().nonnegative(),
+ audio: z.string(), // base64 encoded chunk
+ format: z.string(), // e.g. "audio/pcm;rate=16000;bits=16"
+});
+
+export const DictationStreamFinishMessageSchema = z.object({
+ type: z.literal("dictation_stream_finish"),
+ dictationId: z.string(),
+ finalSeq: z.number().int().nonnegative(),
+});
+
+export const DictationStreamCancelMessageSchema = z.object({
+ type: z.literal("dictation_stream_cancel"),
+ dictationId: z.string(),
});
const GitSetupOptionsSchema = z.object({
@@ -683,7 +705,10 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
DeleteAgentRequestMessageSchema,
SetVoiceConversationMessageSchema,
SendAgentMessageSchema,
- TranscribeAudioRequestSchema,
+ DictationStreamStartMessageSchema,
+ DictationStreamChunkMessageSchema,
+ DictationStreamFinishMessageSchema,
+ DictationStreamCancelMessageSchema,
CreateAgentRequestMessageSchema,
ListProviderModelsRequestMessageSchema,
ResumeAgentRequestMessageSchema,
@@ -780,6 +805,31 @@ export const TranscriptionResultMessageSchema = z.object({
}),
});
+export const DictationStreamAckMessageSchema = z.object({
+ type: z.literal("dictation_stream_ack"),
+ payload: z.object({
+ dictationId: z.string(),
+ ackSeq: z.number().int(),
+ }),
+});
+
+export const DictationStreamFinalMessageSchema = z.object({
+ type: z.literal("dictation_stream_final"),
+ payload: z.object({
+ dictationId: z.string(),
+ text: z.string(),
+ }),
+});
+
+export const DictationStreamErrorMessageSchema = z.object({
+ type: z.literal("dictation_stream_error"),
+ payload: z.object({
+ dictationId: z.string(),
+ error: z.string(),
+ retryable: z.boolean(),
+ }),
+});
+
export const StatusMessageSchema = z.object({
type: z.literal("status"),
payload: z
@@ -1244,6 +1294,9 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
AssistantChunkMessageSchema,
AudioOutputMessageSchema,
TranscriptionResultMessageSchema,
+ DictationStreamAckMessageSchema,
+ DictationStreamFinalMessageSchema,
+ DictationStreamErrorMessageSchema,
StatusMessageSchema,
InitializeAgentResponseMessageSchema,
ArtifactMessageSchema,
@@ -1323,7 +1376,10 @@ export type ActivityLogPayload = z.infer;
export type UserTextMessage = z.infer;
export type RealtimeAudioChunkMessage = z.infer;
export type SendAgentMessage = z.infer;
-export type TranscribeAudioRequest = z.infer;
+export type DictationStreamStartMessage = z.infer;
+export type DictationStreamChunkMessage = z.infer;
+export type DictationStreamFinishMessage = z.infer;
+export type DictationStreamCancelMessage = z.infer;
export type CreateAgentRequestMessage = z.infer;
export type ListProviderModelsRequestMessage = z.infer<
typeof ListProviderModelsRequestMessageSchema