diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index e514a5244..a0a0f42ad 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -8,6 +8,7 @@ import type { DictationStatus } from "@/hooks/use-dictation"; interface DictationControlsProps { volume: number; duration: number; + transcript?: string; isRecording: boolean; isProcessing: boolean; status: DictationStatus; @@ -131,6 +132,7 @@ export function DictationControls({ export function DictationOverlay({ volume, duration, + transcript, isRecording, isProcessing, status, @@ -169,21 +171,34 @@ export function DictationOverlay({ - - - {formatDuration(duration)} - + + + + {formatDuration(duration)} + + + {!!transcript && ( + + {transcript} + + )} @@ -334,6 +349,12 @@ const overlayStyles = StyleSheet.create((theme) => ({ }, centerContainer: { flex: 1, + flexDirection: "column", + alignItems: "center", + justifyContent: "center", + gap: theme.spacing[2], + }, + meterRow: { flexDirection: "row", alignItems: "center", justifyContent: "center", @@ -344,6 +365,13 @@ const overlayStyles = StyleSheet.create((theme) => ({ fontWeight: theme.fontWeight.semibold, fontVariant: ["tabular-nums"], }, + transcriptText: { + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.normal, + textAlign: "center", + paddingHorizontal: theme.spacing[2], + opacity: 0.9, + }, actionButtonsContainer: { flexDirection: "row", alignItems: "center", diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx index 5b9a1fc2e..4f31afb6a 100644 --- a/packages/app/src/components/message-input.tsx +++ b/packages/app/src/components/message-input.tsx @@ -191,6 +191,7 @@ export const MessageInput = forwardRef( const { isRecording: isDictating, isProcessing: isDictationProcessing, + partialTranscript: dictationPartialTranscript, volume: dictationVolume, duration: dictationDuration, status: dictationStatus, @@ -591,6 +592,7 @@ export const MessageInput = forwardRef( { try { + // When env vars define a default daemon, always reset to that daemon only. + // This ensures the app uses the configured daemon regardless of stored state. + const envDefaults = parseEnvDaemonDefaults(); + if (envDefaults.length > 0) { + await AsyncStorage.setItem(REGISTRY_STORAGE_KEY, JSON.stringify(envDefaults)); + return envDefaults; + } + const stored = await AsyncStorage.getItem(REGISTRY_STORAGE_KEY); if (stored) { const parsed = JSON.parse(stored) as unknown; @@ -390,12 +398,6 @@ async function loadDaemonRegistryFromStorage(): Promise { } } - const envDefaults = parseEnvDaemonDefaults(); - if (envDefaults.length > 0) { - await AsyncStorage.setItem(REGISTRY_STORAGE_KEY, JSON.stringify(envDefaults)); - return envDefaults; - } - // No implicit localhost fallback: a fresh install starts with zero hosts. return []; } catch (error) { diff --git a/packages/app/src/hooks/use-dictation.shared.ts b/packages/app/src/hooks/use-dictation.shared.ts index 68540a809..e5a047749 100644 --- a/packages/app/src/hooks/use-dictation.shared.ts +++ b/packages/app/src/hooks/use-dictation.shared.ts @@ -3,6 +3,7 @@ export type DictationStatus = "idle" | "recording" | "uploading" | "failed"; export type UseDictationOptions = { client: import("@server/client/daemon-client-v2").DaemonClientV2 | null; onTranscript: (text: string, meta: { requestId: string }) => void; + onPartialTranscript?: (text: string, meta: { requestId: string }) => void; onError?: (error: Error) => void; onPermanentFailure?: (error: Error, context: { requestId: string }) => void; canStart?: () => boolean; @@ -14,6 +15,7 @@ export type UseDictationOptions = { export type UseDictationResult = { isRecording: boolean; isProcessing: boolean; + partialTranscript: string; volume: number; duration: number; error: string | null; diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index 3a13bafdd..0a532fdd5 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -17,6 +17,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const { client, onTranscript, + onPartialTranscript, onError, onPermanentFailure, canStart, @@ -27,6 +28,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const [isRecording, setIsRecording] = useState(false); const [isProcessing, setIsProcessing] = useState(false); + const [partialTranscript, setPartialTranscript] = useState(""); const [duration, setDuration] = useState(0); const [error, setError] = useState(null); const [status, setStatus] = useState("idle"); @@ -36,6 +38,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { onTranscriptRef.current = onTranscript; }, [onTranscript]); + const onPartialTranscriptRef = useRef(onPartialTranscript); + useEffect(() => { + onPartialTranscriptRef.current = onPartialTranscript; + }, [onPartialTranscript]); + const onErrorRef = useRef(onError); useEffect(() => { onErrorRef.current = onError; @@ -123,6 +130,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const clearStreamingState = useCallback(() => { senderRef.current?.clearAll(); + setPartialTranscript(""); }, []); const startNewStream = useCallback( @@ -155,6 +163,27 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { }); }, [client, startNewStream]); + useEffect(() => { + if (!client) { + return; + } + return client.on("dictation_stream_partial", (message) => { + if (message.type !== "dictation_stream_partial") { + return; + } + const activeDictationId = senderRef.current?.getDictationId(); + if (!activeDictationId) { + return; + } + if (message.payload.dictationId !== activeDictationId) { + return; + } + const next = message.payload.text ?? ""; + setPartialTranscript(next); + onPartialTranscriptRef.current?.(next, { requestId: generateMessageId() }); + }); + }, [client]); + const audio = useDictationAudioSource({ onPcmSegment: (audioData) => { senderRef.current?.enqueueSegment(audioData); @@ -171,6 +200,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const handleStreamingTranscriptionSuccess = useCallback( (text: string, requestId: string) => { setIsProcessing(false); + setPartialTranscript(""); setDuration(0); setStatus("idle"); @@ -219,6 +249,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { actionGateRef.current.starting = true; setError(null); + setPartialTranscript(""); setDuration(0); setIsProcessing(false); setStatus("recording"); @@ -434,6 +465,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { return { isRecording, isProcessing, + partialTranscript, volume: audio.volume, duration, error, diff --git a/packages/server/src/server/agent/openai-realtime-transcription.ts b/packages/server/src/server/agent/openai-realtime-transcription.ts index 9de64ed80..a7beaced8 100644 --- a/packages/server/src/server/agent/openai-realtime-transcription.ts +++ b/packages/server/src/server/agent/openai-realtime-transcription.ts @@ -2,6 +2,21 @@ import type pino from "pino"; import WebSocket from "ws"; import { EventEmitter } from "node:events"; +type OpenAITurnDetection = + | null + | { + type: "server_vad"; + create_response?: boolean; + threshold?: number; + prefix_padding_ms?: number; + silence_duration_ms?: number; + } + | { + type: "semantic_vad"; + create_response?: boolean; + eagerness?: "low" | "medium" | "high"; + }; + type OpenAIClientEvent = | { type: "session.update"; @@ -15,7 +30,7 @@ type OpenAIClientEvent = language?: string; prompt?: string; }; - turn_detection: null; + turn_detection: OpenAITurnDetection; }; }; }; @@ -31,6 +46,13 @@ type OpenAIServerEvent = item_id: string; previous_item_id: string | null; } + | { type: "input_audio_buffer.speech_started" } + | { type: "input_audio_buffer.speech_stopped" } + | { + type: "conversation.item.input_audio_transcription.delta"; + item_id: string; + delta: string; + } | { type: "conversation.item.input_audio_transcription.completed"; item_id: string; @@ -43,22 +65,29 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter { private readonly logger: pino.Logger; private readonly transcriptionModel: string; private readonly language?: string; + private readonly prompt?: string; + private readonly turnDetection: OpenAITurnDetection; private ws: WebSocket | null = null; private ready: Promise | null = null; private closing = false; + private partialByItemId = new Map(); constructor(params: { apiKey: string; logger: pino.Logger; transcriptionModel: string; language?: string; + prompt?: string; + turnDetection?: OpenAITurnDetection; }) { super(); this.apiKey = params.apiKey; this.logger = params.logger.child({ provider: "openai", component: "realtime-transcription" }); this.transcriptionModel = params.transcriptionModel; this.language = params.language; + this.prompt = params.prompt; + this.turnDetection = params.turnDetection ?? null; } public async connect(): Promise { @@ -99,9 +128,9 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter { transcription: { model: this.transcriptionModel, ...(this.language ? { language: this.language } : {}), + ...(this.prompt ? { prompt: this.prompt } : {}), }, - // We commit periodically ourselves; no server-side VAD for dictation. - turn_detection: null, + turn_detection: this.turnDetection, }, }, }, @@ -138,8 +167,28 @@ export class OpenAIRealtimeTranscriptionSession extends EventEmitter { return; } + if (event.type === "input_audio_buffer.speech_started") { + this.emit("speech_started"); + return; + } + + if (event.type === "input_audio_buffer.speech_stopped") { + this.emit("speech_stopped"); + return; + } + + if (event.type === "conversation.item.input_audio_transcription.delta") { + const replaceDelta = this.transcriptionModel === "whisper-1"; + const prev = this.partialByItemId.get(event.item_id) ?? ""; + const next = replaceDelta ? event.delta : prev + event.delta; + this.partialByItemId.set(event.item_id, next); + this.emit("transcript", { itemId: event.item_id, transcript: next, isFinal: false }); + return; + } + if (event.type === "conversation.item.input_audio_transcription.completed") { - this.emit("transcript", { itemId: event.item_id, transcript: event.transcript }); + this.partialByItemId.set(event.item_id, event.transcript); + this.emit("transcript", { itemId: event.item_id, transcript: event.transcript, isFinal: true }); return; } diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 1336a9f05..ff38c077a 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -141,20 +141,63 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round( const DICTATION_PCM_INPUT_RATE = 16000; const DICTATION_PCM_OUTPUT_RATE = 24000; -const DICTATION_COMMIT_INTERVAL_MS = Number.parseInt( - process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ?? "5000", +const DICTATION_FINAL_TIMEOUT_MS = 120000; +const DICTATION_SILENCE_PEAK_THRESHOLD = Number.parseInt( + process.env.OPENAI_REALTIME_DICTATION_SILENCE_PEAK_THRESHOLD ?? "300", 10 ); -const DICTATION_COMMIT_TARGET_BYTES = Math.round( - (DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_COMMIT_INTERVAL_MS) / 1000 +const DICTATION_TURN_DETECTION = (process.env.OPENAI_REALTIME_DICTATION_TURN_DETECTION ?? "semantic_vad").trim(); +const DICTATION_SEMANTIC_VAD_EAGERNESS = (process.env.OPENAI_REALTIME_DICTATION_SEMANTIC_VAD_EAGERNESS ?? "medium").trim(); +const DICTATION_FLUSH_SILENCE_MS = Number.parseInt( + process.env.OPENAI_REALTIME_DICTATION_FLUSH_SILENCE_MS ?? "800", + 10 ); -const DICTATION_MIN_COMMIT_MS = 100; -const DICTATION_MIN_COMMIT_BYTES = Math.round( - (DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_MIN_COMMIT_MS) / 1000 -); -const DICTATION_FINAL_TIMEOUT_MS = 120000; const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/; +function pcm16lePeakAbs(pcm16le: Buffer): number { + if (pcm16le.length === 0) { + return 0; + } + if (pcm16le.length % 2 !== 0) { + throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`); + } + const samples = new Int16Array( + pcm16le.buffer, + pcm16le.byteOffset, + pcm16le.byteLength / 2 + ); + let peak = 0; + for (let i = 0; i < samples.length; i += 1) { + const v = samples[i]!; + const abs = v < 0 ? -v : v; + if (abs > peak) { + peak = abs; + if (peak >= 32767) { + break; + } + } + } + return peak; +} + +function parseDictationTurnDetection(): + | null + | { type: "server_vad"; create_response: false; threshold?: number; prefix_padding_ms?: number; silence_duration_ms?: number } + | { type: "semantic_vad"; create_response: false; eagerness?: "low" | "medium" | "high" } { + if (!DICTATION_TURN_DETECTION || DICTATION_TURN_DETECTION === "none" || DICTATION_TURN_DETECTION === "null") { + return null; + } + if (DICTATION_TURN_DETECTION === "server_vad") { + return { type: "server_vad", create_response: false }; + } + const eagerness = + DICTATION_SEMANTIC_VAD_EAGERNESS === "low" || + DICTATION_SEMANTIC_VAD_EAGERNESS === "high" + ? (DICTATION_SEMANTIC_VAD_EAGERNESS as "low" | "high") + : ("medium" as const); + return { type: "semantic_vad", create_response: false, eagerness }; +} + /** * Type for present_artifact tool arguments */ @@ -278,10 +321,11 @@ export class Session { nextSeqToForward: number; ackSeq: number; bytesSinceCommit: number; - expectedCommits: number; - committedCount: number; + peakSinceCommit: number; committedItemIds: string[]; transcriptsByItemId: Map; + finalTranscriptItemIds: Set; + awaitingFinalCommit: boolean; finishRequested: boolean; finalSeq: number | null; finalTimeout: ReturnType | null; @@ -1258,6 +1302,13 @@ export class Session { this.dictationStreams.delete(dictationId); } + private emitDictationPartial(dictationId: string, text: string): void { + this.emit({ + type: "dictation_stream_partial", + payload: { dictationId, text }, + }); + } + private maybeFinalizeDictationStream(dictationId: string): void { const state = this.dictationStreams.get(dictationId); if (!state) { @@ -1270,12 +1321,16 @@ export class Session { if (state.ackSeq < state.finalSeq) { return; } - if (state.committedCount < state.expectedCommits) { + if (state.awaitingFinalCommit) { + return; + } + + if (state.committedItemIds.length === 0) { return; } const allTranscriptsReady = state.committedItemIds.every((itemId) => - state.transcriptsByItemId.has(itemId) + state.finalTranscriptItemIds.has(itemId) ); if (!allTranscriptsReady) { return; @@ -1308,12 +1363,17 @@ export class Session { const transcriptionModel = process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe"; + const transcriptionPrompt = + process.env.OPENAI_REALTIME_DICTATION_TRANSCRIPTION_PROMPT ?? + "Transcribe only what the speaker says. Do not add words. Preserve punctuation and casing. If the audio is silence or non-speech noise, return an empty transcript."; const openai = new OpenAIRealtimeTranscriptionSession({ apiKey, logger: this.sessionLogger.child({ dictationId }), transcriptionModel, language: "en", + prompt: transcriptionPrompt, + turnDetection: parseDictationTurnDetection(), }); openai.on("committed", ({ itemId }: { itemId: string }) => { @@ -1321,23 +1381,46 @@ export class Session { if (!state) { return; } - state.committedCount += 1; state.committedItemIds.push(itemId); + state.bytesSinceCommit = 0; + state.peakSinceCommit = 0; + + // When finishing, we require at least one commit after finish if we flushed pending audio. + if (state.finishRequested && state.awaitingFinalCommit) { + state.awaitingFinalCommit = false; + } + this.maybeFinalizeDictationStream(dictationId); }); - openai.on("transcript", ({ itemId, transcript }: { itemId: string; transcript: string }) => { - const state = this.dictationStreams.get(dictationId); - if (!state) { - return; + openai.on( + "transcript", + ({ itemId, transcript, isFinal }: { itemId: string; transcript: string; isFinal: boolean }) => { + const state = this.dictationStreams.get(dictationId); + if (!state) { + return; + } + state.transcriptsByItemId.set(itemId, transcript); + if (isFinal) { + state.finalTranscriptItemIds.add(itemId); + } + + const orderedIds = state.committedItemIds.includes(itemId) + ? state.committedItemIds + : [...state.committedItemIds, itemId]; + const partialText = orderedIds + .map((id) => state.transcriptsByItemId.get(id) ?? "") + .join(" ") + .trim(); + this.emitDictationPartial(dictationId, partialText); + + this.maybeFinalizeDictationStream(dictationId); } - state.transcriptsByItemId.set(itemId, transcript); - this.maybeFinalizeDictationStream(dictationId); - }); + ); openai.on("error", (err) => { - const error = err instanceof Error ? err.message : String(err); - this.failDictationStream(dictationId, error, true); + const message = err instanceof Error ? err.message : String(err); + this.failDictationStream(dictationId, message, true); this.cleanupDictationStream(dictationId); }); @@ -1355,10 +1438,11 @@ export class Session { nextSeqToForward: 0, ackSeq: -1, bytesSinceCommit: 0, - expectedCommits: 0, - committedCount: 0, + peakSinceCommit: 0, committedItemIds: [], transcriptsByItemId: new Map(), + finalTranscriptItemIds: new Set(), + awaitingFinalCommit: false, finishRequested: false, finalSeq: null, finalTimeout: null, @@ -1403,20 +1487,14 @@ export class Session { if (resampled.length > 0) { state.openai.appendPcm16Base64(resampled.toString("base64")); state.bytesSinceCommit += resampled.length; + state.peakSinceCommit = Math.max(state.peakSinceCommit, pcm16lePeakAbs(resampled)); } state.nextSeqToForward += 1; state.ackSeq = state.nextSeqToForward - 1; - - if (state.bytesSinceCommit >= DICTATION_COMMIT_TARGET_BYTES) { - state.expectedCommits += 1; - state.openai.commit(); - state.bytesSinceCommit = 0; - } } this.emitDictationAck(msg.dictationId, state.ackSeq); - this.maybeFinalizeDictationStream(msg.dictationId); } private async handleDictationStreamFinish( @@ -1441,8 +1519,7 @@ export class Session { ackSeq: state.ackSeq, nextSeqToForward: state.nextSeqToForward, receivedChunks: state.receivedChunks.size, - expectedCommits: state.expectedCommits, - committedCount: state.committedCount, + bytesSinceCommit: state.bytesSinceCommit, }, "Dictation finish: no chunks received (failing fast)" ); @@ -1455,31 +1532,41 @@ export class Session { return; } - // Commit any remaining uncommitted audio. - // IMPORTANT: OpenAI requires at least ~100ms of audio in the buffer for a commit. - // Also, committing an empty buffer throws "buffer too small ... 0.00ms". + // Force-flush any remaining audio so we get a final transcription even if VAD hasn't committed yet. if (state.bytesSinceCommit > 0) { - this.sessionLogger.debug( - { dictationId: msg.dictationId, bytesSinceCommit: state.bytesSinceCommit }, - "Dictation finish: committing pending audio" - ); - if (state.bytesSinceCommit < DICTATION_MIN_COMMIT_BYTES) { - const padBytes = DICTATION_MIN_COMMIT_BYTES - state.bytesSinceCommit; + if (state.peakSinceCommit < DICTATION_SILENCE_PEAK_THRESHOLD) { this.sessionLogger.debug( - { dictationId: msg.dictationId, padBytes }, - "Dictation finish: padding to minimum commit size" + { + dictationId: msg.dictationId, + bytesSinceCommit: state.bytesSinceCommit, + peakSinceCommit: state.peakSinceCommit, + }, + "Dictation finish: clearing silence-only tail (skip final commit)" ); - state.openai.appendPcm16Base64(Buffer.alloc(padBytes).toString("base64")); - state.bytesSinceCommit += padBytes; + state.openai.clear(); + state.bytesSinceCommit = 0; + state.peakSinceCommit = 0; + state.awaitingFinalCommit = false; + } else { + // We do not manually commit for dictation (to avoid chopping words). Instead we rely on + // OpenAI semantic VAD to commit utterances. When the user presses OK, we append a short + // silence tail so VAD has enough silence to close out the final utterance promptly. + const silenceBytes = Math.max( + 0, + Math.round((DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_FLUSH_SILENCE_MS) / 1000) + ); + if (silenceBytes > 0) { + this.sessionLogger.debug( + { dictationId: msg.dictationId, silenceMs: DICTATION_FLUSH_SILENCE_MS, silenceBytes }, + "Dictation finish: appending silence tail for semantic VAD flush" + ); + state.openai.appendPcm16Base64(Buffer.alloc(silenceBytes).toString("base64")); + state.bytesSinceCommit += silenceBytes; + } + state.awaitingFinalCommit = true; } - state.expectedCommits += 1; - state.openai.commit(); - state.bytesSinceCommit = 0; } else { - this.sessionLogger.debug( - { dictationId: msg.dictationId }, - "Dictation finish: no pending audio to commit" - ); + state.awaitingFinalCommit = false; } if (state.finalTimeout) { diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index 3e6cf3ba5..1fc63feba 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -813,6 +813,14 @@ export const DictationStreamAckMessageSchema = z.object({ }), }); +export const DictationStreamPartialMessageSchema = z.object({ + type: z.literal("dictation_stream_partial"), + payload: z.object({ + dictationId: z.string(), + text: z.string(), + }), +}); + export const DictationStreamFinalMessageSchema = z.object({ type: z.literal("dictation_stream_final"), payload: z.object({ @@ -1324,6 +1332,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [ AudioOutputMessageSchema, TranscriptionResultMessageSchema, DictationStreamAckMessageSchema, + DictationStreamPartialMessageSchema, DictationStreamFinalMessageSchema, DictationStreamErrorMessageSchema, StatusMessageSchema,