diff --git a/packages/app/src/components/dictation-controls.tsx b/packages/app/src/components/dictation-controls.tsx index a0a0f42ad..5ef241307 100644 --- a/packages/app/src/components/dictation-controls.tsx +++ b/packages/app/src/components/dictation-controls.tsx @@ -136,12 +136,13 @@ export function DictationOverlay({ isRecording, isProcessing, status, + errorText, onCancel, onAccept, onAcceptAndSend, onRetry, onDiscard, -}: Omit) { +}: Omit & { errorText?: string }) { const { theme } = useUnistyles(); const isFailed = status === "failed"; const showActiveState = isRecording || isProcessing || isFailed; @@ -199,6 +200,17 @@ export function DictationOverlay({ {transcript} )} + {isFailed ? ( + + {errorText ? `Dictation failed: ${errorText}` : "Dictation failed. Tap retry."} + + ) : null} diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx index e9fce0891..23a431cdd 100644 --- a/packages/app/src/components/message-input.tsx +++ b/packages/app/src/components/message-input.tsx @@ -199,6 +199,7 @@ export const MessageInput = forwardRef( partialTranscript: dictationPartialTranscript, volume: dictationVolume, duration: dictationDuration, + error: dictationError, status: dictationStatus, startDictation, cancelDictation, @@ -631,6 +632,7 @@ export const MessageInput = forwardRef( isRecording={isDictating} isProcessing={isDictationProcessing} status={dictationStatus} + errorText={dictationStatus === "failed" ? dictationError ?? undefined : undefined} onCancel={handleCancelRecording} onAccept={handleAcceptRecording} onAcceptAndSend={handleAcceptAndSendRecording} diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index 0a532fdd5..0001f11e0 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -200,6 +200,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const handleStreamingTranscriptionSuccess = useCallback( (text: string, requestId: string) => { setIsProcessing(false); + isProcessingRef.current = false; setPartialTranscript(""); setDuration(0); setStatus("idle"); @@ -220,6 +221,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { const normalized = toError(failure); const failureId = generateMessageId(); setIsProcessing(false); + isProcessingRef.current = false; isRecordingRef.current = false; setIsRecording(false); @@ -447,9 +449,14 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { stopDurationTracking(); setDuration(0); setIsProcessing(false); + isProcessingRef.current = false; setError(null); - setStatus("idle"); - clearStreamingState(); + if (senderRef.current?.hasSegments()) { + setStatus("failed"); + } else { + setStatus("idle"); + clearStreamingState(); + } } } }, [autoStopWhenHidden?.isVisible, clearStreamingState, stopDurationTracking]); diff --git a/packages/server/.env.example b/packages/server/.env.example index 174d94e4a..e2a088b38 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -17,3 +17,7 @@ PASEO_PORT=6767 # Legacy web server port used by the legacy Express app PORT=3000 NODE_ENV=development + +# Debug recordings (dictation + STT input + TTS output) +# When enabled, recordings are saved under `${cwd}/.debug/recordings/` +PASEO_DICTATION_DEBUG=1 diff --git a/packages/server/src/client/daemon-client-v2.test.ts b/packages/server/src/client/daemon-client-v2.test.ts index f5eb347fa..088a529ce 100644 --- a/packages/server/src/client/daemon-client-v2.test.ts +++ b/packages/server/src/client/daemon-client-v2.test.ts @@ -102,6 +102,7 @@ describe("DaemonClientV2", () => { baseRef: null, aheadBehind: null, hasRemote: false, + remoteUrl: null, }, }, }; diff --git a/packages/server/src/client/daemon-client-v2.ts b/packages/server/src/client/daemon-client-v2.ts index 2305b64c7..980abc0a7 100644 --- a/packages/server/src/client/daemon-client-v2.ts +++ b/packages/server/src/client/daemon-client-v2.ts @@ -999,7 +999,7 @@ export class DaemonClientV2 { } return msg.payload; }, - 120000, + 30000, { skipQueue: true } ); @@ -1013,7 +1013,7 @@ export class DaemonClientV2 { } return msg.payload; }, - 120000, + 30000, { skipQueue: true } ).then((payload) => { throw new Error(payload.error); diff --git a/packages/server/src/server/agent/dictation-debug.ts b/packages/server/src/server/agent/dictation-debug.ts new file mode 100644 index 000000000..60f8a2647 --- /dev/null +++ b/packages/server/src/server/agent/dictation-debug.ts @@ -0,0 +1,39 @@ +import type pino from "pino"; +import { mkdir, writeFile } from "fs/promises"; +import { join } from "path"; +import { inferAudioExtension, sanitizeForFilename } from "./audio-utils.js"; +import { resolveRecordingsDebugDir } from "./recordings-debug.js"; + +const debugDir = resolveRecordingsDebugDir("DICTATION_DEBUG_AUDIO_DIR"); +let announced = false; + +export interface DictationDebugAudioMetadata { + sessionId: string; + dictationId: string; + format: string; +} + +export async function maybePersistDictationDebugAudio( + audio: Buffer, + metadata: DictationDebugAudioMetadata, + logger: pino.Logger +): Promise { + if (!debugDir) { + return null; + } + + if (!announced) { + logger.info({ debugDir }, "Dictation audio capture enabled"); + announced = true; + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const folder = join(debugDir, sanitizeForFilename(metadata.sessionId, "session")); + await mkdir(folder, { recursive: true }); + + const parts = [timestamp, sanitizeForFilename(metadata.dictationId, "dictation")]; + const ext = inferAudioExtension(metadata.format); + const filePath = join(folder, `${parts.join("_")}.${ext}`); + await writeFile(filePath, audio); + return filePath; +} diff --git a/packages/server/src/server/agent/recordings-debug.ts b/packages/server/src/server/agent/recordings-debug.ts new file mode 100644 index 000000000..aa83ef091 --- /dev/null +++ b/packages/server/src/server/agent/recordings-debug.ts @@ -0,0 +1,24 @@ +import { resolve } from "path"; + +function isTruthyEnv(value: string | undefined): boolean { + const normalized = (value ?? "").trim().toLowerCase(); + return normalized === "1" || normalized === "true" || normalized === "yes" || normalized === "on"; +} + +export function isPaseoDictationDebugEnabled(): boolean { + return isTruthyEnv(process.env.PASEO_DICTATION_DEBUG); +} + +export function resolveRecordingsDebugDir(explicitEnvVarName: string): string | null { + const explicit = process.env[explicitEnvVarName]; + if (explicit && explicit.trim()) { + return resolve(explicit.trim()); + } + + if (!isPaseoDictationDebugEnabled()) { + return null; + } + + return resolve(process.cwd(), ".debug/recordings"); +} + diff --git a/packages/server/src/server/agent/stt-debug.ts b/packages/server/src/server/agent/stt-debug.ts index a3c50bac9..146f06fa7 100644 --- a/packages/server/src/server/agent/stt-debug.ts +++ b/packages/server/src/server/agent/stt-debug.ts @@ -1,11 +1,10 @@ import type pino from "pino"; import { mkdir, writeFile } from "fs/promises"; -import { join, resolve } from "path"; +import { join } from "path"; import { inferAudioExtension, sanitizeForFilename } from "./audio-utils.js"; +import { resolveRecordingsDebugDir } from "./recordings-debug.js"; -const debugDir = process.env.STT_DEBUG_AUDIO_DIR - ? resolve(process.env.STT_DEBUG_AUDIO_DIR) - : null; +const debugDir = resolveRecordingsDebugDir("STT_DEBUG_AUDIO_DIR"); let announced = false; export interface DebugAudioMetadata { diff --git a/packages/server/src/server/agent/tts-debug.ts b/packages/server/src/server/agent/tts-debug.ts new file mode 100644 index 000000000..49cfcb9b7 --- /dev/null +++ b/packages/server/src/server/agent/tts-debug.ts @@ -0,0 +1,39 @@ +import type pino from "pino"; +import { mkdir, writeFile } from "fs/promises"; +import { join } from "path"; +import { inferAudioExtension, sanitizeForFilename } from "./audio-utils.js"; +import { resolveRecordingsDebugDir } from "./recordings-debug.js"; + +const debugDir = resolveRecordingsDebugDir("TTS_DEBUG_AUDIO_DIR"); +let announced = false; + +export interface TtsDebugAudioMetadata { + sessionId: string; + groupId: string; + format: string; +} + +export async function maybePersistTtsDebugAudio( + audio: Buffer, + metadata: TtsDebugAudioMetadata, + logger: pino.Logger +): Promise { + if (!debugDir) { + return null; + } + + if (!announced) { + logger.info({ debugDir }, "TTS audio capture enabled"); + announced = true; + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const folder = join(debugDir, sanitizeForFilename(metadata.sessionId, "session")); + await mkdir(folder, { recursive: true }); + + const parts = [timestamp, sanitizeForFilename(metadata.groupId, "tts")]; + const ext = inferAudioExtension(metadata.format); + const filePath = join(folder, `${parts.join("_")}.${ext}`); + await writeFile(filePath, audio); + return filePath; +} diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index dd56d5a5b..fecc765c6 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -37,6 +37,9 @@ import type { OpenAISTT } from "./agent/stt-openai.js"; import type { OpenAITTS } from "./agent/tts-openai.js"; import { OpenAIRealtimeTranscriptionSession } from "./agent/openai-realtime-transcription.js"; import { Pcm16MonoResampler } from "./agent/pcm16-resampler.js"; +import { maybePersistDictationDebugAudio } from "./agent/dictation-debug.js"; +import { maybePersistTtsDebugAudio } from "./agent/tts-debug.js"; +import { isPaseoDictationDebugEnabled } from "./agent/recordings-debug.js"; import type { VoiceConversationStore } from "./voice-conversation-store.js"; import { buildConfigOverrides, @@ -145,7 +148,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round( const DICTATION_PCM_INPUT_RATE = 16000; const DICTATION_PCM_OUTPUT_RATE = 24000; -const DICTATION_FINAL_TIMEOUT_MS = 120000; +const DICTATION_FINAL_TIMEOUT_MS = 30000; const DICTATION_SILENCE_PEAK_THRESHOLD = Number.parseInt( process.env.OPENAI_REALTIME_DICTATION_SILENCE_PEAK_THRESHOLD ?? "300", 10 @@ -322,6 +325,8 @@ export class Session { inputFormat: string; openai: OpenAIRealtimeTranscriptionSession; resampler: Pcm16MonoResampler; + debugAudioChunks: Buffer[]; + debugRecordingPath: string | null; receivedChunks: Map; nextSeqToForward: number; ackSeq: number; @@ -342,6 +347,12 @@ export class Session { private bufferTimeout: NodeJS.Timeout | null = null; private audioBuffer: AudioBufferState | null = null; + // Optional TTS debug capture (persisted per utterance) + private readonly ttsDebugStreams = new Map< + string, + { format: string; chunks: Buffer[] } + >(); + // Conversation history private messages: ModelMessage[] = []; private turnIndex = 0; @@ -1299,6 +1310,65 @@ export class Session { }); } + private async maybePersistDictationStreamAudio(dictationId: string): Promise { + const state = this.dictationStreams.get(dictationId) ?? null; + if (!state) { + return null; + } + if (state.debugRecordingPath) { + return state.debugRecordingPath; + } + if (state.debugAudioChunks.length === 0) { + return null; + } + + const pcmBuffer = Buffer.concat(state.debugAudioChunks); + const wavBuffer = convertPCMToWavBuffer( + pcmBuffer, + DICTATION_PCM_OUTPUT_RATE, + PCM_CHANNELS, + PCM_BITS_PER_SAMPLE + ); + + const path = await maybePersistDictationDebugAudio( + wavBuffer, + { sessionId: this.sessionId, dictationId, format: "audio/wav" }, + this.sessionLogger + ); + state.debugRecordingPath = path; + return path; + } + + private async failAndCleanupDictationStream( + dictationId: string, + error: string, + retryable: boolean + ): Promise { + const debugRecordingPath = await this.maybePersistDictationStreamAudio(dictationId); + this.emit({ + type: "dictation_stream_error", + payload: { + dictationId, + error, + retryable, + ...(debugRecordingPath ? { debugRecordingPath } : {}), + }, + }); + if (debugRecordingPath) { + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "system", + content: `Saved dictation audio: ${debugRecordingPath}`, + metadata: { recordingPath: debugRecordingPath, dictationId }, + }, + }); + } + this.cleanupDictationStream(dictationId); + } + private cleanupDictationStream(dictationId: string): void { const state = this.dictationStreams.get(dictationId) ?? null; if (!state) { @@ -1354,12 +1424,30 @@ export class Session { .join(" ") .trim(); - this.emit({ - type: "dictation_stream_final", - payload: { dictationId, text: orderedText }, - }); - - this.cleanupDictationStream(dictationId); + void (async () => { + const debugRecordingPath = await this.maybePersistDictationStreamAudio(dictationId); + this.emit({ + type: "dictation_stream_final", + payload: { + dictationId, + text: orderedText, + ...(debugRecordingPath ? { debugRecordingPath } : {}), + }, + }); + if (debugRecordingPath) { + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "system", + content: `Saved dictation audio: ${debugRecordingPath}`, + metadata: { recordingPath: debugRecordingPath, dictationId }, + }, + }); + } + this.cleanupDictationStream(dictationId); + })(); } private async handleDictationStreamStart( @@ -1433,8 +1521,7 @@ export class Session { openai.on("error", (err) => { const message = err instanceof Error ? err.message : String(err); - this.failDictationStream(dictationId, message, true); - this.cleanupDictationStream(dictationId); + void this.failAndCleanupDictationStream(dictationId, message, true); }); await openai.connect(); @@ -1447,6 +1534,8 @@ export class Session { inputRate: DICTATION_PCM_INPUT_RATE, outputRate: DICTATION_PCM_OUTPUT_RATE, }), + debugAudioChunks: [], + debugRecordingPath: null, receivedChunks: new Map(), nextSeqToForward: 0, ackSeq: -1, @@ -1474,7 +1563,7 @@ export class Session { } if (msg.format !== state.inputFormat) { - this.failDictationStream( + void this.failAndCleanupDictationStream( msg.dictationId, `Mismatched dictation stream format: ${msg.format}`, false @@ -1499,6 +1588,7 @@ export class Session { const resampled = state.resampler.processChunk(pcm16); if (resampled.length > 0) { state.openai.appendPcm16Base64(resampled.toString("base64")); + state.debugAudioChunks.push(resampled); state.bytesSinceCommit += resampled.length; state.peakSinceCommit = Math.max(state.peakSinceCommit, pcm16lePeakAbs(resampled)); } @@ -1573,7 +1663,9 @@ export class Session { { dictationId: msg.dictationId, silenceMs: DICTATION_FLUSH_SILENCE_MS, silenceBytes }, "Dictation finish: appending silence tail for semantic VAD flush" ); - state.openai.appendPcm16Base64(Buffer.alloc(silenceBytes).toString("base64")); + const silence = Buffer.alloc(silenceBytes); + state.openai.appendPcm16Base64(silence.toString("base64")); + state.debugAudioChunks.push(silence); state.bytesSinceCommit += silenceBytes; } state.awaitingFinalCommit = true; @@ -1586,12 +1678,11 @@ export class Session { clearTimeout(state.finalTimeout); } state.finalTimeout = setTimeout(() => { - this.failDictationStream( + void this.failAndCleanupDictationStream( msg.dictationId, "Timed out waiting for final transcription", true ); - this.cleanupDictationStream(msg.dictationId); }, DICTATION_FINAL_TIMEOUT_MS); this.maybeFinalizeDictationStream(msg.dictationId); @@ -4109,7 +4200,9 @@ export class Session { }); try { + const requestId = uuidv4(); const result = await this.sttManager.transcribe(audio, format, { + requestId, label: this.isRealtimeMode ? "realtime" : "buffered", }); @@ -4137,7 +4230,7 @@ export class Session { text: result.text, language: result.language, duration: result.duration, - requestId: uuidv4(), + requestId, avgLogprob: result.avgLogprob, isLowConfidence: result.isLowConfidence, byteLength: result.byteLength, @@ -4146,6 +4239,23 @@ export class Session { }, }); + if (result.debugRecordingPath) { + this.emit({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "system", + content: `Saved input audio: ${result.debugRecordingPath}`, + metadata: { + recordingPath: result.debugRecordingPath, + format: result.format, + requestId, + }, + }, + }); + } + // Emit activity log this.emit({ type: "activity_log", @@ -4657,6 +4767,7 @@ export class Session { private createAbortController(): AbortController { this.abortController.abort(); this.abortController = new AbortController(); + this.ttsDebugStreams.clear(); return this.abortController; } @@ -4702,6 +4813,54 @@ export class Session { * Emit a message to the client */ private emit(msg: SessionOutboundMessage): void { + if ( + msg.type === "audio_output" && + (process.env.TTS_DEBUG_AUDIO_DIR || isPaseoDictationDebugEnabled()) && + msg.payload.groupId && + typeof msg.payload.audio === "string" + ) { + const groupId = msg.payload.groupId; + const existing = + this.ttsDebugStreams.get(groupId) ?? + ({ format: msg.payload.format, chunks: [] } satisfies { + format: string; + chunks: Buffer[]; + }); + + try { + existing.chunks.push(Buffer.from(msg.payload.audio, "base64")); + existing.format = msg.payload.format; + this.ttsDebugStreams.set(groupId, existing); + } catch { + // ignore malformed base64 + } + + if (msg.payload.isLastChunk) { + const final = this.ttsDebugStreams.get(groupId); + this.ttsDebugStreams.delete(groupId); + if (final && final.chunks.length > 0) { + void (async () => { + const recordingPath = await maybePersistTtsDebugAudio( + Buffer.concat(final.chunks), + { sessionId: this.sessionId, groupId, format: final.format }, + this.sessionLogger + ); + if (recordingPath) { + this.onMessage({ + type: "activity_log", + payload: { + id: uuidv4(), + timestamp: new Date(), + type: "system", + content: `Saved TTS audio: ${recordingPath}`, + metadata: { recordingPath, format: final.format, groupId }, + }, + }); + } + })(); + } + } + } this.onMessage(msg); } diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index 57fc61a2a..a59258f64 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -842,6 +842,7 @@ export const DictationStreamFinalMessageSchema = z.object({ payload: z.object({ dictationId: z.string(), text: z.string(), + debugRecordingPath: z.string().optional(), }), }); @@ -851,6 +852,7 @@ export const DictationStreamErrorMessageSchema = z.object({ dictationId: z.string(), error: z.string(), retryable: z.boolean(), + debugRecordingPath: z.string().optional(), }), });