diff --git a/packages/voice-assistant/src/server/agent/orchestrator.ts b/packages/voice-assistant/src/server/agent/orchestrator.ts index ecdb1e653..3316eedea 100644 --- a/packages/voice-assistant/src/server/agent/orchestrator.ts +++ b/packages/voice-assistant/src/server/agent/orchestrator.ts @@ -72,11 +72,10 @@ export async function processUserMessage(params: { // No need to broadcast again here to avoid duplication let assistantResponse = ""; + // Track pending TTS playback promise outside of streamLLM scope + let pendingTTS: Promise | null = null; try { - // Track pending TTS playback promise - let pendingTTS: Promise | null = null; - // Stream LLM response with tool execution assistantResponse = await streamLLM({ systemPrompt: getSystemPrompt(), @@ -161,20 +160,35 @@ export async function processUserMessage(params: { } }, onFinish: async () => { - // Wait for any final pending TTS to complete - if (pendingTTS) { - await pendingTTS; - pendingTTS = null; - } + // Don't wait for TTS here - we'll handle it after adding to history }, }); - // Add assistant response to context + // Add assistant response to context IMMEDIATELY after stream completes + // This ensures partial responses are saved even if TTS fails or is interrupted conversation.messages.push({ role: "assistant", content: assistantResponse, }); + + // Now wait for any pending TTS, but don't fail the entire operation if it times out + if (pendingTTS) { + try { + await pendingTTS; + } catch (ttsError) { + // TTS failed but message is already in history - just log the error + console.error("TTS playback failed (message already saved):", ttsError); + } + } } catch (error) { + // If stream itself failed or was aborted, still save any partial response to history + if (assistantResponse) { + conversation.messages.push({ + role: "assistant", + content: assistantResponse, + }); + } + // Broadcast error to WebSocket if (params.wsServer) { params.wsServer.broadcastActivityLog({ diff --git a/packages/voice-assistant/src/server/agent/tts-manager.ts b/packages/voice-assistant/src/server/agent/tts-manager.ts index cf0293c37..61005c16e 100644 --- a/packages/voice-assistant/src/server/agent/tts-manager.ts +++ b/packages/voice-assistant/src/server/agent/tts-manager.ts @@ -5,7 +5,6 @@ import type { VoiceAssistantWebSocketServer } from "../websocket-server.js"; interface PendingPlayback { resolve: () => void; reject: (error: Error) => void; - timeout: NodeJS.Timeout; } /** @@ -14,11 +13,6 @@ interface PendingPlayback { */ const pendingPlaybacks = new Map(); -/** - * Timeout for waiting on client playback confirmation (30 seconds) - */ -const PLAYBACK_TIMEOUT_MS = 30000; - /** * Generate TTS audio, broadcast to clients, and wait for playback confirmation * Returns a Promise that resolves when the client confirms playback completed @@ -35,14 +29,8 @@ export async function generateTTSAndWaitForPlayback( // Create promise that will be resolved when client confirms playback const playbackPromise = new Promise((resolve, reject) => { - // Set timeout to prevent hanging forever - const timeout = setTimeout(() => { - pendingPlaybacks.delete(audioId); - reject(new Error(`Audio playback timeout for ${audioId}`)); - }, PLAYBACK_TIMEOUT_MS); - - // Store handlers - pendingPlaybacks.set(audioId, { resolve, reject, timeout }); + // Store handlers (no timeout - will resolve when client confirms or connection closes) + pendingPlaybacks.set(audioId, { resolve, reject }); }); // Broadcast audio to clients @@ -81,8 +69,7 @@ export function confirmAudioPlayed(audioId: string): void { return; } - // Clear timeout and resolve promise - clearTimeout(pending.timeout); + // Resolve promise and cleanup pending.resolve(); pendingPlaybacks.delete(audioId); } diff --git a/packages/voice-assistant/src/ui/App.tsx b/packages/voice-assistant/src/ui/App.tsx index c97bc7354..7e5cb2f8a 100644 --- a/packages/voice-assistant/src/ui/App.tsx +++ b/packages/voice-assistant/src/ui/App.tsx @@ -248,6 +248,10 @@ function App() { const handleSendMessage = () => { if (!userInput.trim() || !ws.isConnected) return; + // Stop any currently playing audio and clear the queue + audioPlayerRef.current.stop(); + setIsPlayingAudio(false); + // Send message to server ws.sendUserMessage(userInput); @@ -292,6 +296,11 @@ function App() { console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`); } else { console.log('[App] Starting recording...'); + + // Stop any currently playing audio when starting to record + audioPlayerRef.current.stop(); + setIsPlayingAudio(false); + await recorder.start(); setIsRecording(true); } diff --git a/packages/voice-assistant/src/ui/lib/audio-playback.ts b/packages/voice-assistant/src/ui/lib/audio-playback.ts index 5862220b2..98f237b73 100644 --- a/packages/voice-assistant/src/ui/lib/audio-playback.ts +++ b/packages/voice-assistant/src/ui/lib/audio-playback.ts @@ -16,6 +16,7 @@ export function createAudioPlayer(): AudioPlayer { let playing = false; let queue: QueuedAudio[] = []; let isProcessingQueue = false; + let currentReject: ((error: Error) => void) | null = null; async function play(audioData: Blob): Promise { return new Promise((resolve, reject) => { @@ -51,6 +52,9 @@ export function createAudioPlayer(): AudioPlayer { async function playAudio(audioData: Blob): Promise { return new Promise((resolve, reject) => { + // Store reject handler so stop() can properly cancel current playback + currentReject = reject; + try { // Create blob URL const audioUrl = URL.createObjectURL(audioData); @@ -71,6 +75,7 @@ export function createAudioPlayer(): AudioPlayer { ); playing = false; currentAudio = null; + currentReject = null; // Clean up blob URL URL.revokeObjectURL(audioUrl); @@ -82,6 +87,7 @@ export function createAudioPlayer(): AudioPlayer { console.error("[AudioPlayer] Playback error:", error); playing = false; currentAudio = null; + currentReject = null; // Clean up blob URL URL.revokeObjectURL(audioUrl); @@ -94,6 +100,7 @@ export function createAudioPlayer(): AudioPlayer { console.error("[AudioPlayer] Failed to start playback:", error); playing = false; currentAudio = null; + currentReject = null; URL.revokeObjectURL(audioUrl); reject(error); }); @@ -101,19 +108,29 @@ export function createAudioPlayer(): AudioPlayer { console.error("[AudioPlayer] Error creating audio element:", error); playing = false; currentAudio = null; + currentReject = null; reject(error); } }); } function stop(): void { + // Stop and reject currently playing audio if (currentAudio) { currentAudio.pause(); currentAudio.currentTime = 0; + // Remove src to fully stop the audio + currentAudio.src = ""; currentAudio = null; } playing = false; + // Reject the current playing promise if it exists + if (currentReject) { + currentReject(new Error("Playback stopped")); + currentReject = null; + } + // Reject all pending promises in the queue while (queue.length > 0) { const item = queue.shift()!;