```

fix: improve audio playback interruption and response persistence

- Remove TTS timeout to prevent premature playback cancellation
- Save assistant responses to history before waiting for TTS completion
- Stop audio playback when sending message or starting recording
- Properly reject current audio promise when stopping playback
- Ensure partial responses are saved even if stream is aborted
```

packages/voice-assistant/src/server/agent/orchestrator.ts

@@ -72,11 +72,10 @@ export async function processUserMessage(params: {
   // No need to broadcast again here to avoid duplication
 
   let assistantResponse = "";
+  // Track pending TTS playback promise outside of streamLLM scope
+  let pendingTTS: Promise<void> | null = null;
 
   try {
-    // Track pending TTS playback promise
-    let pendingTTS: Promise<void> | null = null;
-
     // Stream LLM response with tool execution
     assistantResponse = await streamLLM({
       systemPrompt: getSystemPrompt(),
@@ -161,20 +160,35 @@ export async function processUserMessage(params: {
         }
       },
       onFinish: async () => {
-        // Wait for any final pending TTS to complete
-        if (pendingTTS) {
-          await pendingTTS;
-          pendingTTS = null;
-        }
+        // Don't wait for TTS here - we'll handle it after adding to history
       },
     });
 
-    // Add assistant response to context
+    // Add assistant response to context IMMEDIATELY after stream completes
+    // This ensures partial responses are saved even if TTS fails or is interrupted
     conversation.messages.push({
       role: "assistant",
       content: assistantResponse,
     });
+
+    // Now wait for any pending TTS, but don't fail the entire operation if it times out
+    if (pendingTTS) {
+      try {
+        await pendingTTS;
+      } catch (ttsError) {
+        // TTS failed but message is already in history - just log the error
+        console.error("TTS playback failed (message already saved):", ttsError);
+      }
+    }
   } catch (error) {
+    // If stream itself failed or was aborted, still save any partial response to history
+    if (assistantResponse) {
+      conversation.messages.push({
+        role: "assistant",
+        content: assistantResponse,
+      });
+    }
+
     // Broadcast error to WebSocket
     if (params.wsServer) {
       params.wsServer.broadcastActivityLog({

packages/voice-assistant/src/server/agent/tts-manager.ts

@@ -5,7 +5,6 @@ import type { VoiceAssistantWebSocketServer } from "../websocket-server.js";
 interface PendingPlayback {
   resolve: () => void;
   reject: (error: Error) => void;
-  timeout: NodeJS.Timeout;
 }
 
 /**
@@ -14,11 +13,6 @@ interface PendingPlayback {
  */
 const pendingPlaybacks = new Map<string, PendingPlayback>();
 
-/**
- * Timeout for waiting on client playback confirmation (30 seconds)
- */
-const PLAYBACK_TIMEOUT_MS = 30000;
-
 /**
  * Generate TTS audio, broadcast to clients, and wait for playback confirmation
  * Returns a Promise that resolves when the client confirms playback completed
@@ -35,14 +29,8 @@ export async function generateTTSAndWaitForPlayback(
 
   // Create promise that will be resolved when client confirms playback
   const playbackPromise = new Promise<void>((resolve, reject) => {
-    // Set timeout to prevent hanging forever
-    const timeout = setTimeout(() => {
-      pendingPlaybacks.delete(audioId);
-      reject(new Error(`Audio playback timeout for ${audioId}`));
-    }, PLAYBACK_TIMEOUT_MS);
-
-    // Store handlers
-    pendingPlaybacks.set(audioId, { resolve, reject, timeout });
+    // Store handlers (no timeout - will resolve when client confirms or connection closes)
+    pendingPlaybacks.set(audioId, { resolve, reject });
   });
 
   // Broadcast audio to clients
@@ -81,8 +69,7 @@ export function confirmAudioPlayed(audioId: string): void {
     return;
   }
 
-  // Clear timeout and resolve promise
-  clearTimeout(pending.timeout);
+  // Resolve promise and cleanup
   pending.resolve();
   pendingPlaybacks.delete(audioId);
 }

packages/voice-assistant/src/ui/App.tsx

@@ -248,6 +248,10 @@ function App() {
   const handleSendMessage = () => {
     if (!userInput.trim() || !ws.isConnected) return;
 
+    // Stop any currently playing audio and clear the queue
+    audioPlayerRef.current.stop();
+    setIsPlayingAudio(false);
+
     // Send message to server
     ws.sendUserMessage(userInput);
 
@@ -292,6 +296,11 @@ function App() {
         console.log(`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`);
       } else {
         console.log('[App] Starting recording...');
+
+        // Stop any currently playing audio when starting to record
+        audioPlayerRef.current.stop();
+        setIsPlayingAudio(false);
+
         await recorder.start();
         setIsRecording(true);
       }

packages/voice-assistant/src/ui/lib/audio-playback.ts

@@ -16,6 +16,7 @@ export function createAudioPlayer(): AudioPlayer {
   let playing = false;
   let queue: QueuedAudio[] = [];
   let isProcessingQueue = false;
+  let currentReject: ((error: Error) => void) | null = null;
 
   async function play(audioData: Blob): Promise<number> {
     return new Promise((resolve, reject) => {
@@ -51,6 +52,9 @@ export function createAudioPlayer(): AudioPlayer {
 
   async function playAudio(audioData: Blob): Promise<number> {
     return new Promise((resolve, reject) => {
+      // Store reject handler so stop() can properly cancel current playback
+      currentReject = reject;
+
       try {
         // Create blob URL
         const audioUrl = URL.createObjectURL(audioData);
@@ -71,6 +75,7 @@ export function createAudioPlayer(): AudioPlayer {
           );
           playing = false;
           currentAudio = null;
+          currentReject = null;
 
           // Clean up blob URL
           URL.revokeObjectURL(audioUrl);
@@ -82,6 +87,7 @@ export function createAudioPlayer(): AudioPlayer {
           console.error("[AudioPlayer] Playback error:", error);
           playing = false;
           currentAudio = null;
+          currentReject = null;
 
           // Clean up blob URL
           URL.revokeObjectURL(audioUrl);
@@ -94,6 +100,7 @@ export function createAudioPlayer(): AudioPlayer {
           console.error("[AudioPlayer] Failed to start playback:", error);
           playing = false;
           currentAudio = null;
+          currentReject = null;
           URL.revokeObjectURL(audioUrl);
           reject(error);
         });
@@ -101,19 +108,29 @@ export function createAudioPlayer(): AudioPlayer {
         console.error("[AudioPlayer] Error creating audio element:", error);
         playing = false;
         currentAudio = null;
+        currentReject = null;
         reject(error);
       }
     });
   }
 
   function stop(): void {
+    // Stop and reject currently playing audio
     if (currentAudio) {
       currentAudio.pause();
       currentAudio.currentTime = 0;
+      // Remove src to fully stop the audio
+      currentAudio.src = "";
       currentAudio = null;
     }
     playing = false;
 
+    // Reject the current playing promise if it exists
+    if (currentReject) {
+      currentReject(new Error("Playback stopped"));
+      currentReject = null;
+    }
+
     // Reject all pending promises in the queue
     while (queue.length > 0) {
       const item = queue.shift()!;