Add barge-in telemetry

packages/app/src/contexts/realtime-context.tsx

@@ -1,4 +1,4 @@
-import { createContext, useContext, useState, ReactNode, useCallback, useEffect } from "react";
+import { createContext, useContext, useState, ReactNode, useCallback, useEffect, useRef } from "react";
 import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
 import { useSession } from "./session-context";
 import { generateMessageId } from "@/types/stream";
@@ -39,12 +39,16 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
     setVoiceDetectionFlags,
   } = useSession();
   const [isRealtimeMode, setIsRealtimeMode] = useState(false);
+  const bargeInPlaybackStopRef = useRef<number | null>(null);
 
   const realtimeAudio = useSpeechmaticsAudio({
     onSpeechStart: () => {
       console.log("[Realtime] Speech detected");
       // Stop audio playback if playing
       if (isPlayingAudio) {
+        if (bargeInPlaybackStopRef.current === null) {
+          bargeInPlaybackStopRef.current = Date.now();
+        }
         audioPlayer.stop();
       }
 
@@ -112,6 +116,18 @@ export function RealtimeProvider({ children }: RealtimeProviderProps) {
     setVoiceDetectionFlags(realtimeAudio.isDetecting, realtimeAudio.isSpeaking);
   }, [realtimeAudio.isDetecting, realtimeAudio.isSpeaking, setVoiceDetectionFlags]);
 
+  useEffect(() => {
+    if (!isPlayingAudio && bargeInPlaybackStopRef.current !== null) {
+      const latencyMs = Date.now() - bargeInPlaybackStopRef.current;
+      console.log("[Telemetry] barge_in.playback_stop_latency", {
+        latencyMs,
+        startedAt: new Date(bargeInPlaybackStopRef.current).toISOString(),
+        completedAt: new Date().toISOString(),
+      });
+      bargeInPlaybackStopRef.current = null;
+    }
+  }, [isPlayingAudio]);
+
   const startRealtime = useCallback(async () => {
     try {
       await realtimeAudio.start();

packages/server/src/server/session.ts

@@ -2067,6 +2067,10 @@ export class Session {
       return;
     }
 
+    const chunkReceivedAt = Date.now();
+    const phaseBeforeAbort = this.processingPhase;
+    const hadActiveStream = Boolean(this.currentStreamPromise);
+
     this.speechInProgress = true;
     console.log(
       `[Session ${this.clientId}] Realtime speech chunk detected – aborting playback and LLM`
@@ -2091,6 +2095,15 @@ export class Session {
 
     this.abortController.abort();
     await this.handleAbort();
+
+    const latencyMs = Date.now() - chunkReceivedAt;
+    console.log("[Telemetry] barge_in.llm_abort_latency", {
+      latencyMs,
+      conversationId: this.conversationId,
+      phaseBeforeAbort,
+      hadActiveStream,
+      timestamp: new Date().toISOString(),
+    });
   }
 
   /**

plan.md

@@ -72,7 +72,8 @@
   - Captured the current audio pipeline, design goals, candidate approaches (self-hosted Whisper vs managed APIs), and production requirements (transport, VAD, streaming STT engine, infra, observability, compliance) in `docs/streaming-stt-vad.md`, along with an implementation phase plan and open questions for the follow-up agent.
 - [x] Prototype streaming TTS output (chunked `audio_output` messages) and update the player to start playback as soon as chunks arrive.
   - Split OpenAI TTS responses into chunked `audio_output` payloads via a streaming `TTSManager` pipeline (`packages/server/src/server/agent/tts-manager.ts`, `packages/server/src/server/agent/tts-openai.ts`) that tracks per-chunk acknowledgements, adds utterance IDs, and resolves once every chunk finishes. The app’s session context now keeps audio groups active until the last chunk (or an error) lands so playback begins as soon as each chunk arrives without waiting for the full clip (`packages/app/src/contexts/session-context.tsx`). `npm run typecheck --workspace=@voice-dev/server` and `npm run typecheck --workspace=@voice-dev/app` still fail in the pre-existing cross-workspace stream harness files (see earlier tasks); no new regressions were introduced.
-- [ ] Add telemetry for barge-in latency (speech start → playback stop, chunk arrival → LLM abort) to measure improvements per milestone.
+- [x] Add telemetry for barge-in latency (speech start → playback stop, chunk arrival → LLM abort) to measure improvements per milestone.
+  - `RealtimeProvider` now records when speech detection interrupts active playback and logs `[Telemetry] barge_in.playback_stop_latency` once the app reports audio stopped, while the server's `Session.handleRealtimeSpeechStart` logs `[Telemetry] barge_in.llm_abort_latency` after aborting live LLM runs—giving us measurable client + server latency metrics per turn.
 - [ ] Codex tool calls aren’t visible in the live stream (they only show up after hydrating/refreshing); fix the reducer/rendering path so Codex tool events appear in real time the same way Claude’s do.
 - [ ] Update the AgentInput “Cancel” control to use a square stop icon button (consistent with media players) so interrupting an agent is visually distinct from other actions.
 - [ ] When the agent input has text and we’re not in the cancellable state, replace both the mic (Dictate) and Realtime buttons with a single Send button—only surface Dictate/Realtime when the input is empty or the agent is running.