Merge pull request #10 from boudra/web-voice-dictation-fix

Fix web voice mode; share VAD/segmenting

packages/app/src/components/git-diff-pane.tsx

@@ -16,7 +16,7 @@ import { ScrollView, type ScrollView as ScrollViewType } from "react-native-gest
 import { StyleSheet, useUnistyles } from "react-native-unistyles";
 import AsyncStorage from "@react-native-async-storage/async-storage";
 import * as Linking from "expo-linking";
-import { ChevronDown, ChevronRight, GitBranch, MoreVertical, ArrowLeftRight } from "lucide-react-native";
+import { ChevronDown, ChevronRight, GitBranch, MoreVertical, ArrowLeftRight, ListChevronsDownUp, ListChevronsUpDown } from "lucide-react-native";
 import { useMutation, useQueryClient } from "@tanstack/react-query";
 import { useSessionStore } from "@/stores/session-store";
 import {
@@ -583,6 +583,23 @@ export function GitDiffPane({ serverId, agentId, cwd }: GitDiffPaneProps) {
     }));
   }, []);
 
+  const allExpanded = useMemo(() => {
+    if (files.length === 0) return false;
+    return files.every((file) => expandedByPath[file.path]);
+  }, [files, expandedByPath]);
+
+  const handleToggleExpandAll = useCallback(() => {
+    if (allExpanded) {
+      setExpandedByPath({});
+    } else {
+      const newExpanded: Record<string, boolean> = {};
+      for (const file of files) {
+        newExpanded[file.path] = true;
+      }
+      setExpandedByPath(newExpanded);
+    }
+  }, [allExpanded, files]);
+
   // Reset manual refresh flag when fetch completes
   useEffect(() => {
     if (!(isDiffFetching || isStatusFetching) && isManualRefresh) {
@@ -1157,27 +1174,42 @@ export function GitDiffPane({ serverId, agentId, cwd }: GitDiffPaneProps) {
 
       {isGit && (hasUncommittedChanges || aheadCount > 0) ? (
         <View style={styles.diffStatusContainer}>
-          <Pressable
-            style={({ hovered, pressed }) => [
-              styles.diffStatusRow,
-              (hovered || pressed) && styles.diffStatusRowHovered,
-            ]}
-            testID="changes-diff-status"
-            onPress={() => setDiffModeOverride(diffMode === "uncommitted" ? "base" : "uncommitted")}
-          >
-            {({ hovered, pressed }) => (
-              <>
-                <Text style={styles.diffStatusText}>
-                  {diffMode === "uncommitted" ? "Uncommitted" : "Committed"}
-                </Text>
-                <ArrowLeftRight
-                  size={12}
-                  color={theme.colors.foregroundMuted}
-                  style={hovered || pressed ? undefined : styles.diffStatusIconHidden}
-                />
-              </>
+          <View style={styles.diffStatusInner}>
+            <Pressable
+              style={({ hovered, pressed }) => [
+                styles.diffStatusRow,
+                (hovered || pressed) && styles.diffStatusRowHovered,
+              ]}
+              testID="changes-diff-status"
+              onPress={() => setDiffModeOverride(diffMode === "uncommitted" ? "base" : "uncommitted")}
+            >
+              {({ hovered, pressed }) => (
+                <>
+                  <Text style={styles.diffStatusText}>
+                    {diffMode === "uncommitted" ? "Uncommitted" : "Committed"}
+                  </Text>
+                  {(hovered || pressed) ? (
+                    <ArrowLeftRight size={12} color={theme.colors.foregroundMuted} />
+                  ) : null}
+                </>
             )}
-          </Pressable>
+            </Pressable>
+            {files.length > 0 ? (
+              <Pressable
+                style={({ hovered, pressed }) => [
+                  styles.expandAllButton,
+                  (hovered || pressed) && styles.diffStatusRowHovered,
+                ]}
+                onPress={handleToggleExpandAll}
+              >
+                {allExpanded ? (
+                  <ListChevronsDownUp size={14} color={theme.colors.foregroundMuted} />
+                ) : (
+                  <ListChevronsUpDown size={14} color={theme.colors.foregroundMuted} />
+                )}
+              </Pressable>
+            ) : null}
+          </View>
         </View>
       ) : null}
 
@@ -1229,15 +1261,20 @@ const styles = StyleSheet.create((theme) => ({
     borderBottomWidth: 1,
     borderBottomColor: theme.colors.border,
   },
+  diffStatusInner: {
+    flexDirection: "row",
+    alignItems: "center",
+    justifyContent: "space-between",
+    paddingRight: theme.spacing[3],
+  },
   diffStatusRow: {
     flexDirection: "row",
     alignItems: "center",
-    alignSelf: "flex-start",
     gap: theme.spacing[1],
-    // Align text with chevron/branch icons (at spacing[2] from edge)
+    // Align text with header branch icon (at spacing[3] from edge, minus our horizontal padding)
+    marginLeft: theme.spacing[3] - theme.spacing[1],
     marginVertical: theme.spacing[2],
-    paddingLeft: theme.spacing[2],
-    paddingRight: theme.spacing[2],
+    paddingHorizontal: theme.spacing[1],
     paddingVertical: theme.spacing[1],
     borderRadius: theme.borderRadius.base,
   },
@@ -1251,6 +1288,15 @@ const styles = StyleSheet.create((theme) => ({
   diffStatusIconHidden: {
     opacity: 0,
   },
+  expandAllButton: {
+    flexDirection: "row",
+    alignItems: "center",
+    gap: theme.spacing[1],
+    marginVertical: theme.spacing[2],
+    paddingHorizontal: theme.spacing[1],
+    paddingVertical: theme.spacing[1],
+    borderRadius: theme.borderRadius.base,
+  },
   splitButton: {
     flexDirection: "row",
     alignItems: "stretch",

packages/app/src/contexts/voice-context.tsx

@@ -129,11 +129,11 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
 
   // Update voice detection flags whenever they change
   useEffect(() => {
-    const session = realtimeSessionRef.current;
-    if (session?.client) {
-      // Note: voice detection updates need backend support before sending.
-    }
-  }, [realtimeAudio.isDetecting, realtimeAudio.isSpeaking]);
+    activeSession?.methods?.setVoiceDetectionFlags(
+      realtimeAudio.isDetecting,
+      realtimeAudio.isSpeaking
+    );
+  }, [activeSession?.methods, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]);
 
   useEffect(() => {
     realtimeSessionRef.current = activeSession;
@@ -163,6 +163,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
       try {
         realtimeSessionRef.current = session;
         setActiveServerId(serverId);
+        await session.audioPlayer?.warmup?.();
         await realtimeAudio.start();
         setIsVoiceMode(true);
         console.log("[Voice] Mode enabled");
@@ -195,6 +196,7 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
       const session = realtimeSessionRef.current;
       session?.audioPlayer?.stop();
       await realtimeAudio.stop();
+      session?.methods?.setVoiceDetectionFlags(false, false);
       setIsVoiceMode(false);
       setActiveServerId(null);
       console.log("[Voice] Mode disabled");

packages/app/src/hooks/use-audio-player.native.ts

@@ -56,6 +56,10 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
   const queueRef = useRef<QueuedAudio[]>([]);
   const suppressedQueueRef = useRef<QueuedAudio[]>([]);
   const isProcessingQueueRef = useRef(false);
+  const activePlaybackRef = useRef<{
+    resolve: (duration: number) => void;
+    reject: (error: Error) => void;
+  } | null>(null);
   const playbackTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
   const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
 
@@ -179,6 +183,7 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
 
   async function playAudio(audioData: Blob): Promise<number> {
     return new Promise(async (resolve, reject) => {
+      activePlaybackRef.current = { resolve, reject };
       try {
         console.log(
           `[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
@@ -234,6 +239,7 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
           console.log("[AudioPlayer] ✅ Playback finished");
           setIsPlaying(false);
           playbackTimeoutRef.current = null;
+          activePlaybackRef.current = null;
           resolve(durationSec);
         }, durationSec * 1000);
       } catch (error) {
@@ -246,6 +252,7 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
         }
 
         setIsPlaying(false);
+        activePlaybackRef.current = null;
         reject(error);
       }
     });
@@ -267,6 +274,12 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
 
     setIsPlaying(false);
 
+    // Reject the currently playing promise, if any.
+    if (activePlaybackRef.current) {
+      activePlaybackRef.current.reject(new Error("Playback stopped"));
+      activePlaybackRef.current = null;
+    }
+
     // Reject all pending promises in the main queue
     while (queueRef.current.length > 0) {
       const item = queueRef.current.shift()!;
@@ -313,5 +326,13 @@ export function useAudioPlayer(options?: AudioPlayerOptions) {
     stop,
     isPlaying: () => isPlaying,
     clearQueue,
+    warmup: async () => {
+      if (!audioInitialized) {
+        await initialize();
+        setAudioInitialized(true);
+      }
+      // Ensure playback engine isn't suspended after a previous stop.
+      resumePlayback();
+    },
   };
 }

packages/app/src/hooks/use-audio-player.web.ts

@@ -1,4 +1,4 @@
-import { useMemo } from "react";
+import { useMemo, useRef, useState } from "react";
 
 export interface AudioPlayerOptions {
   isDetecting?: () => boolean;
@@ -6,22 +6,270 @@ export interface AudioPlayerOptions {
 }
 
 /**
- * Web shim for the native two-way audio player.
- * We currently don't support Voice mode playback on web.
+ * Web audio player for server-provided audio chunks.
+ *
+ * Supports:
+ * - `audio/pcm` (assumed PCM16 LE, mono, default 24kHz unless `rate=` is present)
+ * - formats supported by `AudioContext.decodeAudioData` (e.g. mp3)
  */
-export function useAudioPlayer(_options?: AudioPlayerOptions) {
+export function useAudioPlayer(options?: AudioPlayerOptions) {
+  const [isPlayingState, setIsPlayingState] = useState(false);
+
+  type QueuedAudio = {
+    audioData: Blob;
+    resolve: (duration: number) => void;
+    reject: (error: Error) => void;
+  };
+
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const queueRef = useRef<QueuedAudio[]>([]);
+  const suppressedQueueRef = useRef<QueuedAudio[]>([]);
+  const isProcessingQueueRef = useRef(false);
+  const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
+  const activePlaybackRef = useRef<{
+    source: AudioBufferSourceNode;
+    resolve: (duration: number) => void;
+    reject: (error: Error) => void;
+    settled: boolean;
+  } | null>(null);
+
+  const decodeAudioData = async (context: AudioContext, buffer: ArrayBuffer): Promise<AudioBuffer> => {
+    const maybePromise = context.decodeAudioData(buffer);
+    if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
+      return maybePromise as Promise<AudioBuffer>;
+    }
+    return await new Promise<AudioBuffer>((resolve, reject) => {
+      context.decodeAudioData(buffer, resolve, reject);
+    });
+  };
+
+  const parsePcmSampleRate = (mimeType: string): number | null => {
+    const match = /rate=(\\d+)/i.exec(mimeType);
+    if (!match) {
+      return null;
+    }
+    const rate = Number(match[1]);
+    return Number.isFinite(rate) && rate > 0 ? rate : null;
+  };
+
+  const pcm16LeToAudioBuffer = (context: AudioContext, bytes: Uint8Array, sampleRate: number): AudioBuffer => {
+    const sampleCount = Math.floor(bytes.length / 2);
+    const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
+    const channel = audioBuffer.getChannelData(0);
+    for (let i = 0; i < sampleCount; i++) {
+      const lo = bytes[i * 2]!;
+      const hi = bytes[i * 2 + 1]!;
+      let value = (hi << 8) | lo;
+      if (value & 0x8000) {
+        value = value - 0x10000;
+      }
+      channel[i] = value / 0x8000;
+    }
+    return audioBuffer;
+  };
+
+  const ensureContext = async (): Promise<AudioContext> => {
+    if (audioContextRef.current) {
+      if (audioContextRef.current.state === "suspended") {
+        try {
+          await audioContextRef.current.resume();
+        } catch {
+          // Best effort. If this fails due to autoplay policies, a later user gesture
+          // (or explicit warmup call) can unlock it.
+        }
+      }
+      return audioContextRef.current;
+    }
+    const context = new AudioContext();
+    if (context.state === "suspended") {
+      try {
+        await context.resume();
+      } catch {
+        // See note above.
+      }
+    }
+    audioContextRef.current = context;
+    return context;
+  };
+
+  const shouldSuppressPlayback = (): boolean => {
+    return Boolean(options?.isDetecting?.()) || Boolean(options?.isSpeaking?.());
+  };
+
+  const startCheckingForClearFlags = (): void => {
+    if (checkIntervalRef.current !== null) {
+      return;
+    }
+
+    checkIntervalRef.current = setInterval(() => {
+      const isStillBlocked = shouldSuppressPlayback();
+      if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
+        const suppressedItems = [...suppressedQueueRef.current];
+        suppressedQueueRef.current = [];
+        queueRef.current = [...suppressedItems, ...queueRef.current];
+
+        if (checkIntervalRef.current !== null) {
+          clearInterval(checkIntervalRef.current);
+          checkIntervalRef.current = null;
+        }
+
+        if (!isProcessingQueueRef.current) {
+          void processQueue();
+        }
+      } else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
+        if (checkIntervalRef.current !== null) {
+          clearInterval(checkIntervalRef.current);
+          checkIntervalRef.current = null;
+        }
+      }
+    }, 100);
+  };
+
+  const playAudio = async (audioData: Blob): Promise<number> => {
+    const context = await ensureContext();
+    const arrayBuffer = await audioData.arrayBuffer();
+
+    let audioBuffer: AudioBuffer;
+    const type = (audioData.type || "").toLowerCase();
+
+    if (type.startsWith("audio/pcm")) {
+      const sampleRate = parsePcmSampleRate(type) ?? 24000;
+      audioBuffer = pcm16LeToAudioBuffer(context, new Uint8Array(arrayBuffer), sampleRate);
+    } else {
+      audioBuffer = await decodeAudioData(context, arrayBuffer);
+    }
+
+    const durationSec = audioBuffer.duration;
+    const source = context.createBufferSource();
+    source.buffer = audioBuffer;
+    source.connect(context.destination);
+
+    return await new Promise<number>((resolve, reject) => {
+      activePlaybackRef.current = { source, resolve, reject, settled: false };
+      setIsPlayingState(true);
+
+      const settleOnce = (fn: () => void) => {
+        const active = activePlaybackRef.current;
+        if (!active || active.source !== source || active.settled) {
+          return;
+        }
+        active.settled = true;
+        activePlaybackRef.current = null;
+        setIsPlayingState(false);
+        fn();
+      };
+
+      source.onended = () => {
+        settleOnce(() => resolve(durationSec));
+      };
+
+      try {
+        source.start();
+      } catch (e) {
+        settleOnce(() => reject(e instanceof Error ? e : new Error(String(e))));
+      }
+    });
+  };
+
+  const processQueue = async (): Promise<void> => {
+    if (isProcessingQueueRef.current || queueRef.current.length === 0) {
+      return;
+    }
+
+    isProcessingQueueRef.current = true;
+
+    while (queueRef.current.length > 0) {
+      if (shouldSuppressPlayback()) {
+        suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
+        queueRef.current = [];
+        startCheckingForClearFlags();
+        break;
+      }
+
+      const item = queueRef.current.shift()!;
+      try {
+        const duration = await playAudio(item.audioData);
+        item.resolve(duration);
+      } catch (error) {
+        item.reject(error as Error);
+      }
+    }
+
+    isProcessingQueueRef.current = false;
+  };
+
+  const play = async (audioData: Blob): Promise<number> => {
+    return await new Promise<number>((resolve, reject) => {
+      if (shouldSuppressPlayback()) {
+        suppressedQueueRef.current.push({ audioData, resolve, reject });
+        startCheckingForClearFlags();
+        return;
+      }
+
+      queueRef.current.push({ audioData, resolve, reject });
+      if (!isProcessingQueueRef.current) {
+        void processQueue();
+      }
+    });
+  };
+
+  const stop = (): void => {
+    // Stop currently playing audio (and reject its promise)
+    if (activePlaybackRef.current) {
+      const active = activePlaybackRef.current;
+      activePlaybackRef.current = null;
+      try {
+        active.source.onended = null;
+        active.source.stop();
+      } catch {
+        // ignore
+      }
+      if (!active.settled) {
+        active.settled = true;
+        active.reject(new Error("Playback stopped"));
+      }
+    }
+
+    setIsPlayingState(false);
+
+    while (queueRef.current.length > 0) {
+      queueRef.current.shift()!.reject(new Error("Playback stopped"));
+    }
+    while (suppressedQueueRef.current.length > 0) {
+      suppressedQueueRef.current.shift()!.reject(new Error("Playback stopped"));
+    }
+
+    if (checkIntervalRef.current !== null) {
+      clearInterval(checkIntervalRef.current);
+      checkIntervalRef.current = null;
+    }
+
+    isProcessingQueueRef.current = false;
+  };
+
+  const clearQueue = (): void => {
+    while (queueRef.current.length > 0) {
+      queueRef.current.shift()!.reject(new Error("Queue cleared"));
+    }
+    while (suppressedQueueRef.current.length > 0) {
+      suppressedQueueRef.current.shift()!.reject(new Error("Queue cleared"));
+    }
+    if (checkIntervalRef.current !== null) {
+      clearInterval(checkIntervalRef.current);
+      checkIntervalRef.current = null;
+    }
+  };
+
   return useMemo(
     () => ({
-      play: async () => {
-        console.warn("[AudioPlayer] Web playback is disabled");
-        return 0;
+      play,
+      stop,
+      isPlaying: () => isPlayingState,
+      clearQueue,
+      warmup: async () => {
+        await ensureContext();
       },
-      stop: () => {
-        console.warn("[AudioPlayer] Web playback stop noop");
-      },
-      isPlaying: () => false,
-      clearQueue: () => {},
     }),
-    []
+    [isPlayingState]
   );
 }

packages/app/src/hooks/use-speechmatics-audio.native.ts

@@ -1,5 +1,4 @@
 import { useState, useEffect, useRef, useCallback } from "react";
-import { Buffer } from "buffer";
 import {
   initialize,
   useMicrophonePermissions,
@@ -10,6 +9,8 @@ import {
   type VolumeLevelCallback,
 } from "@boudra/expo-two-way-audio";
 
+import { SpeechSegmenter } from "@/voice/speech-segmenter";
+
 export interface SpeechmaticsAudioConfig {
   onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
   onSpeechStart?: () => void;
@@ -35,31 +36,6 @@ export interface SpeechmaticsAudio {
   segmentDuration: number;
 }
 
-function uint8ArrayToBase64(bytes: Uint8Array): string {
-  // NOTE: This is performance-sensitive during continuous streaming.
-  // Buffer-backed base64 is significantly faster than manual string building.
-  try {
-    return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
-  } catch {
-    let binary = "";
-    for (let i = 0; i < bytes.length; i++) {
-      binary += String.fromCharCode(bytes[i]);
-    }
-    return btoa(binary);
-  }
-}
-
-function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
-  const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
-  const result = new Uint8Array(totalLength);
-  let offset = 0;
-  for (const arr of arrays) {
-    result.set(arr, offset);
-    offset += arr.length;
-  }
-  return result;
-}
-
 /**
  * Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
  */
@@ -78,49 +54,64 @@ export function useSpeechmaticsAudio(
 
   const enableContinuousStreaming = config.enableContinuousStreaming === true;
 
-  const audioBufferRef = useRef<Uint8Array[]>([]);
-  const silenceStartRef = useRef<number | null>(null);
-  const isSpeakingRef = useRef(false);
-  const speechDetectionStartRef = useRef<number | null>(null);
-  const speechConfirmedRef = useRef(false);
-  const detectionSilenceStartRef = useRef<number | null>(null);
-  const bufferedBytesRef = useRef(0);
+  const isActiveRef = useRef(isActive);
+  useEffect(() => {
+    isActiveRef.current = isActive;
+  }, [isActive]);
 
-  const PCM_SAMPLE_RATE = 16000;
-  const PCM_CHANNELS = 1;
-  const PCM_BITS_PER_SAMPLE = 16;
-  const PCM_BYTES_PER_MS =
-    (PCM_SAMPLE_RATE * PCM_CHANNELS * (PCM_BITS_PER_SAMPLE / 8)) / 1000;
-  const MIN_CHUNK_DURATION_MS = 1000;
-  const MIN_CHUNK_BYTES = Math.round(
-    PCM_BYTES_PER_MS * MIN_CHUNK_DURATION_MS
-  );
+  const isMutedRef = useRef(isMuted);
+  useEffect(() => {
+    isMutedRef.current = isMuted;
+  }, [isMuted]);
 
-  const flushBufferedAudio = useCallback(
-    (isLast: boolean) => {
-      if (audioBufferRef.current.length === 0) {
-        bufferedBytesRef.current = 0;
-        return;
+  const callbacksRef = useRef({
+    onAudioSegment: config.onAudioSegment,
+    onSpeechStart: config.onSpeechStart,
+    onSpeechEnd: config.onSpeechEnd,
+  });
+  useEffect(() => {
+    callbacksRef.current = {
+      onAudioSegment: config.onAudioSegment,
+      onSpeechStart: config.onSpeechStart,
+      onSpeechEnd: config.onSpeechEnd,
+    };
+  }, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd]);
+
+  const segmenterRef = useRef<SpeechSegmenter | null>(null);
+  if (segmenterRef.current === null) {
+    segmenterRef.current = new SpeechSegmenter(
+      {
+        enableContinuousStreaming,
+        volumeThreshold: config.volumeThreshold,
+        silenceDurationMs: config.silenceDuration,
+        speechConfirmationMs: config.speechConfirmationDuration,
+        detectionGracePeriodMs: config.detectionGracePeriod,
+      },
+      {
+        onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
+        onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
+        onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
+        onDetectingChange: (next) => setIsDetecting(next),
+        onSpeakingChange: (next) => setIsSpeaking(next),
       }
+    );
+  }
 
-      const combinedBinary = concatenateUint8Arrays(audioBufferRef.current);
-      const pcmBase64 = uint8ArrayToBase64(combinedBinary);
-
-      config.onAudioSegment?.({
-        audioData: pcmBase64,
-        isLast,
-      });
-
-      audioBufferRef.current = [];
-      bufferedBytesRef.current = 0;
-    },
-    [config]
-  );
-
-  const VOLUME_THRESHOLD = config.volumeThreshold;
-  const SILENCE_DURATION_MS = config.silenceDuration;
-  const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration;
-  const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod;
+  useEffect(() => {
+    segmenterRef.current?.updateConfig({
+      enableContinuousStreaming,
+      volumeThreshold: config.volumeThreshold,
+      silenceDurationMs: config.silenceDuration,
+      speechConfirmationMs: config.speechConfirmationDuration,
+      detectionGracePeriodMs: config.detectionGracePeriod,
+    });
+  }, [
+    enableContinuousStreaming,
+    config.volumeThreshold,
+    config.silenceDuration,
+    config.speechConfirmationDuration,
+    config.detectionGracePeriod,
+  ]);
 
   // Update segment duration timer
   useEffect(() => {
@@ -129,7 +120,7 @@ export function useSpeechmaticsAudio(
       return;
     }
 
-    const startTime = speechDetectionStartRef.current || Date.now();
+    const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
     const interval = setInterval(() => {
       const elapsed = Date.now() - startTime;
       setSegmentDuration(elapsed);
@@ -143,34 +134,12 @@ export function useSpeechmaticsAudio(
     "onMicrophoneData",
     useCallback<MicrophoneDataCallback>(
       (event) => {
-        if (!isActive || isMuted) return;
+        if (!isActiveRef.current || isMutedRef.current) return;
 
         const pcmData: Uint8Array = event.data;
-
-        if (enableContinuousStreaming) {
-          audioBufferRef.current.push(pcmData);
-          bufferedBytesRef.current += pcmData.length;
-          if (bufferedBytesRef.current >= MIN_CHUNK_BYTES) {
-            flushBufferedAudio(false);
-          }
-          return;
-        }
-
-        // Buffer the audio chunk if we're detecting or speaking
-        // Start buffering from first spike to capture beginning of speech
-        if (speechDetectionStartRef.current !== null || isSpeakingRef.current) {
-          audioBufferRef.current.push(pcmData);
-          bufferedBytesRef.current += pcmData.length;
-
-          if (
-            speechConfirmedRef.current &&
-            bufferedBytesRef.current >= MIN_CHUNK_BYTES
-          ) {
-            flushBufferedAudio(false);
-          }
-        }
+        segmenterRef.current?.pushPcmChunk(pcmData);
       },
-      [enableContinuousStreaming, isActive, isMuted, flushBufferedAudio, MIN_CHUNK_BYTES]
+      []
     )
   );
 
@@ -179,128 +148,15 @@ export function useSpeechmaticsAudio(
     "onInputVolumeLevelData",
     useCallback<VolumeLevelCallback>(
       (event) => {
-        if (!isActive) return;
+        if (!isActiveRef.current) return;
 
         const volumeLevel: number = event.data;
         setVolume(volumeLevel);
 
-        if (isMuted) return;
-        if (enableContinuousStreaming) return;
-
-        const speechDetected = volumeLevel > VOLUME_THRESHOLD;
-
-        // console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD);
-
-        if (
-          speechDetected &&
-          !isSpeakingRef.current &&
-          !speechConfirmedRef.current
-        ) {
-          // Initial speech detection - start tracking
-          if (speechDetectionStartRef.current === null) {
-            console.log(
-              "[SpeechmaticsAudio] Speech started (volume:",
-              volumeLevel.toFixed(4),
-              ")"
-            );
-            speechDetectionStartRef.current = Date.now();
-            detectionSilenceStartRef.current = null;
-            audioBufferRef.current = [];
-            bufferedBytesRef.current = 0;
-            setIsDetecting(true);
-          } else {
-            // Volume is back above threshold - reset grace period
-            detectionSilenceStartRef.current = null;
-
-            // Check if speech has been sustained long enough
-            const speechDuration = Date.now() - speechDetectionStartRef.current;
-            if (speechDuration >= SPEECH_CONFIRMATION_MS) {
-              // Speech CONFIRMED
-              console.log(
-                "[SpeechmaticsAudio] Speech confirmed after",
-                speechDuration,
-                "ms"
-              );
-              isSpeakingRef.current = true;
-              speechConfirmedRef.current = true;
-              silenceStartRef.current = null;
-              setIsDetecting(false);
-              setIsSpeaking(true);
-              config.onSpeechStart?.();
-            }
-          }
-        } else if (
-          speechDetected &&
-          isSpeakingRef.current &&
-          speechConfirmedRef.current
-        ) {
-          // Continuing confirmed speech
-          silenceStartRef.current = null;
-        } else if (
-          !speechDetected &&
-          !speechConfirmedRef.current &&
-          speechDetectionStartRef.current !== null
-        ) {
-          // Volume dropped during detection phase - apply grace period
-          if (detectionSilenceStartRef.current === null) {
-            detectionSilenceStartRef.current = Date.now();
-          } else {
-            const graceDuration = Date.now() - detectionSilenceStartRef.current;
-            if (graceDuration >= DETECTION_GRACE_PERIOD_MS) {
-              // Grace period expired - cancel detection
-              console.log(
-                "[SpeechmaticsAudio] Speech detection cancelled after",
-                graceDuration,
-                "ms grace period"
-              );
-              speechDetectionStartRef.current = null;
-              detectionSilenceStartRef.current = null;
-              audioBufferRef.current = [];
-              bufferedBytesRef.current = 0;
-              setIsDetecting(false);
-            }
-          }
-        } else if (
-          !speechDetected &&
-          isSpeakingRef.current &&
-          speechConfirmedRef.current
-        ) {
-          // Potential speech END
-          if (silenceStartRef.current === null) {
-            silenceStartRef.current = Date.now();
-          } else {
-            const silenceDuration = Date.now() - silenceStartRef.current;
-            if (silenceDuration >= SILENCE_DURATION_MS) {
-              // Speech END confirmed
-              console.log(
-                "[SpeechmaticsAudio] Speech ended after",
-                silenceDuration,
-                "ms silence"
-              );
-              isSpeakingRef.current = false;
-              speechConfirmedRef.current = false;
-              speechDetectionStartRef.current = null;
-              silenceStartRef.current = null;
-              setIsSpeaking(false);
-              config.onSpeechEnd?.();
-
-              // Send buffered audio segment
-              flushBufferedAudio(true);
-            }
-          }
-        }
+        if (isMutedRef.current) return;
+        segmenterRef.current?.pushVolumeLevel(volumeLevel, Date.now());
       },
-      [
-        enableContinuousStreaming,
-        isActive,
-        isMuted,
-        VOLUME_THRESHOLD,
-        SILENCE_DURATION_MS,
-        SPEECH_CONFIRMATION_MS,
-        DETECTION_GRACE_PERIOD_MS,
-        config,
-        flushBufferedAudio,
-      ]
+      []
     )
   );
 
@@ -363,9 +219,7 @@ export function useSpeechmaticsAudio(
       toggleRecording(false);
     }
 
-    if (enableContinuousStreaming) {
-      flushBufferedAudio(true);
-    }
+    segmenterRef.current?.stop(Date.now());
 
     // Tear down audio session
     if (audioInitialized) {
@@ -375,13 +229,7 @@ export function useSpeechmaticsAudio(
     }
 
     // Reset state
-    audioBufferRef.current = [];
-    bufferedBytesRef.current = 0;
-    isSpeakingRef.current = false;
-    speechConfirmedRef.current = false;
-    speechDetectionStartRef.current = null;
-    detectionSilenceStartRef.current = null;
-    silenceStartRef.current = null;
+    segmenterRef.current?.reset();
     setIsActive(false);
     setIsSpeaking(false);
     setIsDetecting(false);
@@ -398,13 +246,7 @@ export function useSpeechmaticsAudio(
 
       if (newMuted) {
         // Clear any ongoing speech detection/speaking state
-        audioBufferRef.current = [];
-        bufferedBytesRef.current = 0;
-        isSpeakingRef.current = false;
-        speechConfirmedRef.current = false;
-        speechDetectionStartRef.current = null;
-        detectionSilenceStartRef.current = null;
-        silenceStartRef.current = null;
+        segmenterRef.current?.reset();
         setIsSpeaking(false);
         setIsDetecting(false);
       }

packages/app/src/hooks/use-speechmatics-audio.web.ts

@@ -1,15 +1,18 @@
-import { useCallback, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+
+import { SpeechSegmenter } from "@/voice/speech-segmenter";
 
 export interface SpeechmaticsAudioConfig {
   onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
   onSpeechStart?: () => void;
   onSpeechEnd?: () => void;
   onError?: (error: Error) => void;
+  /** When true, stream microphone PCM continuously without VAD gating. */
   enableContinuousStreaming?: boolean;
-  volumeThreshold: number;
-  silenceDuration: number;
-  speechConfirmationDuration: number;
-  detectionGracePeriod: number;
+  volumeThreshold: number; // 0-1
+  silenceDuration: number; // ms of silence before ending segment
+  speechConfirmationDuration: number; // ms of sustained speech before confirming
+  detectionGracePeriod: number; // ms grace period for volume dips during detection
 }
 
 export interface SpeechmaticsAudio {
@@ -24,41 +27,324 @@ export interface SpeechmaticsAudio {
   segmentDuration: number;
 }
 
-/**
- * Web shim for the Speechmatics two-way audio hook.
- * Real-time microphone capture is currently unsupported on web.
- */
-export function useSpeechmaticsAudio(
-  _config: SpeechmaticsAudioConfig
-): SpeechmaticsAudio {
+const getAudioContextCtor = (): (typeof AudioContext) | null => {
+  if (typeof window === "undefined") {
+    return null;
+  }
+  const ctor =
+    (window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
+    (window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
+  return ctor ?? null;
+};
+
+const floatToInt16 = (sample: number): number => {
+  const clamped = Math.max(-1, Math.min(1, sample));
+  return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
+};
+
+const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
+  if (input.length === 0) {
+    return new Int16Array(0);
+  }
+  if (inputRate === outputRate) {
+    const out = new Int16Array(input.length);
+    for (let i = 0; i < input.length; i++) {
+      out[i] = floatToInt16(input[i]);
+    }
+    return out;
+  }
+
+  const ratio = inputRate / outputRate;
+  const outputLength = Math.max(1, Math.round(input.length / ratio));
+  const out = new Int16Array(outputLength);
+  for (let i = 0; i < outputLength; i++) {
+    const sourceIndex = i * ratio;
+    const i0 = Math.floor(sourceIndex);
+    const i1 = Math.min(input.length - 1, i0 + 1);
+    const frac = sourceIndex - i0;
+    const sample = input[i0] * (1 - frac) + input[i1] * frac;
+    out[i] = floatToInt16(sample);
+  }
+  return out;
+};
+
+export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): SpeechmaticsAudio {
   const [isActive, setIsActive] = useState(false);
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const [isDetecting, setIsDetecting] = useState(false);
+  const [volume, setVolume] = useState(0);
   const [isMuted, setIsMuted] = useState(false);
+  const [segmentDuration, setSegmentDuration] = useState(0);
+
+  const enableContinuousStreaming = config.enableContinuousStreaming === true;
+
+  const callbacksRef = useRef({
+    onAudioSegment: config.onAudioSegment,
+    onSpeechStart: config.onSpeechStart,
+    onSpeechEnd: config.onSpeechEnd,
+    onError: config.onError,
+  });
+
+  useEffect(() => {
+    callbacksRef.current = {
+      onAudioSegment: config.onAudioSegment,
+      onSpeechStart: config.onSpeechStart,
+      onSpeechEnd: config.onSpeechEnd,
+      onError: config.onError,
+    };
+  }, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd, config.onError]);
+
+  const segmenterRef = useRef<SpeechSegmenter | null>(null);
+  if (segmenterRef.current === null) {
+    segmenterRef.current = new SpeechSegmenter(
+      {
+        enableContinuousStreaming,
+        volumeThreshold: config.volumeThreshold,
+        silenceDurationMs: config.silenceDuration,
+        speechConfirmationMs: config.speechConfirmationDuration,
+        detectionGracePeriodMs: config.detectionGracePeriod,
+      },
+      {
+        onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
+        onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
+        onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
+        onDetectingChange: (next) => setIsDetecting(next),
+        onSpeakingChange: (next) => setIsSpeaking(next),
+      }
+    );
+  }
+
+  useEffect(() => {
+    segmenterRef.current?.updateConfig({
+      enableContinuousStreaming,
+      volumeThreshold: config.volumeThreshold,
+      silenceDurationMs: config.silenceDuration,
+      speechConfirmationMs: config.speechConfirmationDuration,
+      detectionGracePeriodMs: config.detectionGracePeriod,
+    });
+  }, [
+    enableContinuousStreaming,
+    config.volumeThreshold,
+    config.silenceDuration,
+    config.speechConfirmationDuration,
+    config.detectionGracePeriod,
+  ]);
+
+  const refs = useRef<{
+    started: boolean;
+    stream: MediaStream | null;
+    context: AudioContext | null;
+    source: MediaStreamAudioSourceNode | null;
+    processor: ScriptProcessorNode | null;
+    gain: GainNode | null;
+  }>({
+    started: false,
+    stream: null,
+    context: null,
+    source: null,
+    processor: null,
+    gain: null,
+  });
+
+  const isMutedRef = useRef(isMuted);
+  useEffect(() => {
+    isMutedRef.current = isMuted;
+  }, [isMuted]);
+
+  // Update segment duration timer
+  useEffect(() => {
+    if (!isDetecting && !isSpeaking) {
+      setSegmentDuration(0);
+      return;
+    }
+
+    const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
+    const interval = setInterval(() => {
+      setSegmentDuration(Date.now() - startTime);
+    }, 100);
+
+    return () => clearInterval(interval);
+  }, [isDetecting, isSpeaking]);
+
+  const stopInternal = useCallback(async () => {
+    refs.current.started = false;
+
+    try {
+      refs.current.processor?.disconnect();
+      refs.current.source?.disconnect();
+      refs.current.gain?.disconnect();
+    } catch {
+      // best-effort teardown
+    }
+
+    if (refs.current.stream) {
+      for (const track of refs.current.stream.getTracks()) {
+        try {
+          track.stop();
+        } catch {
+          // ignore
+        }
+      }
+    }
+
+    const context = refs.current.context;
+    if (context && context.state !== "closed") {
+      try {
+        await context.close();
+      } catch {
+        // ignore
+      }
+    }
+
+    refs.current.stream = null;
+    refs.current.context = null;
+    refs.current.source = null;
+    refs.current.processor = null;
+    refs.current.gain = null;
+
+    segmenterRef.current?.stop(Date.now());
+    setIsActive(false);
+    setIsSpeaking(false);
+    setIsDetecting(false);
+    setVolume(0);
+    setIsMuted(false);
+  }, []);
 
   const start = useCallback(async () => {
-    console.warn("[SpeechmaticsAudio] Web microphone capture disabled");
-    setIsActive(true);
-  }, []);
+    if (refs.current.started) {
+      return;
+    }
+
+    const missingNavigator =
+      typeof navigator === "undefined" ||
+      !navigator.mediaDevices ||
+      typeof navigator.mediaDevices.getUserMedia !== "function";
+
+    const secureContext =
+      typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
+        ? window.isSecureContext
+        : true;
+    const currentOrigin = typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
+
+    try {
+      if (missingNavigator) {
+        throw new Error("Microphone capture is not supported in this environment");
+      }
+      if (!secureContext) {
+        throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
+      }
+
+      const AudioContextCtor = getAudioContextCtor();
+      if (!AudioContextCtor) {
+        throw new Error("AudioContext unavailable");
+      }
+
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          channelCount: 1,
+          noiseSuppression: true,
+          echoCancellation: true,
+          autoGainControl: true,
+        },
+      });
+
+      const context = new AudioContextCtor();
+      if (context.state === "suspended") {
+        await context.resume();
+      }
+
+      const source = context.createMediaStreamSource(stream);
+      const processor = context.createScriptProcessor(4096, 1, 1);
+      const gain = context.createGain();
+      gain.gain.value = 0;
+
+      refs.current = {
+        started: true,
+        stream,
+        context,
+        source,
+        processor,
+        gain,
+      };
+
+      processor.onaudioprocess = (event) => {
+        if (!refs.current.started) {
+          return;
+        }
+
+        const input = event.inputBuffer.getChannelData(0);
+        let sumSquares = 0;
+        for (let i = 0; i < input.length; i++) {
+          const sample = input[i];
+          sumSquares += sample * sample;
+        }
+        const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
+        const normalized = Math.min(1, Math.max(0, rms * 2));
+        setVolume(normalized);
+
+        if (isMutedRef.current) {
+          return;
+        }
+
+        const nowMs = Date.now();
+        segmenterRef.current?.pushVolumeLevel(normalized, nowMs);
+
+        const pcm16 = resampleToPcm16(input, context.sampleRate, 16000);
+        const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength);
+        segmenterRef.current?.pushPcmChunk(pcmBytes);
+      };
+
+      source.connect(processor);
+      processor.connect(gain);
+      gain.connect(context.destination);
+
+      setIsActive(true);
+      return;
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      callbacksRef.current.onError?.(err);
+      await stopInternal();
+      throw err;
+    }
+  }, [stopInternal]);
 
   const stop = useCallback(async () => {
-    setIsActive(false);
-  }, []);
+    await stopInternal();
+  }, [stopInternal]);
 
   const toggleMute = useCallback(() => {
-    setIsMuted((prev) => !prev);
+    setIsMuted((prev) => {
+      const nextMuted = !prev;
+      if (nextMuted) {
+        segmenterRef.current?.reset();
+        setIsSpeaking(false);
+        setIsDetecting(false);
+      }
+      return nextMuted;
+    });
   }, []);
 
+  useEffect(() => {
+    return () => {
+      if (refs.current.started) {
+        void stopInternal();
+      }
+    };
+  }, [stopInternal]);
+
   return useMemo(
     () => ({
       start,
       stop,
       toggleMute,
       isActive,
-      isSpeaking: false,
-      isDetecting: false,
+      isSpeaking,
+      isDetecting,
       isMuted,
-      volume: 0,
-      segmentDuration: 0,
+      volume,
+      segmentDuration,
     }),
-    [start, stop, toggleMute, isActive, isMuted]
+    [start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration]
   );
 }
+

packages/app/src/voice/speech-segmenter.test.ts

@@ -0,0 +1,91 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { SpeechSegmenter } from "./speech-segmenter";
+
+const mkPcmBytes = (length: number): Uint8Array => {
+  const bytes = new Uint8Array(length);
+  for (let i = 0; i < length; i++) bytes[i] = i % 255;
+  return bytes;
+};
+
+describe("SpeechSegmenter", () => {
+  it("streams continuously and flushes at stop", () => {
+    const onAudioSegment = vi.fn();
+    const segmenter = new SpeechSegmenter(
+      {
+        enableContinuousStreaming: true,
+        volumeThreshold: 0.3,
+        silenceDurationMs: 2000,
+        speechConfirmationMs: 300,
+        detectionGracePeriodMs: 200,
+        minChunkDurationMs: 100, // ensure we only flush on stop
+        pcmSampleRate: 1000,
+      },
+      { onAudioSegment }
+    );
+
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    expect(onAudioSegment).not.toHaveBeenCalled();
+
+    segmenter.stop(Date.now());
+
+    expect(onAudioSegment).toHaveBeenCalledTimes(1);
+    const lastCall = onAudioSegment.mock.calls.at(-1)![0];
+    expect(lastCall.isLast).toBe(true);
+    expect(lastCall.audioData.length).toBeGreaterThan(0);
+  });
+
+  it("detects speech, calls onSpeechStart, and flushes on speech end", () => {
+    const onAudioSegment = vi.fn();
+    const onSpeechStart = vi.fn();
+    const onSpeechEnd = vi.fn();
+    const detectingChanges: boolean[] = [];
+    const speakingChanges: boolean[] = [];
+
+    const segmenter = new SpeechSegmenter(
+      {
+        enableContinuousStreaming: false,
+        volumeThreshold: 0.3,
+        silenceDurationMs: 2000,
+        speechConfirmationMs: 300,
+        detectionGracePeriodMs: 200,
+        minChunkDurationMs: 100,
+        pcmSampleRate: 1000,
+      },
+      {
+        onAudioSegment,
+        onSpeechStart,
+        onSpeechEnd,
+        onDetectingChange: (v) => detectingChanges.push(v),
+        onSpeakingChange: (v) => speakingChanges.push(v),
+      }
+    );
+
+    const t0 = 10_000;
+
+    // Start detection.
+    segmenter.pushVolumeLevel(0.5, t0);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    expect(detectingChanges).toEqual([true]);
+    expect(onSpeechStart).not.toHaveBeenCalled();
+
+    // Confirm speech.
+    segmenter.pushVolumeLevel(0.5, t0 + 300);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    expect(onSpeechStart).toHaveBeenCalledTimes(1);
+    expect(speakingChanges).toEqual([true]);
+
+    // End speech after silence.
+    segmenter.pushVolumeLevel(0.0, t0 + 301);
+    segmenter.pushPcmChunk(mkPcmBytes(50));
+    segmenter.pushVolumeLevel(0.0, t0 + 301 + 2000);
+
+    expect(onSpeechEnd).toHaveBeenCalledTimes(1);
+    expect(speakingChanges).toEqual([true, false]);
+    expect(onAudioSegment).toHaveBeenCalled();
+    const lastCall = onAudioSegment.mock.calls.at(-1)![0];
+    expect(lastCall.isLast).toBe(true);
+    expect(lastCall.audioData.length).toBeGreaterThan(0);
+  });
+});

packages/app/src/voice/speech-segmenter.ts

@@ -0,0 +1,285 @@
+import { Buffer } from "buffer";
+
+export type SpeechSegment = { audioData: string; isLast: boolean };
+
+export type SpeechSegmenterConfig = {
+  enableContinuousStreaming: boolean;
+  volumeThreshold: number; // 0..1
+  silenceDurationMs: number;
+  speechConfirmationMs: number;
+  detectionGracePeriodMs: number;
+  minChunkDurationMs?: number;
+  pcmSampleRate?: number;
+  pcmChannels?: number;
+  pcmBitsPerSample?: number;
+};
+
+export type SpeechSegmenterCallbacks = {
+  onAudioSegment?: (segment: SpeechSegment) => void;
+  onSpeechStart?: () => void;
+  onSpeechEnd?: () => void;
+  onDetectingChange?: (isDetecting: boolean) => void;
+  onSpeakingChange?: (isSpeaking: boolean) => void;
+};
+
+function uint8ArrayToBase64(bytes: Uint8Array): string {
+  // NOTE: This is performance-sensitive during continuous streaming.
+  // Buffer-backed base64 is significantly faster than manual string building.
+  try {
+    return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
+  } catch {
+    let binary = "";
+    for (let i = 0; i < bytes.length; i++) {
+      binary += String.fromCharCode(bytes[i]);
+    }
+    // eslint-disable-next-line no-restricted-globals
+    return btoa(binary);
+  }
+}
+
+function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array {
+  const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const arr of arrays) {
+    result.set(arr, offset);
+    offset += arr.length;
+  }
+  return result;
+}
+
+export class SpeechSegmenter {
+  private config: Required<Omit<SpeechSegmenterConfig, "minChunkDurationMs" | "pcmSampleRate" | "pcmChannels" | "pcmBitsPerSample">> &
+    Required<Pick<SpeechSegmenterConfig, "minChunkDurationMs" | "pcmSampleRate" | "pcmChannels" | "pcmBitsPerSample">>;
+  private callbacks: SpeechSegmenterCallbacks;
+
+  private audioBuffer: Uint8Array[] = [];
+  private bufferedBytes = 0;
+
+  private speechDetectionStartMs: number | null = null;
+  private detectionSilenceStartMs: number | null = null;
+  private silenceStartMs: number | null = null;
+  private isSpeaking = false;
+  private isDetecting = false;
+  private speechConfirmed = false;
+
+  private pcmBytesPerMs: number;
+  private minChunkBytes: number;
+
+  constructor(config: SpeechSegmenterConfig, callbacks: SpeechSegmenterCallbacks = {}) {
+    this.config = {
+      ...config,
+      minChunkDurationMs: config.minChunkDurationMs ?? 1000,
+      pcmSampleRate: config.pcmSampleRate ?? 16000,
+      pcmChannels: config.pcmChannels ?? 1,
+      pcmBitsPerSample: config.pcmBitsPerSample ?? 16,
+    };
+    this.callbacks = callbacks;
+    this.pcmBytesPerMs = 0;
+    this.minChunkBytes = 0;
+    this.recomputeDerived();
+  }
+
+  updateConfig(next: SpeechSegmenterConfig): void {
+    this.config = {
+      ...next,
+      minChunkDurationMs: next.minChunkDurationMs ?? this.config.minChunkDurationMs,
+      pcmSampleRate: next.pcmSampleRate ?? this.config.pcmSampleRate,
+      pcmChannels: next.pcmChannels ?? this.config.pcmChannels,
+      pcmBitsPerSample: next.pcmBitsPerSample ?? this.config.pcmBitsPerSample,
+    };
+    this.recomputeDerived();
+  }
+
+  setCallbacks(callbacks: SpeechSegmenterCallbacks): void {
+    this.callbacks = callbacks;
+  }
+
+  getSpeechDetectionStartMs(): number | null {
+    return this.speechDetectionStartMs;
+  }
+
+  getIsSpeaking(): boolean {
+    return this.isSpeaking;
+  }
+
+  getIsDetecting(): boolean {
+    return this.isDetecting;
+  }
+
+  reset(): void {
+    this.audioBuffer = [];
+    this.bufferedBytes = 0;
+    this.speechDetectionStartMs = null;
+    this.detectionSilenceStartMs = null;
+    this.silenceStartMs = null;
+    this.isSpeaking = false;
+    this.isDetecting = false;
+    this.speechConfirmed = false;
+  }
+
+  flush(isLast: boolean): void {
+    if (this.audioBuffer.length === 0) {
+      this.bufferedBytes = 0;
+      return;
+    }
+
+    const combinedBinary = concatenateUint8Arrays(this.audioBuffer);
+    const pcmBase64 = uint8ArrayToBase64(combinedBinary);
+    this.callbacks.onAudioSegment?.({ audioData: pcmBase64, isLast });
+
+    this.audioBuffer = [];
+    this.bufferedBytes = 0;
+  }
+
+  /**
+   * Called when the input stream emits a PCM16 chunk (Uint8Array of bytes).
+   * In non-continuous mode, chunk buffering is gated by the VAD state.
+   */
+  pushPcmChunk(chunk: Uint8Array): void {
+    if (chunk.length === 0) {
+      return;
+    }
+
+    if (this.config.enableContinuousStreaming) {
+      this.audioBuffer.push(chunk);
+      this.bufferedBytes += chunk.length;
+      if (this.bufferedBytes >= this.minChunkBytes) {
+        this.flush(false);
+      }
+      return;
+    }
+
+    // Buffer if we're detecting or speaking. When speech is confirmed, we also
+    // stream partial chunks to reduce latency.
+    if (this.speechDetectionStartMs !== null || this.isSpeaking) {
+      this.audioBuffer.push(chunk);
+      this.bufferedBytes += chunk.length;
+
+      if (this.speechConfirmed && this.bufferedBytes >= this.minChunkBytes) {
+        this.flush(false);
+      }
+    }
+  }
+
+  /**
+   * Called with a normalized volume level (0..1). Drives VAD transitions.
+   */
+  pushVolumeLevel(volume: number, nowMs: number): void {
+    if (this.config.enableContinuousStreaming) {
+      return;
+    }
+
+    const speechDetected = volume > this.config.volumeThreshold;
+
+    if (speechDetected && !this.isSpeaking && !this.speechConfirmed) {
+      // Initial detection phase.
+      if (this.speechDetectionStartMs === null) {
+        this.speechDetectionStartMs = nowMs;
+        this.detectionSilenceStartMs = null;
+        this.audioBuffer = [];
+        this.bufferedBytes = 0;
+        this.setDetecting(true);
+        return;
+      }
+
+      // Volume is back above threshold - reset grace period.
+      this.detectionSilenceStartMs = null;
+
+      const speechDuration = nowMs - this.speechDetectionStartMs;
+      if (speechDuration >= this.config.speechConfirmationMs) {
+        this.speechConfirmed = true;
+        this.silenceStartMs = null;
+        this.setDetecting(false);
+        this.setSpeaking(true);
+        this.callbacks.onSpeechStart?.();
+      }
+      return;
+    }
+
+    if (speechDetected && this.isSpeaking && this.speechConfirmed) {
+      // Continuing confirmed speech.
+      this.silenceStartMs = null;
+      return;
+    }
+
+    if (!speechDetected && !this.speechConfirmed && this.speechDetectionStartMs !== null) {
+      // Volume dropped during detection phase - apply grace period.
+      if (this.detectionSilenceStartMs === null) {
+        this.detectionSilenceStartMs = nowMs;
+        return;
+      }
+
+      const graceDuration = nowMs - this.detectionSilenceStartMs;
+      if (graceDuration >= this.config.detectionGracePeriodMs) {
+        // Cancel detection.
+        this.speechDetectionStartMs = null;
+        this.detectionSilenceStartMs = null;
+        this.audioBuffer = [];
+        this.bufferedBytes = 0;
+        this.setDetecting(false);
+      }
+      return;
+    }
+
+    if (!speechDetected && this.isSpeaking && this.speechConfirmed) {
+      // Potential speech END.
+      if (this.silenceStartMs === null) {
+        this.silenceStartMs = nowMs;
+        return;
+      }
+
+      const silenceDuration = nowMs - this.silenceStartMs;
+      if (silenceDuration >= this.config.silenceDurationMs) {
+        // Speech END confirmed.
+        this.speechConfirmed = false;
+        this.speechDetectionStartMs = null;
+        this.silenceStartMs = null;
+        this.setSpeaking(false);
+        this.callbacks.onSpeechEnd?.();
+        if (this.audioBuffer.length === 0) {
+          // Important: downstream expects an explicit segment boundary marker.
+          this.callbacks.onAudioSegment?.({ audioData: "", isLast: true });
+        } else {
+          this.flush(true);
+        }
+      }
+    }
+  }
+
+  stop(nowMs: number): void {
+    void nowMs;
+    if (this.config.enableContinuousStreaming) {
+      this.flush(true);
+    }
+    // Ensure observers see state clear transitions.
+    this.setDetecting(false);
+    this.setSpeaking(false);
+    this.reset();
+  }
+
+  private setDetecting(next: boolean): void {
+    if (this.isDetecting === next) {
+      return;
+    }
+    this.isDetecting = next;
+    this.callbacks.onDetectingChange?.(next);
+  }
+
+  private setSpeaking(next: boolean): void {
+    if (this.isSpeaking === next) {
+      return;
+    }
+    this.isSpeaking = next;
+    this.callbacks.onSpeakingChange?.(next);
+  }
+
+  private recomputeDerived(): void {
+    this.pcmBytesPerMs =
+      (this.config.pcmSampleRate *
+        this.config.pcmChannels *
+        (this.config.pcmBitsPerSample / 8)) /
+      1000;
+    this.minChunkBytes = Math.round(this.pcmBytesPerMs * this.config.minChunkDurationMs);
+  }
+}