Stream dictation via OpenAI Realtime and simplify retry UI

packages/server/src/client/daemon-client-v2.ts

@@ -42,7 +42,6 @@ import type {
   SendAgentMessage,
   SessionInboundMessage,
   SessionOutboundMessage,
-  TranscriptionResultMessage,
 } from "../shared/messages.js";
 import type {
   AgentPermissionRequest,
@@ -150,12 +149,6 @@ export type DaemonClientV2Config = {
 
 export type SendMessageOptions = Pick<SendAgentMessage, "messageId" | "images">;
 
-export type TranscribeAudioOptions = {
-  audio: string; // base64 encoded
-  format: string;
-  timeout?: number;
-};
-
 type AgentConfigOverrides = Partial<Omit<AgentSessionConfig, "provider" | "cwd">>;
 
 export type CreateAgentRequestOptions = {
@@ -188,7 +181,6 @@ type FileDownloadTokenPayload = FileDownloadTokenResponse["payload"];
 type ListProviderModelsPayload = ListProviderModelsResponseMessage["payload"];
 type ListCommandsPayload = ListCommandsResponse["payload"];
 type ExecuteCommandPayload = ExecuteCommandResponse["payload"];
-type TranscriptionResultPayload = TranscriptionResultMessage["payload"];
 type AgentPermissionResolvedPayload = AgentPermissionResolvedMessage["payload"];
 type ListTerminalsPayload = ListTerminalsResponse["payload"];
 type CreateTerminalPayload = CreateTerminalResponse["payload"];
@@ -472,7 +464,26 @@ export class DaemonClientV2 {
       throw new Error("Transport not connected");
     }
     const payload = SessionInboundMessageSchema.parse(message);
-    this.transport.send(JSON.stringify({ type: "session", message: payload }));
+    try {
+      this.transport.send(JSON.stringify({ type: "session", message: payload }));
+    } catch (error) {
+      if (this.config.suppressSendErrors) {
+        return;
+      }
+      throw error instanceof Error ? error : new Error(String(error));
+    }
+  }
+
+  private sendSessionMessageStrict(message: SessionInboundMessage): void {
+    if (!this.transport || this.connectionState.status !== "connected") {
+      throw new Error("Transport not connected");
+    }
+    const payload = SessionInboundMessageSchema.parse(message);
+    try {
+      this.transport.send(JSON.stringify({ type: "session", message: payload }));
+    } catch (error) {
+      throw error instanceof Error ? error : new Error(String(error));
+    }
   }
 
   sendUserMessage(text: string): void {
@@ -839,37 +850,6 @@ export class DaemonClientV2 {
     await this.sendAgentMessage(agentId, text, options);
   }
 
-  async transcribeAudio(
-    options: TranscribeAudioOptions
-  ): Promise<TranscriptionResultPayload> {
-    const requestId = this.createRequestId();
-    const timeout = options.timeout ?? 120000;
-
-    const responsePromise = this.waitFor(
-      (msg) => {
-        if (msg.type !== "transcription_result") {
-          return null;
-        }
-        if (msg.payload.requestId !== requestId) {
-          return null;
-        }
-        return msg.payload;
-      },
-      timeout,
-      { skipQueue: true }
-    );
-
-    const message = SessionInboundMessageSchema.parse({
-      type: "transcribe_audio_request",
-      audio: options.audio,
-      format: options.format,
-      requestId,
-    });
-    this.sendSessionMessage(message);
-
-    return responsePromise;
-  }
-
   async cancelAgent(agentId: string): Promise<void> {
     this.sendSessionMessage({ type: "cancel_agent_request", agentId });
   }
@@ -929,6 +909,87 @@ export class DaemonClientV2 {
     this.sendSessionMessage({ type: "realtime_audio_chunk", audio, format, isLast });
   }
 
+  startDictationStream(dictationId: string, format: string): Promise<void> {
+    const ackPromise = this.waitFor(
+      (msg) => {
+        if (msg.type !== "dictation_stream_ack") {
+          return null;
+        }
+        if (msg.payload.dictationId !== dictationId) {
+          return null;
+        }
+        if (msg.payload.ackSeq !== -1) {
+          return null;
+        }
+        return msg.payload;
+      },
+      30000,
+      { skipQueue: true }
+    ).then(() => undefined);
+
+    const errorPromise = this.waitFor(
+      (msg) => {
+        if (msg.type !== "dictation_stream_error") {
+          return null;
+        }
+        if (msg.payload.dictationId !== dictationId) {
+          return null;
+        }
+        return msg.payload;
+      },
+      30000,
+      { skipQueue: true }
+    ).then((payload) => {
+      throw new Error(payload.error);
+    });
+
+    this.sendSessionMessageStrict({ type: "dictation_stream_start", dictationId, format });
+    return Promise.race([ackPromise, errorPromise]);
+  }
+
+  sendDictationStreamChunk(dictationId: string, seq: number, audio: string, format: string): void {
+    this.sendSessionMessageStrict({ type: "dictation_stream_chunk", dictationId, seq, audio, format });
+  }
+
+  finishDictationStream(dictationId: string, finalSeq: number): Promise<{ dictationId: string; text: string }> {
+    const finalPromise = this.waitFor(
+      (msg) => {
+        if (msg.type !== "dictation_stream_final") {
+          return null;
+        }
+        if (msg.payload.dictationId !== dictationId) {
+          return null;
+        }
+        return msg.payload;
+      },
+      120000,
+      { skipQueue: true }
+    );
+
+    const errorPromise = this.waitFor(
+      (msg) => {
+        if (msg.type !== "dictation_stream_error") {
+          return null;
+        }
+        if (msg.payload.dictationId !== dictationId) {
+          return null;
+        }
+        return msg.payload;
+      },
+      120000,
+      { skipQueue: true }
+    ).then((payload) => {
+      throw new Error(payload.error);
+    });
+
+    this.sendSessionMessageStrict({ type: "dictation_stream_finish", dictationId, finalSeq });
+    return Promise.race([finalPromise, errorPromise]);
+  }
+
+  cancelDictationStream(dictationId: string): void {
+    this.sendSessionMessageStrict({ type: "dictation_stream_cancel", dictationId });
+  }
+
   async abortRequest(): Promise<void> {
     this.sendSessionMessage({ type: "abort_request" });
   }
@@ -2097,7 +2158,12 @@ function createWebSocketTransportFactory(
   return ({ url, headers }) => {
     const ws = factory(url, { headers });
     return {
-      send: (data) => ws.send(data),
+      send: (data) => {
+        if (typeof ws.readyState === "number" && ws.readyState !== 1) {
+          throw new Error(`WebSocket not open (readyState=${ws.readyState})`);
+        }
+        ws.send(data);
+      },
       close: (code?: number, reason?: string) => ws.close(code, reason),
       onOpen: (handler) => bindWsHandler(ws, "open", handler),
       onClose: (handler) => bindWsHandler(ws, "close", handler),

packages/server/src/server/agent/openai-realtime-transcription.ts

@@ -0,0 +1,207 @@
+import type pino from "pino";
+import WebSocket from "ws";
+import { EventEmitter } from "node:events";
+
+type OpenAIClientEvent =
+  | {
+      type: "session.update";
+      session: {
+        type: "transcription";
+        audio: {
+          input: {
+            format: { type: "audio/pcm"; rate: 24000 };
+            transcription: {
+              model: string;
+              language?: string;
+              prompt?: string;
+            };
+            turn_detection: null;
+          };
+        };
+      };
+    }
+  | { type: "input_audio_buffer.append"; audio: string }
+  | { type: "input_audio_buffer.commit" }
+  | { type: "input_audio_buffer.clear" };
+
+type OpenAIServerEvent =
+  | { type: "session.created" | "session.updated" }
+  | {
+      type: "input_audio_buffer.committed";
+      item_id: string;
+      previous_item_id: string | null;
+    }
+  | {
+      type: "conversation.item.input_audio_transcription.completed";
+      item_id: string;
+      transcript: string;
+    }
+  | { type: "error"; error?: { message?: string } };
+
+export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
+  private readonly apiKey: string;
+  private readonly logger: pino.Logger;
+  private readonly transcriptionModel: string;
+  private readonly language?: string;
+
+  private ws: WebSocket | null = null;
+  private ready: Promise<void> | null = null;
+  private closing = false;
+
+  constructor(params: {
+    apiKey: string;
+    logger: pino.Logger;
+    transcriptionModel: string;
+    language?: string;
+  }) {
+    super();
+    this.apiKey = params.apiKey;
+    this.logger = params.logger.child({ provider: "openai", component: "realtime-transcription" });
+    this.transcriptionModel = params.transcriptionModel;
+    this.language = params.language;
+  }
+
+  public async connect(): Promise<void> {
+    if (this.ready) {
+      return this.ready;
+    }
+
+    this.closing = false;
+    this.ready = new Promise<void>((resolve, reject) => {
+      const url = "wss://api.openai.com/v1/realtime?intent=transcription";
+      const ws = new WebSocket(url, {
+        headers: {
+          Authorization: `Bearer ${this.apiKey}`,
+        },
+      });
+      this.ws = ws;
+
+      let resolved = false;
+
+      const fail = (error: Error) => {
+        if (resolved) {
+          this.emit("error", error);
+          return;
+        }
+        resolved = true;
+        reject(error);
+      };
+
+      ws.on("open", () => {
+        this.logger.debug("OpenAI realtime transcription websocket connected");
+        const update: OpenAIClientEvent = {
+          type: "session.update",
+          session: {
+            type: "transcription",
+            audio: {
+              input: {
+                format: { type: "audio/pcm", rate: 24000 },
+                transcription: {
+                  model: this.transcriptionModel,
+                  ...(this.language ? { language: this.language } : {}),
+                },
+                // We commit periodically ourselves; no server-side VAD for dictation.
+                turn_detection: null,
+              },
+            },
+          },
+        };
+        ws.send(JSON.stringify(update));
+      });
+
+      ws.on("message", (data) => {
+        const text = typeof data === "string" ? data : data.toString("utf-8");
+        let parsed: unknown;
+        try {
+          parsed = JSON.parse(text);
+        } catch {
+          return;
+        }
+
+        const event = parsed as OpenAIServerEvent;
+        if (
+          event.type === "session.created" ||
+          event.type === "session.updated"
+        ) {
+          if (!resolved) {
+            resolved = true;
+            resolve();
+          }
+          return;
+        }
+
+        if (event.type === "input_audio_buffer.committed") {
+          this.emit("committed", {
+            itemId: event.item_id,
+            previousItemId: event.previous_item_id,
+          });
+          return;
+        }
+
+        if (event.type === "conversation.item.input_audio_transcription.completed") {
+          this.emit("transcript", { itemId: event.item_id, transcript: event.transcript });
+          return;
+        }
+
+        if (event.type === "error") {
+          const message = event.error?.message ?? "OpenAI realtime error";
+          fail(new Error(message));
+        }
+      });
+
+      ws.on("error", (err) => {
+        fail(err instanceof Error ? err : new Error(String(err)));
+      });
+
+      ws.on("close", () => {
+        this.logger.debug("OpenAI realtime websocket closed");
+        if (this.closing) {
+          return;
+        }
+        if (!resolved) {
+          fail(new Error("OpenAI realtime websocket closed before ready"));
+          return;
+        }
+        fail(new Error("OpenAI realtime websocket closed"));
+      });
+    });
+
+    return this.ready;
+  }
+
+  public appendPcm16Base64(base64Audio: string): void {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      throw new Error("OpenAI realtime websocket not connected");
+    }
+    const event: OpenAIClientEvent = { type: "input_audio_buffer.append", audio: base64Audio };
+    this.ws.send(JSON.stringify(event));
+  }
+
+  public commit(): void {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      throw new Error("OpenAI realtime websocket not connected");
+    }
+    const event: OpenAIClientEvent = { type: "input_audio_buffer.commit" };
+    this.ws.send(JSON.stringify(event));
+  }
+
+  public clear(): void {
+    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+      return;
+    }
+    const event: OpenAIClientEvent = { type: "input_audio_buffer.clear" };
+    this.ws.send(JSON.stringify(event));
+  }
+
+  public close(): void {
+    try {
+      this.closing = true;
+      this.ws?.close();
+    } catch {
+      // no-op
+    } finally {
+      this.ws = null;
+      this.ready = null;
+    }
+  }
+}

packages/server/src/server/agent/pcm16-resampler.ts

@@ -0,0 +1,83 @@
+export class Pcm16MonoResampler {
+  private readonly inputRate: number;
+  private readonly outputRate: number;
+  private readonly step: number;
+  private pos: number;
+  private carrySample: number | null;
+
+  constructor(params: { inputRate: number; outputRate: number }) {
+    this.inputRate = params.inputRate;
+    this.outputRate = params.outputRate;
+    this.step = this.inputRate / this.outputRate;
+    this.pos = 0;
+    this.carrySample = null;
+  }
+
+  public reset(): void {
+    this.pos = 0;
+    this.carrySample = null;
+  }
+
+  public processChunk(pcm16le: Buffer): Buffer {
+    if (pcm16le.length === 0) {
+      return Buffer.alloc(0);
+    }
+    if (pcm16le.length % 2 !== 0) {
+      throw new Error(`PCM16 chunk byteLength must be even, got ${pcm16le.length}`);
+    }
+
+    const srcChunk = new Int16Array(
+      pcm16le.buffer,
+      pcm16le.byteOffset,
+      pcm16le.byteLength / 2
+    );
+
+    const hasCarry = this.carrySample !== null;
+    const srcLen = srcChunk.length + (hasCarry ? 1 : 0);
+    if (srcLen < 2) {
+      this.carrySample = srcChunk.length ? srcChunk[srcChunk.length - 1] : this.carrySample;
+      return Buffer.alloc(0);
+    }
+
+    const src = new Float32Array(srcLen);
+    let offset = 0;
+    if (hasCarry) {
+      src[0] = (this.carrySample as number) / 32768;
+      offset = 1;
+    }
+    for (let i = 0; i < srcChunk.length; i += 1) {
+      src[offset + i] = srcChunk[i] / 32768;
+    }
+
+    const out: number[] = [];
+    const maxPos = src.length - 1;
+
+    while (this.pos < maxPos) {
+      const i = Math.floor(this.pos);
+      const frac = this.pos - i;
+      const s0 = src[i]!;
+      const s1 = src[i + 1]!;
+      const sample = s0 + (s1 - s0) * frac;
+      const clamped = Math.max(-1, Math.min(1, sample));
+      const int16 = Math.round(clamped * 32767);
+      out.push(int16);
+      this.pos += this.step;
+    }
+
+    // Keep the last input sample as carry for the next chunk.
+    const lastInput = srcChunk[srcChunk.length - 1]!;
+    this.carrySample = lastInput;
+
+    // Shift position so next chunk (which will include carry sample) continues smoothly.
+    const shift = src.length - 1;
+    this.pos = this.pos - shift;
+    if (this.pos < 0) {
+      // Guard against floating point drift.
+      this.pos = 0;
+    }
+
+    const outArr = Int16Array.from(out);
+    return Buffer.from(outArr.buffer, outArr.byteOffset, outArr.byteLength);
+  }
+}
+

packages/server/src/server/daemon-client-v2.e2e.test.ts

@@ -23,6 +23,9 @@ if (envPath) {
   dotenv.config({ path: envPath });
 }
 
+// Make dictation streaming commit frequently in tests so we exercise multi-commit assembly logic.
+process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ??= "1000";
+
 function tmpCwd(): string {
   return mkdtempSync(path.join(tmpdir(), "daemon-client-v2-"));
 }
@@ -630,6 +633,216 @@ describe("daemon client v2 E2E", () => {
     180000
   );
 
+  test(
+    "streams dictation PCM and returns final transcript via OpenAI Realtime transcription",
+    async () => {
+      if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+        // Requires OpenAI realtime transcription and is inherently network-flaky.
+        return;
+      }
+      if (!process.env.OPENAI_API_KEY) {
+        return;
+      }
+
+      const fixturePath = path.resolve(
+        process.cwd(),
+        "..",
+        "app",
+        "e2e",
+        "fixtures",
+        "recording.wav"
+      );
+      const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
+
+      const parsePcm16MonoWav = (buffer: Buffer): Buffer => {
+        if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
+          throw new Error("Invalid WAV header");
+        }
+        let offset = 12;
+        let fmt: { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } | null = null;
+        let dataChunk: Buffer | null = null;
+
+        while (offset + 8 <= buffer.length) {
+          const id = buffer.toString("ascii", offset, offset + 4);
+          const size = buffer.readUInt32LE(offset + 4);
+          const payloadStart = offset + 8;
+          const payloadEnd = payloadStart + size;
+          if (payloadEnd > buffer.length) {
+            break;
+          }
+
+          if (id === "fmt ") {
+            const audioFormat = buffer.readUInt16LE(payloadStart);
+            const channels = buffer.readUInt16LE(payloadStart + 2);
+            const sampleRate = buffer.readUInt32LE(payloadStart + 4);
+            const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
+            fmt = { audioFormat, channels, sampleRate, bitsPerSample };
+          } else if (id === "data") {
+            dataChunk = buffer.subarray(payloadStart, payloadEnd);
+          }
+
+          offset = payloadEnd + (size % 2);
+        }
+
+        if (!fmt || !dataChunk) {
+          throw new Error("Missing WAV fmt/data chunks");
+        }
+        if (fmt.audioFormat !== 1) {
+          throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
+        }
+        if (fmt.channels !== 1 || fmt.sampleRate !== 16000 || fmt.bitsPerSample !== 16) {
+          throw new Error(
+            `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
+          );
+        }
+        if (dataChunk.length % 2 !== 0) {
+          throw new Error("WAV PCM16 data length must be even");
+        }
+        return dataChunk;
+      };
+
+      const pcm16 = parsePcm16MonoWav(wav);
+      const dictationId = `dict-${Date.now()}`;
+      const format = "audio/pcm;rate=16000;bits=16";
+
+      await ctx.client.startDictationStream(dictationId, format);
+
+      const chunkBytes = 3200; // ~100ms @ 16kHz mono PCM16 (1600 samples * 2 bytes)
+      let seq = 0;
+      for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
+        const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
+        ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
+        seq += 1;
+      }
+
+      const finalSeq = seq - 1;
+      const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
+
+      expect(result.dictationId).toBe(dictationId);
+      expect(result.text.toLowerCase()).toContain("voice note");
+    },
+    180000
+  );
+
+  test(
+    "does not commit an empty OpenAI realtime audio buffer on finish",
+    async () => {
+      if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+        return;
+      }
+      if (!process.env.OPENAI_API_KEY) {
+        return;
+      }
+
+      const fixturePath = path.resolve(
+        process.cwd(),
+        "..",
+        "app",
+        "e2e",
+        "fixtures",
+        "recording.wav"
+      );
+      const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath));
+
+      const parsePcm16MonoWav = (buffer: Buffer): Buffer => {
+        if (buffer.toString("ascii", 0, 4) !== "RIFF" || buffer.toString("ascii", 8, 12) !== "WAVE") {
+          throw new Error("Invalid WAV header");
+        }
+        let offset = 12;
+        let fmt: { audioFormat: number; channels: number; sampleRate: number; bitsPerSample: number } | null = null;
+        let dataChunk: Buffer | null = null;
+
+        while (offset + 8 <= buffer.length) {
+          const id = buffer.toString("ascii", offset, offset + 4);
+          const size = buffer.readUInt32LE(offset + 4);
+          const payloadStart = offset + 8;
+          const payloadEnd = payloadStart + size;
+          if (payloadEnd > buffer.length) {
+            break;
+          }
+
+          if (id === "fmt ") {
+            const audioFormat = buffer.readUInt16LE(payloadStart);
+            const channels = buffer.readUInt16LE(payloadStart + 2);
+            const sampleRate = buffer.readUInt32LE(payloadStart + 4);
+            const bitsPerSample = buffer.readUInt16LE(payloadStart + 14);
+            fmt = { audioFormat, channels, sampleRate, bitsPerSample };
+          } else if (id === "data") {
+            dataChunk = buffer.subarray(payloadStart, payloadEnd);
+          }
+
+          offset = payloadEnd + (size % 2);
+        }
+
+        if (!fmt || !dataChunk) {
+          throw new Error("Missing WAV fmt/data chunks");
+        }
+        if (fmt.audioFormat !== 1) {
+          throw new Error(`Unsupported WAV encoding (audioFormat=${fmt.audioFormat})`);
+        }
+        if (fmt.channels !== 1 || fmt.sampleRate !== 16000 || fmt.bitsPerSample !== 16) {
+          throw new Error(
+            `Unexpected WAV format: channels=${fmt.channels} rate=${fmt.sampleRate} bits=${fmt.bitsPerSample}`
+          );
+        }
+        if (dataChunk.length % 2 !== 0) {
+          throw new Error("WAV PCM16 data length must be even");
+        }
+        return dataChunk;
+      };
+
+      const pcm16 = parsePcm16MonoWav(wav);
+      const dictationId = `dict-empty-commit-${Date.now()}`;
+      const format = "audio/pcm;rate=16000;bits=16";
+
+      await ctx.client.startDictationStream(dictationId, format);
+
+      // Send exactly 10x 100ms chunks. With OPENAI_REALTIME_DICTATION_COMMIT_MS=1000 in this test file,
+      // the server will auto-commit at the 1s boundary. Finishing immediately after that should not
+      // attempt an additional empty commit.
+      const chunkBytes = 3200; // 100ms @ 16kHz mono PCM16
+      const targetChunks = 10;
+      let seq = 0;
+      for (let i = 0; i < targetChunks; i += 1) {
+        const start = i * chunkBytes;
+        const end = Math.min(pcm16.length, start + chunkBytes);
+        const chunk = pcm16.subarray(start, end);
+        ctx.client.sendDictationStreamChunk(dictationId, seq, chunk.toString("base64"), format);
+        seq += 1;
+      }
+
+      const finalSeq = seq - 1;
+      const result = await ctx.client.finishDictationStream(dictationId, finalSeq);
+
+      expect(result.dictationId).toBe(dictationId);
+      expect(typeof result.text).toBe("string");
+    },
+    180000
+  );
+
+  test(
+    "fails fast if dictation finishes without sending required chunks",
+    async () => {
+      if (process.env.PASEO_E2E_DICTATION_REALTIME !== "1") {
+        return;
+      }
+      if (!process.env.OPENAI_API_KEY) {
+        return;
+      }
+
+      const dictationId = `dict-missing-chunks-${Date.now()}`;
+      const format = "audio/pcm;rate=16000;bits=16";
+
+      await ctx.client.startDictationStream(dictationId, format);
+
+      // Claim that we sent chunk 0, but actually send no chunks.
+      await expect(ctx.client.finishDictationStream(dictationId, 0)).rejects.toThrow(
+        /no audio chunks were received/i
+      );
+    },
+    180000
+  );
+
   test(
     "supports git and file operations",
     async () => {

packages/server/src/server/session.ts

@@ -35,6 +35,8 @@ import { TTSManager } from "./agent/tts-manager.js";
 import { STTManager } from "./agent/stt-manager.js";
 import type { OpenAISTT } from "./agent/stt-openai.js";
 import type { OpenAITTS } from "./agent/tts-openai.js";
+import { OpenAIRealtimeTranscriptionSession } from "./agent/openai-realtime-transcription.js";
+import { Pcm16MonoResampler } from "./agent/pcm16-resampler.js";
 import type { VoiceConversationStore } from "./voice-conversation-store.js";
 import {
   buildConfigOverrides,
@@ -135,6 +137,21 @@ const MIN_STREAMING_SEGMENT_DURATION_MS = 1000;
 const MIN_STREAMING_SEGMENT_BYTES = Math.round(
   PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
 );
+
+const DICTATION_PCM_INPUT_RATE = 16000;
+const DICTATION_PCM_OUTPUT_RATE = 24000;
+const DICTATION_COMMIT_INTERVAL_MS = Number.parseInt(
+  process.env.OPENAI_REALTIME_DICTATION_COMMIT_MS ?? "5000",
+  10
+);
+const DICTATION_COMMIT_TARGET_BYTES = Math.round(
+  (DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_COMMIT_INTERVAL_MS) / 1000
+);
+const DICTATION_MIN_COMMIT_MS = 100;
+const DICTATION_MIN_COMMIT_BYTES = Math.round(
+  (DICTATION_PCM_OUTPUT_RATE * 2 * DICTATION_MIN_COMMIT_MS) / 1000
+);
+const DICTATION_FINAL_TIMEOUT_MS = 120000;
 const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
 
 /**
@@ -249,6 +266,27 @@ export class Session {
   private speechInProgress = false;
   private voiceConversationId: string | null = null;
 
+  private dictationStreams = new Map<
+    string,
+    {
+      dictationId: string;
+      inputFormat: string;
+      openai: OpenAIRealtimeTranscriptionSession;
+      resampler: Pcm16MonoResampler;
+      receivedChunks: Map<number, Buffer>;
+      nextSeqToForward: number;
+      ackSeq: number;
+      bytesSinceCommit: number;
+      expectedCommits: number;
+      committedCount: number;
+      committedItemIds: string[];
+      transcriptsByItemId: Map<string, string>;
+      finishRequested: boolean;
+      finalSeq: number | null;
+      finalTimeout: ReturnType<typeof setTimeout> | null;
+    }
+  >();
+
   // Audio buffering for interruption handling
   private pendingAudioSegments: Array<{ audio: Buffer; format: string }> = [];
   private bufferTimeout: NodeJS.Timeout | null = null;
@@ -735,8 +773,20 @@ export class Session {
           );
           break;
 
-        case "transcribe_audio_request":
-          await this.handleTranscribeAudio(msg);
+        case "dictation_stream_start":
+          await this.handleDictationStreamStart(msg);
+          break;
+
+        case "dictation_stream_chunk":
+          await this.handleDictationStreamChunk(msg);
+          break;
+
+        case "dictation_stream_finish":
+          await this.handleDictationStreamFinish(msg);
+          break;
+
+        case "dictation_stream_cancel":
+          await this.handleDictationStreamCancel(msg);
           break;
 
         case "create_agent_request":
@@ -1177,56 +1227,279 @@ export class Session {
     this.startAgentStream(agentId, prompt);
   }
 
-  /**
-   * Handle audio transcription request
-   */
-  private async handleTranscribeAudio(
-    msg: Extract<SessionInboundMessage, { type: "transcribe_audio_request" }>
-  ): Promise<void> {
-    const { audio, format, requestId } = msg;
+  private emitDictationAck(dictationId: string, ackSeq: number): void {
+    this.emit({
+      type: "dictation_stream_ack",
+      payload: { dictationId, ackSeq },
+    });
+  }
 
-    const audioBuffer = Buffer.from(audio, "base64");
+  private failDictationStream(dictationId: string, error: string, retryable: boolean): void {
+    this.emit({
+      type: "dictation_stream_error",
+      payload: { dictationId, error, retryable },
+    });
+  }
 
-    this.sessionLogger.debug({ requestId }, "Transcribing audio");
-
-    try {
-      const result = await this.sttManager.transcribe(audioBuffer, format, {
-        requestId,
-        label: "dictation",
-      });
-
-      this.sessionLogger.info(
-        { requestId, textLength: result.text.length },
-        "Transcription complete"
-      );
-
-      this.emit({
-        type: "transcription_result",
-        payload: {
-          text: result.text,
-          language: result.language,
-          duration: result.duration,
-          requestId,
-          avgLogprob: result.avgLogprob,
-          isLowConfidence: result.isLowConfidence,
-          byteLength: result.byteLength,
-          format: result.format,
-          debugRecordingPath: result.debugRecordingPath,
-        },
-      });
-    } catch (error: any) {
-      this.sessionLogger.error({ err: error, requestId }, "Transcription failed");
-      this.emit({
-        type: "activity_log",
-        payload: {
-          id: uuidv4(),
-          timestamp: new Date(),
-          type: "error",
-          content: `Transcription failed: ${error.message}`,
-        },
-      });
-      throw error;
+  private cleanupDictationStream(dictationId: string): void {
+    const state = this.dictationStreams.get(dictationId) ?? null;
+    if (!state) {
+      return;
     }
+    if (state.finalTimeout) {
+      clearTimeout(state.finalTimeout);
+    }
+    try {
+      state.openai.close();
+    } catch {
+      // no-op
+    }
+    this.dictationStreams.delete(dictationId);
+  }
+
+  private maybeFinalizeDictationStream(dictationId: string): void {
+    const state = this.dictationStreams.get(dictationId);
+    if (!state) {
+      return;
+    }
+
+    if (!state.finishRequested || state.finalSeq === null) {
+      return;
+    }
+    if (state.ackSeq < state.finalSeq) {
+      return;
+    }
+    if (state.committedCount < state.expectedCommits) {
+      return;
+    }
+
+    const allTranscriptsReady = state.committedItemIds.every((itemId) =>
+      state.transcriptsByItemId.has(itemId)
+    );
+    if (!allTranscriptsReady) {
+      return;
+    }
+
+    const orderedText = state.committedItemIds
+      .map((itemId) => state.transcriptsByItemId.get(itemId) ?? "")
+      .join(" ")
+      .trim();
+
+    this.emit({
+      type: "dictation_stream_final",
+      payload: { dictationId, text: orderedText },
+    });
+
+    this.cleanupDictationStream(dictationId);
+  }
+
+  private async handleDictationStreamStart(
+    msg: Extract<SessionInboundMessage, { type: "dictation_stream_start" }>
+  ): Promise<void> {
+    const dictationId = msg.dictationId;
+    this.cleanupDictationStream(dictationId);
+
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      this.failDictationStream(dictationId, "OPENAI_API_KEY not set", false);
+      return;
+    }
+
+    const transcriptionModel =
+      process.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ?? "gpt-4o-transcribe";
+
+    const openai = new OpenAIRealtimeTranscriptionSession({
+      apiKey,
+      logger: this.sessionLogger.child({ dictationId }),
+      transcriptionModel,
+      language: "en",
+    });
+
+    openai.on("committed", ({ itemId }: { itemId: string }) => {
+      const state = this.dictationStreams.get(dictationId);
+      if (!state) {
+        return;
+      }
+      state.committedCount += 1;
+      state.committedItemIds.push(itemId);
+      this.maybeFinalizeDictationStream(dictationId);
+    });
+
+    openai.on("transcript", ({ itemId, transcript }: { itemId: string; transcript: string }) => {
+      const state = this.dictationStreams.get(dictationId);
+      if (!state) {
+        return;
+      }
+      state.transcriptsByItemId.set(itemId, transcript);
+      this.maybeFinalizeDictationStream(dictationId);
+    });
+
+    openai.on("error", (err) => {
+      const error = err instanceof Error ? err.message : String(err);
+      this.failDictationStream(dictationId, error, true);
+      this.cleanupDictationStream(dictationId);
+    });
+
+    await openai.connect();
+
+    this.dictationStreams.set(dictationId, {
+      dictationId,
+      inputFormat: msg.format,
+      openai,
+      resampler: new Pcm16MonoResampler({
+        inputRate: DICTATION_PCM_INPUT_RATE,
+        outputRate: DICTATION_PCM_OUTPUT_RATE,
+      }),
+      receivedChunks: new Map(),
+      nextSeqToForward: 0,
+      ackSeq: -1,
+      bytesSinceCommit: 0,
+      expectedCommits: 0,
+      committedCount: 0,
+      committedItemIds: [],
+      transcriptsByItemId: new Map(),
+      finishRequested: false,
+      finalSeq: null,
+      finalTimeout: null,
+    });
+
+    this.emitDictationAck(dictationId, -1);
+  }
+
+  private async handleDictationStreamChunk(
+    msg: Extract<SessionInboundMessage, { type: "dictation_stream_chunk" }>
+  ): Promise<void> {
+    const state = this.dictationStreams.get(msg.dictationId);
+    if (!state) {
+      this.failDictationStream(msg.dictationId, "Dictation stream not started", true);
+      return;
+    }
+
+    if (msg.format !== state.inputFormat) {
+      this.failDictationStream(
+        msg.dictationId,
+        `Mismatched dictation stream format: ${msg.format}`,
+        false
+      );
+      return;
+    }
+
+    if (msg.seq < state.nextSeqToForward) {
+      this.emitDictationAck(msg.dictationId, state.ackSeq);
+      return;
+    }
+
+    if (!state.receivedChunks.has(msg.seq)) {
+      state.receivedChunks.set(msg.seq, Buffer.from(msg.audio, "base64"));
+    }
+
+    while (state.receivedChunks.has(state.nextSeqToForward)) {
+      const seq = state.nextSeqToForward;
+      const pcm16 = state.receivedChunks.get(seq)!;
+      state.receivedChunks.delete(seq);
+
+      const resampled = state.resampler.processChunk(pcm16);
+      if (resampled.length > 0) {
+        state.openai.appendPcm16Base64(resampled.toString("base64"));
+        state.bytesSinceCommit += resampled.length;
+      }
+
+      state.nextSeqToForward += 1;
+      state.ackSeq = state.nextSeqToForward - 1;
+
+      if (state.bytesSinceCommit >= DICTATION_COMMIT_TARGET_BYTES) {
+        state.expectedCommits += 1;
+        state.openai.commit();
+        state.bytesSinceCommit = 0;
+      }
+    }
+
+    this.emitDictationAck(msg.dictationId, state.ackSeq);
+    this.maybeFinalizeDictationStream(msg.dictationId);
+  }
+
+  private async handleDictationStreamFinish(
+    msg: Extract<SessionInboundMessage, { type: "dictation_stream_finish" }>
+  ): Promise<void> {
+    const state = this.dictationStreams.get(msg.dictationId);
+    if (!state) {
+      this.failDictationStream(msg.dictationId, "Dictation stream not started", true);
+      return;
+    }
+
+    state.finishRequested = true;
+    state.finalSeq = msg.finalSeq;
+
+    // If the client claims it sent audio (finalSeq >= 0) but we haven't received any chunks at all,
+    // we should fail fast instead of hanging until DICTATION_FINAL_TIMEOUT_MS.
+    if (msg.finalSeq >= 0 && state.ackSeq < 0 && state.nextSeqToForward === 0 && state.receivedChunks.size === 0) {
+      this.sessionLogger.debug(
+        {
+          dictationId: msg.dictationId,
+          finalSeq: msg.finalSeq,
+          ackSeq: state.ackSeq,
+          nextSeqToForward: state.nextSeqToForward,
+          receivedChunks: state.receivedChunks.size,
+          expectedCommits: state.expectedCommits,
+          committedCount: state.committedCount,
+        },
+        "Dictation finish: no chunks received (failing fast)"
+      );
+      this.failDictationStream(
+        msg.dictationId,
+        `Dictation finished (finalSeq=${msg.finalSeq}) but no audio chunks were received`,
+        true
+      );
+      this.cleanupDictationStream(msg.dictationId);
+      return;
+    }
+
+    // Commit any remaining uncommitted audio.
+    // IMPORTANT: OpenAI requires at least ~100ms of audio in the buffer for a commit.
+    // Also, committing an empty buffer throws "buffer too small ... 0.00ms".
+    if (state.bytesSinceCommit > 0) {
+      this.sessionLogger.debug(
+        { dictationId: msg.dictationId, bytesSinceCommit: state.bytesSinceCommit },
+        "Dictation finish: committing pending audio"
+      );
+      if (state.bytesSinceCommit < DICTATION_MIN_COMMIT_BYTES) {
+        const padBytes = DICTATION_MIN_COMMIT_BYTES - state.bytesSinceCommit;
+        this.sessionLogger.debug(
+          { dictationId: msg.dictationId, padBytes },
+          "Dictation finish: padding to minimum commit size"
+        );
+        state.openai.appendPcm16Base64(Buffer.alloc(padBytes).toString("base64"));
+        state.bytesSinceCommit += padBytes;
+      }
+      state.expectedCommits += 1;
+      state.openai.commit();
+      state.bytesSinceCommit = 0;
+    } else {
+      this.sessionLogger.debug(
+        { dictationId: msg.dictationId },
+        "Dictation finish: no pending audio to commit"
+      );
+    }
+
+    if (state.finalTimeout) {
+      clearTimeout(state.finalTimeout);
+    }
+    state.finalTimeout = setTimeout(() => {
+      this.failDictationStream(
+        msg.dictationId,
+        "Timed out waiting for final transcription",
+        true
+      );
+      this.cleanupDictationStream(msg.dictationId);
+    }, DICTATION_FINAL_TIMEOUT_MS);
+
+    this.maybeFinalizeDictationStream(msg.dictationId);
+  }
+
+  private async handleDictationStreamCancel(
+    msg: Extract<SessionInboundMessage, { type: "dictation_stream_cancel" }>
+  ): Promise<void> {
+    this.cleanupDictationStream(msg.dictationId);
   }
 
   /**
@@ -4209,6 +4482,10 @@ export class Session {
     this.ttsManager.cleanup();
     this.sttManager.cleanup();
 
+    for (const dictationId of this.dictationStreams.keys()) {
+      this.cleanupDictationStream(dictationId);
+    }
+
     // Close MCP clients
     if (this.agentMcpClient) {
       try {

packages/server/src/server/websocket-server.ts

@@ -57,7 +57,13 @@ export class VoiceAssistantWebSocketServer {
       path: "/ws",
       verifyClient: ({ req }, callback) => {
         const origin = req.headers.origin;
-        if (!origin || allowedOrigins.has(origin)) {
+        const requestHost = typeof req.headers.host === "string" ? req.headers.host : null;
+        const sameOrigin =
+          !!origin &&
+          !!requestHost &&
+          (origin === `http://${requestHost}` || origin === `https://${requestHost}`);
+
+        if (!origin || allowedOrigins.has(origin) || sameOrigin) {
           callback(true);
         } else {
           this.logger.warn({ origin }, "Rejected connection from origin");

packages/server/src/shared/messages.ts

@@ -329,11 +329,33 @@ export const SendAgentMessageSchema = z.object({
   })).optional(),
 });
 
-export const TranscribeAudioRequestSchema = z.object({
-  type: z.literal("transcribe_audio_request"),
-  audio: z.string(), // base64 encoded
-  format: z.string(),
-  requestId: z.string(),
+// ============================================================================
+// Dictation Streaming (lossless, resumable)
+// ============================================================================
+
+export const DictationStreamStartMessageSchema = z.object({
+  type: z.literal("dictation_stream_start"),
+  dictationId: z.string(),
+  format: z.string(), // e.g. "audio/pcm;rate=16000;bits=16"
+});
+
+export const DictationStreamChunkMessageSchema = z.object({
+  type: z.literal("dictation_stream_chunk"),
+  dictationId: z.string(),
+  seq: z.number().int().nonnegative(),
+  audio: z.string(), // base64 encoded chunk
+  format: z.string(), // e.g. "audio/pcm;rate=16000;bits=16"
+});
+
+export const DictationStreamFinishMessageSchema = z.object({
+  type: z.literal("dictation_stream_finish"),
+  dictationId: z.string(),
+  finalSeq: z.number().int().nonnegative(),
+});
+
+export const DictationStreamCancelMessageSchema = z.object({
+  type: z.literal("dictation_stream_cancel"),
+  dictationId: z.string(),
 });
 
 const GitSetupOptionsSchema = z.object({
@@ -683,7 +705,10 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
   DeleteAgentRequestMessageSchema,
   SetVoiceConversationMessageSchema,
   SendAgentMessageSchema,
-  TranscribeAudioRequestSchema,
+  DictationStreamStartMessageSchema,
+  DictationStreamChunkMessageSchema,
+  DictationStreamFinishMessageSchema,
+  DictationStreamCancelMessageSchema,
   CreateAgentRequestMessageSchema,
   ListProviderModelsRequestMessageSchema,
   ResumeAgentRequestMessageSchema,
@@ -780,6 +805,31 @@ export const TranscriptionResultMessageSchema = z.object({
   }),
 });
 
+export const DictationStreamAckMessageSchema = z.object({
+  type: z.literal("dictation_stream_ack"),
+  payload: z.object({
+    dictationId: z.string(),
+    ackSeq: z.number().int(),
+  }),
+});
+
+export const DictationStreamFinalMessageSchema = z.object({
+  type: z.literal("dictation_stream_final"),
+  payload: z.object({
+    dictationId: z.string(),
+    text: z.string(),
+  }),
+});
+
+export const DictationStreamErrorMessageSchema = z.object({
+  type: z.literal("dictation_stream_error"),
+  payload: z.object({
+    dictationId: z.string(),
+    error: z.string(),
+    retryable: z.boolean(),
+  }),
+});
+
 export const StatusMessageSchema = z.object({
   type: z.literal("status"),
   payload: z
@@ -1244,6 +1294,9 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
   AssistantChunkMessageSchema,
   AudioOutputMessageSchema,
   TranscriptionResultMessageSchema,
+  DictationStreamAckMessageSchema,
+  DictationStreamFinalMessageSchema,
+  DictationStreamErrorMessageSchema,
   StatusMessageSchema,
   InitializeAgentResponseMessageSchema,
   ArtifactMessageSchema,
@@ -1323,7 +1376,10 @@ export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
 export type UserTextMessage = z.infer<typeof UserTextMessageSchema>;
 export type RealtimeAudioChunkMessage = z.infer<typeof RealtimeAudioChunkMessageSchema>;
 export type SendAgentMessage = z.infer<typeof SendAgentMessageSchema>;
-export type TranscribeAudioRequest = z.infer<typeof TranscribeAudioRequestSchema>;
+export type DictationStreamStartMessage = z.infer<typeof DictationStreamStartMessageSchema>;
+export type DictationStreamChunkMessage = z.infer<typeof DictationStreamChunkMessageSchema>;
+export type DictationStreamFinishMessage = z.infer<typeof DictationStreamFinishMessageSchema>;
+export type DictationStreamCancelMessage = z.infer<typeof DictationStreamCancelMessageSchema>;
 export type CreateAgentRequestMessage = z.infer<typeof CreateAgentRequestMessageSchema>;
 export type ListProviderModelsRequestMessage = z.infer<
   typeof ListProviderModelsRequestMessageSchema