refactor(server): extract voice mode subsystem into VoiceSession (#1640)

* refactor(server): extract voice mode subsystem into VoiceSession

session.ts was a 10.5k-line god object and the most-churned file in the
repo. ~1,100 of those lines were an entire voice/audio subsystem — the
STT/TTS/dictation managers, the barge-in audio-buffering state machine,
voice-turn orchestration, and the MCP voice bridge — interleaved field by
field and method by method with workspace/git/agent/provider concerns.

Move the whole subsystem into a new VoiceSession deep module. Session now
holds one `voiceSession` field, constructs it once, and delegates the
voice/dictation/abort message types, the permission auto-allow gate, and
cleanup to it. VoiceSession owns all 18 voice fields and ~25 methods and
reaches back into the agent run only through a narrow VoiceSessionHost
seam (emit, loadAgent, reloadAgentSession, sendSpokenInput,
interruptAgentIfRunning, hasActiveAgentRun).

Behavior is preserved (verbatim method moves); deleting voice-session.ts
now removes the feature mechanically. The voice unit tests drive the
VoiceSession boundary instead of reaching into Session internals, so
future voice tests can construct a VoiceSession with a fake host.

session.ts: 10,468 -> 9,272 lines.

* refactor(server): move VoiceSession into voice/ and test it at the boundary

Address Greptile review on #1640:

- Move voice-session.ts into the existing voice/ subdirectory, alongside
  voice-turn-controller.ts, instead of adding another peer to the 30+ file
  server/ directory. The directory now carries the domain.
- Add voice/voice-session.test.ts beside the module: it constructs a
  VoiceSession with a fake VoiceSessionHost and drives it through the public
  API (handleSetVoiceMode, then turn-detection/STT events), asserting on the
  host seam (sendSpokenInput) and emitted messages. No Session, no private
  field/method access.
- Remove the voice tests from session.test.ts that reached through Session's
  private voiceSession field into VoiceSession internals.

Same three behaviors are covered (streaming final -> agent, finalization
timeout empty path, low-confidence filtering); driving handleSetVoiceMode
additionally exercises the enable path the old tests bypassed.
This commit is contained in:
Mohamed Boudra
2026-06-21 15:14:23 +08:00
committed by GitHub
parent 9c86a410ea
commit 617cf8a7bf
4 changed files with 1561 additions and 1454 deletions

View File

@@ -1,5 +1,4 @@
import { execSync } from "child_process";
import { EventEmitter } from "events";
import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "fs";
import { homedir, tmpdir } from "os";
import { join, resolve as resolvePath } from "path";
@@ -17,16 +16,6 @@ import { StructuredAgentFallbackError } from "./agent/agent-response-loop.js";
import type { StoredAgentRecord } from "./agent/agent-storage.js";
import type { ProviderSnapshotManager } from "./agent/provider-snapshot-manager.js";
import type { SessionOptions } from "./session.js";
import type {
SpeechToTextProvider,
StreamingTranscriptionCommittedEvent,
StreamingTranscriptionEvent,
StreamingTranscriptionSession,
} from "./speech/speech-provider.js";
import type {
TurnDetectionProvider,
TurnDetectionSession,
} from "./speech/turn-detection-provider.js";
import {
asSessionInternals as asSessionInternalsHelper,
asAgentManager,
@@ -50,8 +39,6 @@ import type {
} from "../services/github-service.js";
interface SessionHandlerInternals {
startVoiceTurnController(): Promise<void>;
stopVoiceTurnController(): Promise<void>;
handleSendAgentMessage(
agentId: string,
text: string,
@@ -82,9 +69,6 @@ interface SessionHandlerInternals {
handleStashPopRequest(params: unknown): Promise<unknown>;
createPaseoWorktree(params: unknown): Promise<unknown>;
handleStartWorkspaceScriptRequest(params: unknown): Promise<unknown>;
sttManager: {
transcribe(audio: Buffer, format: string): Promise<unknown>;
};
}
function asSessionInternals(session: Session): SessionHandlerInternals {
@@ -313,198 +297,6 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session {
});
}
class FakeVoiceTurnDetectionSession extends EventEmitter implements TurnDetectionSession {
public readonly requiredSampleRate = 16000;
async connect(): Promise<void> {}
appendPcm16(_chunk: Buffer): void {}
flush(): void {}
reset(): void {}
close(): void {}
}
class FakeVoiceSttSession extends EventEmitter implements StreamingTranscriptionSession {
public readonly requiredSampleRate = 16000;
public commitCount = 0;
async connect(): Promise<void> {}
appendPcm16(_pcm16le: Buffer): void {}
commit(): void {
this.commitCount += 1;
}
clear(): void {}
close(): void {}
emitCommitted(event: StreamingTranscriptionCommittedEvent): void {
this.emit("committed", event);
}
emitTranscript(event: StreamingTranscriptionEvent): void {
this.emit("transcript", event);
}
}
function createVoiceSessionHarness() {
const messages: unknown[] = [];
const detector = new FakeVoiceTurnDetectionSession();
const sttSession = new FakeVoiceSttSession();
const sttProvider: SpeechToTextProvider = {
id: "local",
createSession: vi.fn(() => sttSession),
};
const turnDetection: TurnDetectionProvider = {
id: "local",
createSession: vi.fn(() => detector),
};
const session = createSessionForTest({
messages,
stt: sttProvider,
voice: { turnDetection },
});
Object.assign(session, {
isVoiceMode: true,
voiceModeAgentId: "11111111-1111-4111-8111-111111111111",
});
const internals = asSessionInternals(session);
const sendAgentMessage = vi
.spyOn(internals, "handleSendAgentMessage")
.mockResolvedValue({ ok: true });
const transcribe = vi.spyOn(asSessionInternals(session).sttManager, "transcribe");
return {
session,
internals,
messages,
detector,
sttSession,
sendAgentMessage,
transcribe,
};
}
async function settleVoiceSession(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
}
describe("session voice mode streaming transcription", () => {
test("submits the streaming final transcript to the agent without batch transcribe", async () => {
const harness = createVoiceSessionHarness();
await harness.internals.startVoiceTurnController();
harness.detector.emit("speech_started");
await settleVoiceSession();
harness.detector.emit("speech_stopped");
await settleVoiceSession();
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
harness.sttSession.emitTranscript({
segmentId: "segment-1",
transcript: "ship the streaming final",
isFinal: true,
language: "en",
avgLogprob: -0.1,
isLowConfidence: false,
});
await settleVoiceSession();
expect(harness.sttSession.commitCount).toBe(1);
expect(harness.transcribe).not.toHaveBeenCalled();
expect(harness.sendAgentMessage).toHaveBeenCalledWith(
"11111111-1111-4111-8111-111111111111",
"ship the streaming final",
undefined,
undefined,
undefined,
undefined,
{ spokenInput: true },
);
expect(harness.messages).toContainEqual(
expect.objectContaining({
type: "transcription_result",
payload: expect.objectContaining({
text: "ship the streaming final",
language: "en",
avgLogprob: -0.1,
}),
}),
);
await harness.internals.stopVoiceTurnController();
});
test("uses the finalization timeout empty transcript path without agent submission", async () => {
vi.useFakeTimers();
try {
const harness = createVoiceSessionHarness();
await harness.internals.startVoiceTurnController();
harness.detector.emit("speech_started");
await settleVoiceSession();
harness.detector.emit("speech_stopped");
await settleVoiceSession();
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
await vi.advanceTimersByTimeAsync(10_000);
await settleVoiceSession();
expect(harness.transcribe).not.toHaveBeenCalled();
expect(harness.sendAgentMessage).not.toHaveBeenCalled();
expect(harness.messages).toContainEqual(
expect.objectContaining({
type: "transcription_result",
payload: expect.objectContaining({
text: "",
}),
}),
);
await harness.internals.stopVoiceTurnController();
} finally {
vi.useRealTimers();
}
});
test("filters low-confidence streaming finals without agent submission", async () => {
const harness = createVoiceSessionHarness();
await harness.internals.startVoiceTurnController();
harness.detector.emit("speech_started");
await settleVoiceSession();
harness.detector.emit("speech_stopped");
await settleVoiceSession();
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
harness.sttSession.emitTranscript({
segmentId: "segment-1",
transcript: "background noise",
isFinal: true,
avgLogprob: -2.5,
isLowConfidence: true,
});
await settleVoiceSession();
expect(harness.transcribe).not.toHaveBeenCalled();
expect(harness.sendAgentMessage).not.toHaveBeenCalled();
expect(harness.messages).toContainEqual(
expect.objectContaining({
type: "transcription_result",
payload: expect.objectContaining({
text: "",
avgLogprob: -2.5,
isLowConfidence: true,
}),
}),
);
await harness.internals.stopVoiceTurnController();
});
});
describe("file explorer binary responses", () => {
const tempDirs: string[] = [];

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,211 @@
import { EventEmitter } from "node:events";
import pino from "pino";
import { describe, expect, test, vi } from "vitest";
import { VoiceSession, type VoiceSessionHost } from "./voice-session.js";
import type { ManagedAgent } from "../agent/agent-manager.js";
import type { SessionOutboundMessage } from "../messages.js";
import type {
SpeechToTextProvider,
StreamingTranscriptionCommittedEvent,
StreamingTranscriptionEvent,
StreamingTranscriptionSession,
} from "../speech/speech-provider.js";
import type {
TurnDetectionProvider,
TurnDetectionSession,
} from "../speech/turn-detection-provider.js";
const VOICE_AGENT_ID = "11111111-1111-4111-8111-111111111111";
class FakeVoiceTurnDetectionSession extends EventEmitter implements TurnDetectionSession {
public readonly requiredSampleRate = 16000;
async connect(): Promise<void> {}
appendPcm16(_chunk: Buffer): void {}
flush(): void {}
reset(): void {}
close(): void {}
}
class FakeVoiceSttSession extends EventEmitter implements StreamingTranscriptionSession {
public readonly requiredSampleRate = 16000;
public commitCount = 0;
async connect(): Promise<void> {}
appendPcm16(_pcm16le: Buffer): void {}
commit(): void {
this.commitCount += 1;
}
clear(): void {}
close(): void {}
emitCommitted(event: StreamingTranscriptionCommittedEvent): void {
this.emit("committed", event);
}
emitTranscript(event: StreamingTranscriptionEvent): void {
this.emit("transcript", event);
}
}
interface FakeVoiceHost extends VoiceSessionHost {
readonly emitted: SessionOutboundMessage[];
readonly spokenInput: Array<{ agentId: string; text: string }>;
}
function createFakeHost(): FakeVoiceHost {
const emitted: SessionOutboundMessage[] = [];
const spokenInput: Array<{ agentId: string; text: string }> = [];
return {
emitted,
spokenInput,
emit: (msg) => {
emitted.push(msg);
},
loadAgent: async (agentId) =>
({ id: agentId, config: { systemPrompt: undefined } }) as unknown as ManagedAgent,
reloadAgentSession: async (agentId) => ({ id: agentId }) as unknown as ManagedAgent,
sendSpokenInput: async (agentId, text) => {
spokenInput.push({ agentId, text });
},
interruptAgentIfRunning: async () => {},
hasActiveAgentRun: () => false,
};
}
function createVoiceSession() {
const detector = new FakeVoiceTurnDetectionSession();
const sttSession = new FakeVoiceSttSession();
const stt: SpeechToTextProvider = {
id: "local",
createSession: vi.fn(() => sttSession),
};
const turnDetection: TurnDetectionProvider = {
id: "local",
createSession: vi.fn(() => detector),
};
const host = createFakeHost();
const voiceSession = new VoiceSession({
host,
logger: pino({ level: "silent" }),
sessionId: "voice-session-test",
sttLanguage: "en",
tts: null,
stt,
voice: { turnDetection },
});
return { voiceSession, detector, sttSession, host };
}
async function settle(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
}
describe("VoiceSession streaming transcription", () => {
test("delivers the streaming final transcript to the agent exactly once", async () => {
const { voiceSession, detector, sttSession, host } = createVoiceSession();
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
detector.emit("speech_started");
await settle();
detector.emit("speech_stopped");
await settle();
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
sttSession.emitTranscript({
segmentId: "segment-1",
transcript: "ship the streaming final",
isFinal: true,
language: "en",
avgLogprob: -0.1,
isLowConfidence: false,
});
await settle();
expect(sttSession.commitCount).toBe(1);
expect(host.spokenInput).toEqual([
{ agentId: VOICE_AGENT_ID, text: "ship the streaming final" },
]);
expect(host.emitted).toContainEqual(
expect.objectContaining({
type: "transcription_result",
payload: expect.objectContaining({
text: "ship the streaming final",
language: "en",
avgLogprob: -0.1,
}),
}),
);
await voiceSession.cleanup();
});
test("emits an empty transcript on finalization timeout without submitting to the agent", async () => {
vi.useFakeTimers();
try {
const { voiceSession, detector, sttSession, host } = createVoiceSession();
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
detector.emit("speech_started");
await settle();
detector.emit("speech_stopped");
await settle();
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
await vi.advanceTimersByTimeAsync(10_000);
await settle();
expect(host.spokenInput).toEqual([]);
expect(host.emitted).toContainEqual(
expect.objectContaining({
type: "transcription_result",
payload: expect.objectContaining({ text: "" }),
}),
);
await voiceSession.cleanup();
} finally {
vi.useRealTimers();
}
});
test("filters a low-confidence streaming final without submitting to the agent", async () => {
const { voiceSession, detector, sttSession, host } = createVoiceSession();
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
detector.emit("speech_started");
await settle();
detector.emit("speech_stopped");
await settle();
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
sttSession.emitTranscript({
segmentId: "segment-1",
transcript: "background noise",
isFinal: true,
avgLogprob: -2.5,
isLowConfidence: true,
});
await settle();
expect(host.spokenInput).toEqual([]);
expect(host.emitted).toContainEqual(
expect.objectContaining({
type: "transcription_result",
payload: expect.objectContaining({
text: "",
avgLogprob: -2.5,
isLowConfidence: true,
}),
}),
);
await voiceSession.cleanup();
});
});

File diff suppressed because it is too large Load Diff