mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
refactor(server): extract voice mode subsystem into VoiceSession (#1640)
* refactor(server): extract voice mode subsystem into VoiceSession session.ts was a 10.5k-line god object and the most-churned file in the repo. ~1,100 of those lines were an entire voice/audio subsystem — the STT/TTS/dictation managers, the barge-in audio-buffering state machine, voice-turn orchestration, and the MCP voice bridge — interleaved field by field and method by method with workspace/git/agent/provider concerns. Move the whole subsystem into a new VoiceSession deep module. Session now holds one `voiceSession` field, constructs it once, and delegates the voice/dictation/abort message types, the permission auto-allow gate, and cleanup to it. VoiceSession owns all 18 voice fields and ~25 methods and reaches back into the agent run only through a narrow VoiceSessionHost seam (emit, loadAgent, reloadAgentSession, sendSpokenInput, interruptAgentIfRunning, hasActiveAgentRun). Behavior is preserved (verbatim method moves); deleting voice-session.ts now removes the feature mechanically. The voice unit tests drive the VoiceSession boundary instead of reaching into Session internals, so future voice tests can construct a VoiceSession with a fake host. session.ts: 10,468 -> 9,272 lines. * refactor(server): move VoiceSession into voice/ and test it at the boundary Address Greptile review on #1640: - Move voice-session.ts into the existing voice/ subdirectory, alongside voice-turn-controller.ts, instead of adding another peer to the 30+ file server/ directory. The directory now carries the domain. - Add voice/voice-session.test.ts beside the module: it constructs a VoiceSession with a fake VoiceSessionHost and drives it through the public API (handleSetVoiceMode, then turn-detection/STT events), asserting on the host seam (sendSpokenInput) and emitted messages. No Session, no private field/method access. - Remove the voice tests from session.test.ts that reached through Session's private voiceSession field into VoiceSession internals. Same three behaviors are covered (streaming final -> agent, finalization timeout empty path, low-confidence filtering); driving handleSetVoiceMode additionally exercises the enable path the old tests bypassed.
This commit is contained in:
@@ -1,5 +1,4 @@
|
||||
import { execSync } from "child_process";
|
||||
import { EventEmitter } from "events";
|
||||
import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "fs";
|
||||
import { homedir, tmpdir } from "os";
|
||||
import { join, resolve as resolvePath } from "path";
|
||||
@@ -17,16 +16,6 @@ import { StructuredAgentFallbackError } from "./agent/agent-response-loop.js";
|
||||
import type { StoredAgentRecord } from "./agent/agent-storage.js";
|
||||
import type { ProviderSnapshotManager } from "./agent/provider-snapshot-manager.js";
|
||||
import type { SessionOptions } from "./session.js";
|
||||
import type {
|
||||
SpeechToTextProvider,
|
||||
StreamingTranscriptionCommittedEvent,
|
||||
StreamingTranscriptionEvent,
|
||||
StreamingTranscriptionSession,
|
||||
} from "./speech/speech-provider.js";
|
||||
import type {
|
||||
TurnDetectionProvider,
|
||||
TurnDetectionSession,
|
||||
} from "./speech/turn-detection-provider.js";
|
||||
import {
|
||||
asSessionInternals as asSessionInternalsHelper,
|
||||
asAgentManager,
|
||||
@@ -50,8 +39,6 @@ import type {
|
||||
} from "../services/github-service.js";
|
||||
|
||||
interface SessionHandlerInternals {
|
||||
startVoiceTurnController(): Promise<void>;
|
||||
stopVoiceTurnController(): Promise<void>;
|
||||
handleSendAgentMessage(
|
||||
agentId: string,
|
||||
text: string,
|
||||
@@ -82,9 +69,6 @@ interface SessionHandlerInternals {
|
||||
handleStashPopRequest(params: unknown): Promise<unknown>;
|
||||
createPaseoWorktree(params: unknown): Promise<unknown>;
|
||||
handleStartWorkspaceScriptRequest(params: unknown): Promise<unknown>;
|
||||
sttManager: {
|
||||
transcribe(audio: Buffer, format: string): Promise<unknown>;
|
||||
};
|
||||
}
|
||||
|
||||
function asSessionInternals(session: Session): SessionHandlerInternals {
|
||||
@@ -313,198 +297,6 @@ function createSessionForTest(options: SessionForTestOptions = {}): Session {
|
||||
});
|
||||
}
|
||||
|
||||
class FakeVoiceTurnDetectionSession extends EventEmitter implements TurnDetectionSession {
|
||||
public readonly requiredSampleRate = 16000;
|
||||
|
||||
async connect(): Promise<void> {}
|
||||
|
||||
appendPcm16(_chunk: Buffer): void {}
|
||||
|
||||
flush(): void {}
|
||||
reset(): void {}
|
||||
close(): void {}
|
||||
}
|
||||
|
||||
class FakeVoiceSttSession extends EventEmitter implements StreamingTranscriptionSession {
|
||||
public readonly requiredSampleRate = 16000;
|
||||
public commitCount = 0;
|
||||
|
||||
async connect(): Promise<void> {}
|
||||
|
||||
appendPcm16(_pcm16le: Buffer): void {}
|
||||
|
||||
commit(): void {
|
||||
this.commitCount += 1;
|
||||
}
|
||||
|
||||
clear(): void {}
|
||||
close(): void {}
|
||||
|
||||
emitCommitted(event: StreamingTranscriptionCommittedEvent): void {
|
||||
this.emit("committed", event);
|
||||
}
|
||||
|
||||
emitTranscript(event: StreamingTranscriptionEvent): void {
|
||||
this.emit("transcript", event);
|
||||
}
|
||||
}
|
||||
|
||||
function createVoiceSessionHarness() {
|
||||
const messages: unknown[] = [];
|
||||
const detector = new FakeVoiceTurnDetectionSession();
|
||||
const sttSession = new FakeVoiceSttSession();
|
||||
const sttProvider: SpeechToTextProvider = {
|
||||
id: "local",
|
||||
createSession: vi.fn(() => sttSession),
|
||||
};
|
||||
const turnDetection: TurnDetectionProvider = {
|
||||
id: "local",
|
||||
createSession: vi.fn(() => detector),
|
||||
};
|
||||
const session = createSessionForTest({
|
||||
messages,
|
||||
stt: sttProvider,
|
||||
voice: { turnDetection },
|
||||
});
|
||||
Object.assign(session, {
|
||||
isVoiceMode: true,
|
||||
voiceModeAgentId: "11111111-1111-4111-8111-111111111111",
|
||||
});
|
||||
const internals = asSessionInternals(session);
|
||||
const sendAgentMessage = vi
|
||||
.spyOn(internals, "handleSendAgentMessage")
|
||||
.mockResolvedValue({ ok: true });
|
||||
const transcribe = vi.spyOn(asSessionInternals(session).sttManager, "transcribe");
|
||||
|
||||
return {
|
||||
session,
|
||||
internals,
|
||||
messages,
|
||||
detector,
|
||||
sttSession,
|
||||
sendAgentMessage,
|
||||
transcribe,
|
||||
};
|
||||
}
|
||||
|
||||
async function settleVoiceSession(): Promise<void> {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
}
|
||||
|
||||
describe("session voice mode streaming transcription", () => {
|
||||
test("submits the streaming final transcript to the agent without batch transcribe", async () => {
|
||||
const harness = createVoiceSessionHarness();
|
||||
|
||||
await harness.internals.startVoiceTurnController();
|
||||
harness.detector.emit("speech_started");
|
||||
await settleVoiceSession();
|
||||
harness.detector.emit("speech_stopped");
|
||||
await settleVoiceSession();
|
||||
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||
harness.sttSession.emitTranscript({
|
||||
segmentId: "segment-1",
|
||||
transcript: "ship the streaming final",
|
||||
isFinal: true,
|
||||
language: "en",
|
||||
avgLogprob: -0.1,
|
||||
isLowConfidence: false,
|
||||
});
|
||||
await settleVoiceSession();
|
||||
|
||||
expect(harness.sttSession.commitCount).toBe(1);
|
||||
expect(harness.transcribe).not.toHaveBeenCalled();
|
||||
expect(harness.sendAgentMessage).toHaveBeenCalledWith(
|
||||
"11111111-1111-4111-8111-111111111111",
|
||||
"ship the streaming final",
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
{ spokenInput: true },
|
||||
);
|
||||
expect(harness.messages).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "transcription_result",
|
||||
payload: expect.objectContaining({
|
||||
text: "ship the streaming final",
|
||||
language: "en",
|
||||
avgLogprob: -0.1,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await harness.internals.stopVoiceTurnController();
|
||||
});
|
||||
|
||||
test("uses the finalization timeout empty transcript path without agent submission", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const harness = createVoiceSessionHarness();
|
||||
|
||||
await harness.internals.startVoiceTurnController();
|
||||
harness.detector.emit("speech_started");
|
||||
await settleVoiceSession();
|
||||
harness.detector.emit("speech_stopped");
|
||||
await settleVoiceSession();
|
||||
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
await settleVoiceSession();
|
||||
|
||||
expect(harness.transcribe).not.toHaveBeenCalled();
|
||||
expect(harness.sendAgentMessage).not.toHaveBeenCalled();
|
||||
expect(harness.messages).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "transcription_result",
|
||||
payload: expect.objectContaining({
|
||||
text: "",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await harness.internals.stopVoiceTurnController();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("filters low-confidence streaming finals without agent submission", async () => {
|
||||
const harness = createVoiceSessionHarness();
|
||||
|
||||
await harness.internals.startVoiceTurnController();
|
||||
harness.detector.emit("speech_started");
|
||||
await settleVoiceSession();
|
||||
harness.detector.emit("speech_stopped");
|
||||
await settleVoiceSession();
|
||||
harness.sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||
harness.sttSession.emitTranscript({
|
||||
segmentId: "segment-1",
|
||||
transcript: "background noise",
|
||||
isFinal: true,
|
||||
avgLogprob: -2.5,
|
||||
isLowConfidence: true,
|
||||
});
|
||||
await settleVoiceSession();
|
||||
|
||||
expect(harness.transcribe).not.toHaveBeenCalled();
|
||||
expect(harness.sendAgentMessage).not.toHaveBeenCalled();
|
||||
expect(harness.messages).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "transcription_result",
|
||||
payload: expect.objectContaining({
|
||||
text: "",
|
||||
avgLogprob: -2.5,
|
||||
isLowConfidence: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await harness.internals.stopVoiceTurnController();
|
||||
});
|
||||
});
|
||||
|
||||
describe("file explorer binary responses", () => {
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
211
packages/server/src/server/voice/voice-session.test.ts
Normal file
211
packages/server/src/server/voice/voice-session.test.ts
Normal file
@@ -0,0 +1,211 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import pino from "pino";
|
||||
import { describe, expect, test, vi } from "vitest";
|
||||
|
||||
import { VoiceSession, type VoiceSessionHost } from "./voice-session.js";
|
||||
import type { ManagedAgent } from "../agent/agent-manager.js";
|
||||
import type { SessionOutboundMessage } from "../messages.js";
|
||||
import type {
|
||||
SpeechToTextProvider,
|
||||
StreamingTranscriptionCommittedEvent,
|
||||
StreamingTranscriptionEvent,
|
||||
StreamingTranscriptionSession,
|
||||
} from "../speech/speech-provider.js";
|
||||
import type {
|
||||
TurnDetectionProvider,
|
||||
TurnDetectionSession,
|
||||
} from "../speech/turn-detection-provider.js";
|
||||
|
||||
const VOICE_AGENT_ID = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
class FakeVoiceTurnDetectionSession extends EventEmitter implements TurnDetectionSession {
|
||||
public readonly requiredSampleRate = 16000;
|
||||
|
||||
async connect(): Promise<void> {}
|
||||
|
||||
appendPcm16(_chunk: Buffer): void {}
|
||||
|
||||
flush(): void {}
|
||||
reset(): void {}
|
||||
close(): void {}
|
||||
}
|
||||
|
||||
class FakeVoiceSttSession extends EventEmitter implements StreamingTranscriptionSession {
|
||||
public readonly requiredSampleRate = 16000;
|
||||
public commitCount = 0;
|
||||
|
||||
async connect(): Promise<void> {}
|
||||
|
||||
appendPcm16(_pcm16le: Buffer): void {}
|
||||
|
||||
commit(): void {
|
||||
this.commitCount += 1;
|
||||
}
|
||||
|
||||
clear(): void {}
|
||||
close(): void {}
|
||||
|
||||
emitCommitted(event: StreamingTranscriptionCommittedEvent): void {
|
||||
this.emit("committed", event);
|
||||
}
|
||||
|
||||
emitTranscript(event: StreamingTranscriptionEvent): void {
|
||||
this.emit("transcript", event);
|
||||
}
|
||||
}
|
||||
|
||||
interface FakeVoiceHost extends VoiceSessionHost {
|
||||
readonly emitted: SessionOutboundMessage[];
|
||||
readonly spokenInput: Array<{ agentId: string; text: string }>;
|
||||
}
|
||||
|
||||
function createFakeHost(): FakeVoiceHost {
|
||||
const emitted: SessionOutboundMessage[] = [];
|
||||
const spokenInput: Array<{ agentId: string; text: string }> = [];
|
||||
return {
|
||||
emitted,
|
||||
spokenInput,
|
||||
emit: (msg) => {
|
||||
emitted.push(msg);
|
||||
},
|
||||
loadAgent: async (agentId) =>
|
||||
({ id: agentId, config: { systemPrompt: undefined } }) as unknown as ManagedAgent,
|
||||
reloadAgentSession: async (agentId) => ({ id: agentId }) as unknown as ManagedAgent,
|
||||
sendSpokenInput: async (agentId, text) => {
|
||||
spokenInput.push({ agentId, text });
|
||||
},
|
||||
interruptAgentIfRunning: async () => {},
|
||||
hasActiveAgentRun: () => false,
|
||||
};
|
||||
}
|
||||
|
||||
function createVoiceSession() {
|
||||
const detector = new FakeVoiceTurnDetectionSession();
|
||||
const sttSession = new FakeVoiceSttSession();
|
||||
const stt: SpeechToTextProvider = {
|
||||
id: "local",
|
||||
createSession: vi.fn(() => sttSession),
|
||||
};
|
||||
const turnDetection: TurnDetectionProvider = {
|
||||
id: "local",
|
||||
createSession: vi.fn(() => detector),
|
||||
};
|
||||
const host = createFakeHost();
|
||||
const voiceSession = new VoiceSession({
|
||||
host,
|
||||
logger: pino({ level: "silent" }),
|
||||
sessionId: "voice-session-test",
|
||||
sttLanguage: "en",
|
||||
tts: null,
|
||||
stt,
|
||||
voice: { turnDetection },
|
||||
});
|
||||
return { voiceSession, detector, sttSession, host };
|
||||
}
|
||||
|
||||
async function settle(): Promise<void> {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
}
|
||||
|
||||
describe("VoiceSession streaming transcription", () => {
|
||||
test("delivers the streaming final transcript to the agent exactly once", async () => {
|
||||
const { voiceSession, detector, sttSession, host } = createVoiceSession();
|
||||
|
||||
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
|
||||
detector.emit("speech_started");
|
||||
await settle();
|
||||
detector.emit("speech_stopped");
|
||||
await settle();
|
||||
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||
sttSession.emitTranscript({
|
||||
segmentId: "segment-1",
|
||||
transcript: "ship the streaming final",
|
||||
isFinal: true,
|
||||
language: "en",
|
||||
avgLogprob: -0.1,
|
||||
isLowConfidence: false,
|
||||
});
|
||||
await settle();
|
||||
|
||||
expect(sttSession.commitCount).toBe(1);
|
||||
expect(host.spokenInput).toEqual([
|
||||
{ agentId: VOICE_AGENT_ID, text: "ship the streaming final" },
|
||||
]);
|
||||
expect(host.emitted).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "transcription_result",
|
||||
payload: expect.objectContaining({
|
||||
text: "ship the streaming final",
|
||||
language: "en",
|
||||
avgLogprob: -0.1,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await voiceSession.cleanup();
|
||||
});
|
||||
|
||||
test("emits an empty transcript on finalization timeout without submitting to the agent", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const { voiceSession, detector, sttSession, host } = createVoiceSession();
|
||||
|
||||
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
|
||||
detector.emit("speech_started");
|
||||
await settle();
|
||||
detector.emit("speech_stopped");
|
||||
await settle();
|
||||
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
await settle();
|
||||
|
||||
expect(host.spokenInput).toEqual([]);
|
||||
expect(host.emitted).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "transcription_result",
|
||||
payload: expect.objectContaining({ text: "" }),
|
||||
}),
|
||||
);
|
||||
|
||||
await voiceSession.cleanup();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("filters a low-confidence streaming final without submitting to the agent", async () => {
|
||||
const { voiceSession, detector, sttSession, host } = createVoiceSession();
|
||||
|
||||
await voiceSession.handleSetVoiceMode(true, VOICE_AGENT_ID);
|
||||
detector.emit("speech_started");
|
||||
await settle();
|
||||
detector.emit("speech_stopped");
|
||||
await settle();
|
||||
sttSession.emitCommitted({ segmentId: "segment-1", previousSegmentId: null });
|
||||
sttSession.emitTranscript({
|
||||
segmentId: "segment-1",
|
||||
transcript: "background noise",
|
||||
isFinal: true,
|
||||
avgLogprob: -2.5,
|
||||
isLowConfidence: true,
|
||||
});
|
||||
await settle();
|
||||
|
||||
expect(host.spokenInput).toEqual([]);
|
||||
expect(host.emitted).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "transcription_result",
|
||||
payload: expect.objectContaining({
|
||||
text: "",
|
||||
avgLogprob: -2.5,
|
||||
isLowConfidence: true,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await voiceSession.cleanup();
|
||||
});
|
||||
});
|
||||
1301
packages/server/src/server/voice/voice-session.ts
Normal file
1301
packages/server/src/server/voice/voice-session.ts
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user