diff --git a/packages/server/src/client/daemon-client.ts b/packages/server/src/client/daemon-client.ts index f77b79aad..43387c76f 100644 --- a/packages/server/src/client/daemon-client.ts +++ b/packages/server/src/client/daemon-client.ts @@ -759,10 +759,6 @@ export class DaemonClient { } } - sendUserMessage(text: string): void { - this.sendSessionMessage({ type: "user_text", text }); - } - clearAgentAttention(agentId: string | string[]): void { this.sendSessionMessage({ type: "clear_agent_attention", agentId }); } diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 3eaea436a..7cd6a0ef0 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -586,8 +586,8 @@ describe("daemon client E2E", () => { 120000 ); - test( - "does not process non-voice turns through the voice agent path", + speechTest( + "does not process non-voice audio through the voice agent path", async () => { await ctx.client.setVoiceMode(false); @@ -624,7 +624,16 @@ describe("daemon client E2E", () => { }; }); - await ctx.client.sendUserMessage("Say 'hello' and nothing else"); + const fixturePath = path.resolve( + process.cwd(), + "..", + "app", + "e2e", + "fixtures", + "recording.wav" + ); + const wav = await import("node:fs/promises").then((fs) => fs.readFile(fixturePath)); + await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true); await transcriptSeen; await new Promise((resolve) => setTimeout(resolve, 1500)); diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 75b689210..2465a5807 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -297,7 +297,7 @@ function toAgentPersistenceHandle( } /** - * Session represents a single client conversation session. + * Session represents a single connected client session. * It owns all state management, orchestration logic, and message processing. * Session has no knowledge of WebSockets - it only emits and receives messages. */ @@ -835,10 +835,6 @@ export class Session { public async handleMessage(msg: SessionInboundMessage): Promise { try { switch (msg.type) { - case "user_text": - await this.handleUserText(msg.text); - break; - case "voice_audio_chunk": await this.handleAudioChunk(msg); break; @@ -4150,36 +4146,6 @@ export class Session { return timeline.length; } - /** - * Handle text message from user - */ - private async handleUserText(text: string): Promise { - // Abort any in-progress stream immediately - this.createAbortController(); - - // Wait for aborted stream to finish cleanup (save partial response) - if (this.currentStreamPromise) { - this.sessionLogger.debug("Waiting for aborted stream to finish cleanup"); - await this.currentStreamPromise; - this.sessionLogger.debug("Aborted stream finished cleanup"); - } - - // Emit user message activity log - this.emit({ - type: "activity_log", - payload: { - id: uuidv4(), - timestamp: new Date(), - type: "transcript", - content: text, - }, - }); - - // Process through LLM (voice path is agent-backed only) - this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, text); - await this.currentStreamPromise; - } - /** * Handle audio chunk for buffering and transcription */ @@ -4445,7 +4411,7 @@ export class Session { // Set phase to LLM and process (TTS enabled in voice mode for voice agents) this.clearSpeechInProgress("transcription complete"); this.setPhase("llm"); - this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, result.text); + this.currentStreamPromise = this.processVoiceTurn(result.text); await this.currentStreamPromise; this.setPhase("idle"); } catch (error: any) { @@ -4655,10 +4621,10 @@ export class Session { /** * Process user message through LLM with streaming and tool execution */ - private async processVoiceTurn(enableTTS: boolean, latestUserText?: string): Promise { + private async processVoiceTurn(latestUserText?: string): Promise { try { - if (!enableTTS) { - this.sessionLogger.warn("Ignoring non-voice processVoiceTurn call; voice is agent-only"); + if (!this.isVoiceMode) { + this.sessionLogger.warn("Ignoring processVoiceTurn call while voice mode is disabled"); return; } const normalized = (latestUserText ?? "").trim(); diff --git a/packages/server/src/server/voice-local-agent.e2e.test.ts b/packages/server/src/server/voice-local-agent.e2e.test.ts index 95e48ecbd..59e23ee8e 100644 --- a/packages/server/src/server/voice-local-agent.e2e.test.ts +++ b/packages/server/src/server/voice-local-agent.e2e.test.ts @@ -1,5 +1,7 @@ import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { randomUUID } from "node:crypto"; +import path from "node:path"; +import { readFile } from "node:fs/promises"; import { createDaemonTestContext, type DaemonTestContext } from "./test-utils/index.js"; @@ -104,9 +106,16 @@ function waitForSignal( }; }); - ctx.client.sendUserMessage( - "Use the speak tool and say exactly: local voice agent path is working." + const fixturePath = path.resolve( + process.cwd(), + "..", + "app", + "e2e", + "fixtures", + "recording.wav" ); + const wav = await readFile(fixturePath); + await ctx.client.sendVoiceAudioChunk(wav.toString("base64"), "audio/wav", true); const [{ chunkId }, assistantText] = await Promise.all([ audioPromise, @@ -114,7 +123,7 @@ function waitForSignal( ]); expect(chunkId.length).toBeGreaterThan(0); - expect(assistantText.toLowerCase()).toContain("local voice agent path is working"); + expect(assistantText.trim().length).toBeGreaterThan(0); const agents = await ctx.client.fetchAgents(); expect( diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index ac39e9228..3b8f88252 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -294,11 +294,6 @@ export type AgentStreamEventPayload = z.infer< // Session Inbound Messages (Session receives these) // ============================================================================ -export const UserTextMessageSchema = z.object({ - type: z.literal("user_text"), - text: z.string(), -}); - export const VoiceAudioChunkMessageSchema = z.object({ type: z.literal("voice_audio_chunk"), audio: z.string(), // base64 encoded @@ -859,7 +854,6 @@ export const KillTerminalRequestSchema = z.object({ }); export const SessionInboundMessageSchema = z.discriminatedUnion("type", [ - UserTextMessageSchema, VoiceAudioChunkMessageSchema, AbortRequestMessageSchema, AudioPlayedMessageSchema, @@ -1713,7 +1707,6 @@ export type InitializeAgentResponseMessage = z.infer; // Type exports for inbound message types -export type UserTextMessage = z.infer; export type VoiceAudioChunkMessage = z.infer; export type FetchAgentsRequestMessage = z.infer; export type FetchAgentRequestMessage = z.infer;