mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Refactor voice mode to agent-only flow
This commit is contained in:
@@ -2,11 +2,8 @@ import { createContext, useContext, useState, ReactNode, useCallback, useEffect,
|
||||
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
|
||||
import type { SessionState } from "@/stores/session-store";
|
||||
import { useSessionStore } from "@/stores/session-store";
|
||||
import AsyncStorage from "@react-native-async-storage/async-storage";
|
||||
import { randomUUID } from "expo-crypto";
|
||||
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
|
||||
|
||||
const VOICE_CONVERSATION_ID_STORAGE_KEY = "@paseo:voice-conversation-id";
|
||||
const KEEP_AWAKE_TAG = "paseo:voice";
|
||||
const VOICE_VAD_VOLUME_THRESHOLD = 0.18;
|
||||
const VOICE_VAD_SILENCE_DURATION_MS = 1400;
|
||||
@@ -170,18 +167,9 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
console.log("[Voice] Mode enabled");
|
||||
|
||||
if (session?.client) {
|
||||
let voiceConversationId =
|
||||
(await AsyncStorage.getItem(VOICE_CONVERSATION_ID_STORAGE_KEY)) ?? null;
|
||||
if (!voiceConversationId) {
|
||||
voiceConversationId = randomUUID();
|
||||
await AsyncStorage.setItem(
|
||||
VOICE_CONVERSATION_ID_STORAGE_KEY,
|
||||
voiceConversationId
|
||||
);
|
||||
}
|
||||
await session.client.setVoiceConversation(true, voiceConversationId);
|
||||
await session.client.setVoiceMode(true);
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[Voice] Failed to start:", error);
|
||||
@@ -204,9 +192,9 @@ export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
console.log("[Voice] Mode disabled");
|
||||
|
||||
if (session?.client) {
|
||||
await session.client.setVoiceConversation(false);
|
||||
await session.client.setVoiceMode(false);
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceConversation skipped: daemon unavailable");
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error("[Voice] Failed to stop:", error);
|
||||
|
||||
@@ -12,9 +12,7 @@ import type {
|
||||
AgentStreamEventPayload,
|
||||
AgentSnapshotPayload,
|
||||
AgentPermissionResolvedMessage,
|
||||
VoiceConversationLoadedMessage,
|
||||
CreateAgentRequestMessage,
|
||||
DeleteVoiceConversationResponseMessage,
|
||||
FileDownloadTokenResponse,
|
||||
FileExplorerResponse,
|
||||
GitDiffResponse,
|
||||
@@ -34,7 +32,6 @@ import type {
|
||||
ProjectIconResponse,
|
||||
ListCommandsResponse,
|
||||
ExecuteCommandResponse,
|
||||
ListVoiceConversationsResponseMessage,
|
||||
ListProviderModelsResponseMessage,
|
||||
SpeechModelsListResponse,
|
||||
SpeechModelsDownloadResponse,
|
||||
@@ -184,9 +181,6 @@ export type CreateAgentRequestOptions = {
|
||||
labels?: Record<string, string>;
|
||||
} & AgentConfigOverrides;
|
||||
|
||||
type VoiceConversationLoadedPayload = VoiceConversationLoadedMessage["payload"];
|
||||
type ListVoiceConversationsPayload = ListVoiceConversationsResponseMessage["payload"];
|
||||
type DeleteVoiceConversationPayload = DeleteVoiceConversationResponseMessage["payload"];
|
||||
type GitDiffPayload = GitDiffResponse["payload"];
|
||||
type HighlightedDiffPayload = HighlightedDiffResponse["payload"];
|
||||
type CheckoutStatusPayload = CheckoutStatusResponse["payload"];
|
||||
@@ -926,87 +920,6 @@ export class DaemonClient {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Voice Conversation RPC
|
||||
// ============================================================================
|
||||
|
||||
async loadVoiceConversation(
|
||||
voiceConversationId: string,
|
||||
requestId?: string
|
||||
): Promise<VoiceConversationLoadedPayload> {
|
||||
const resolvedRequestId = this.createRequestId(requestId);
|
||||
const message = SessionInboundMessageSchema.parse({
|
||||
type: "load_voice_conversation_request",
|
||||
voiceConversationId,
|
||||
requestId: resolvedRequestId,
|
||||
});
|
||||
return this.sendRequest({
|
||||
requestId: resolvedRequestId,
|
||||
message,
|
||||
timeout: 10000,
|
||||
options: { skipQueue: true },
|
||||
select: (msg) => {
|
||||
if (msg.type !== "voice_conversation_loaded") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.requestId !== resolvedRequestId) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async listVoiceConversations(requestId?: string): Promise<ListVoiceConversationsPayload> {
|
||||
const resolvedRequestId = this.createRequestId(requestId);
|
||||
const message = SessionInboundMessageSchema.parse({
|
||||
type: "list_voice_conversations_request",
|
||||
requestId: resolvedRequestId,
|
||||
});
|
||||
return this.sendRequest({
|
||||
requestId: resolvedRequestId,
|
||||
message,
|
||||
timeout: 10000,
|
||||
options: { skipQueue: true },
|
||||
select: (msg) => {
|
||||
if (msg.type !== "list_voice_conversations_response") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.requestId !== resolvedRequestId) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async deleteVoiceConversation(
|
||||
voiceConversationId: string,
|
||||
requestId?: string
|
||||
): Promise<DeleteVoiceConversationPayload> {
|
||||
const resolvedRequestId = this.createRequestId(requestId);
|
||||
const message = SessionInboundMessageSchema.parse({
|
||||
type: "delete_voice_conversation_request",
|
||||
voiceConversationId,
|
||||
requestId: resolvedRequestId,
|
||||
});
|
||||
return this.sendRequest({
|
||||
requestId: resolvedRequestId,
|
||||
message,
|
||||
timeout: 10000,
|
||||
options: { skipQueue: true },
|
||||
select: (msg) => {
|
||||
if (msg.type !== "delete_voice_conversation_response") {
|
||||
return null;
|
||||
}
|
||||
if (msg.payload.requestId !== resolvedRequestId) {
|
||||
return null;
|
||||
}
|
||||
return msg.payload;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Agent Lifecycle
|
||||
// ============================================================================
|
||||
@@ -1376,8 +1289,8 @@ export class DaemonClient {
|
||||
// Audio / Voice
|
||||
// ============================================================================
|
||||
|
||||
async setVoiceConversation(enabled: boolean, voiceConversationId?: string): Promise<void> {
|
||||
this.sendSessionMessage({ type: "set_voice_conversation", enabled, voiceConversationId });
|
||||
async setVoiceMode(enabled: boolean, voiceAgentId?: string): Promise<void> {
|
||||
this.sendSessionMessage({ type: "set_voice_mode", enabled, voiceAgentId });
|
||||
}
|
||||
|
||||
async sendVoiceAudioChunk(
|
||||
|
||||
@@ -144,22 +144,18 @@ describe("daemon client E2E", () => {
|
||||
test("handles session actions", async () => {
|
||||
expect(ctx.client.isConnected).toBe(true);
|
||||
|
||||
const voiceConversationId = randomUUID();
|
||||
const loadResult = await ctx.client.loadVoiceConversation(voiceConversationId);
|
||||
expect(loadResult.voiceConversationId).toBe(voiceConversationId);
|
||||
expect(typeof loadResult.messageCount).toBe("number");
|
||||
|
||||
const agents = await ctx.client.fetchAgents();
|
||||
expect(Array.isArray(agents)).toBe(true);
|
||||
|
||||
const listResult = await ctx.client.listVoiceConversations();
|
||||
expect(Array.isArray(listResult.conversations)).toBe(true);
|
||||
const voiceAgents = await ctx.client.fetchAgents({
|
||||
filter: { labels: { surface: "voice" } },
|
||||
});
|
||||
expect(Array.isArray(voiceAgents)).toBe(true);
|
||||
|
||||
const missingId = randomUUID();
|
||||
const deleteResult = await ctx.client.deleteVoiceConversation(missingId);
|
||||
expect(deleteResult.voiceConversationId).toBe(missingId);
|
||||
expect(deleteResult.success).toBe(false);
|
||||
expect(deleteResult.error).toBeTruthy();
|
||||
await expect(ctx.client.setVoiceMode(true)).resolves.toBeUndefined();
|
||||
await expect(ctx.client.setVoiceMode(false)).resolves.toBeUndefined();
|
||||
|
||||
await ctx.client.deleteAgent(randomUUID());
|
||||
}, 30000);
|
||||
|
||||
test("emits server_info on websocket connect", async () => {
|
||||
@@ -185,19 +181,23 @@ describe("daemon client E2E", () => {
|
||||
await client.close();
|
||||
}, 15000);
|
||||
|
||||
test("matches request IDs for concurrent session requests", async () => {
|
||||
const firstRequestId = `list-${Date.now()}-a`;
|
||||
const secondRequestId = `list-${Date.now()}-b`;
|
||||
test("handles concurrent filtered agent fetch requests", async () => {
|
||||
const firstRequestId = `fetch-${Date.now()}-a`;
|
||||
const secondRequestId = `fetch-${Date.now()}-b`;
|
||||
|
||||
const [first, second] = await Promise.all([
|
||||
ctx.client.listVoiceConversations(firstRequestId),
|
||||
ctx.client.listVoiceConversations(secondRequestId),
|
||||
ctx.client.fetchAgents({
|
||||
requestId: firstRequestId,
|
||||
filter: { labels: { surface: "voice" } },
|
||||
}),
|
||||
ctx.client.fetchAgents({
|
||||
requestId: secondRequestId,
|
||||
filter: { labels: { surface: "voice" } },
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(Array.isArray(first.conversations)).toBe(true);
|
||||
expect(Array.isArray(second.conversations)).toBe(true);
|
||||
expect(first.requestId).toBe(firstRequestId);
|
||||
expect(second.requestId).toBe(secondRequestId);
|
||||
expect(Array.isArray(first)).toBe(true);
|
||||
expect(Array.isArray(second)).toBe(true);
|
||||
}, 15000);
|
||||
|
||||
test(
|
||||
@@ -380,7 +380,7 @@ describe("daemon client E2E", () => {
|
||||
expect(sawAssistantMessage).toBe(true);
|
||||
expect(sawRawAssistantMessage).toBe(true);
|
||||
|
||||
await ctx.client.setVoiceConversation(false);
|
||||
await ctx.client.setVoiceMode(false);
|
||||
|
||||
await ctx.client.abortRequest();
|
||||
await ctx.client.audioPlayed("audio-1");
|
||||
@@ -594,7 +594,7 @@ describe("daemon client E2E", () => {
|
||||
test(
|
||||
"does not process non-voice turns through the voice agent path",
|
||||
async () => {
|
||||
await ctx.client.setVoiceConversation(false);
|
||||
await ctx.client.setVoiceMode(false);
|
||||
|
||||
let sawTranscriptLog = false;
|
||||
let sawAssistantChunk = false;
|
||||
@@ -643,7 +643,7 @@ describe("daemon client E2E", () => {
|
||||
speechTest(
|
||||
"voice mode buffers audio until isLast and emits transcription_result",
|
||||
async () => {
|
||||
await ctx.client.setVoiceConversation(true, randomUUID());
|
||||
await ctx.client.setVoiceMode(true, randomUUID());
|
||||
|
||||
const transcription = waitForSignal(30_000, (resolve) => {
|
||||
const unsubscribe = ctx.client.on("transcription_result", (message) => {
|
||||
@@ -738,7 +738,7 @@ describe("daemon client E2E", () => {
|
||||
}
|
||||
} finally {
|
||||
await Promise.allSettled([transcription, errorSignal]);
|
||||
await ctx.client.setVoiceConversation(false);
|
||||
await ctx.client.setVoiceMode(false);
|
||||
}
|
||||
},
|
||||
90_000
|
||||
|
||||
@@ -182,7 +182,7 @@ const MIN_STREAMING_SEGMENT_BYTES = Math.round(
|
||||
PCM_BYTES_PER_MS * MIN_STREAMING_SEGMENT_DURATION_MS
|
||||
);
|
||||
const SAFE_GIT_REF_PATTERN = /^[A-Za-z0-9._\/-]+$/;
|
||||
const VoiceConversationIdSchema = z.string().uuid();
|
||||
const VoiceAgentIdSchema = z.string().uuid();
|
||||
|
||||
interface AudioBufferState {
|
||||
chunks: Buffer[];
|
||||
@@ -876,21 +876,6 @@ export class Session {
|
||||
}
|
||||
break;
|
||||
|
||||
case "load_voice_conversation_request":
|
||||
await this.handleLoadVoiceConversation(msg.voiceConversationId, msg.requestId);
|
||||
break;
|
||||
|
||||
case "list_voice_conversations_request":
|
||||
await this.handleListVoiceConversations(msg.requestId);
|
||||
break;
|
||||
|
||||
case "delete_voice_conversation_request":
|
||||
await this.handleDeleteVoiceConversation(
|
||||
msg.voiceConversationId,
|
||||
msg.requestId
|
||||
);
|
||||
break;
|
||||
|
||||
case "delete_agent_request":
|
||||
await this.handleDeleteAgentRequest(msg.agentId, msg.requestId);
|
||||
break;
|
||||
@@ -899,8 +884,8 @@ export class Session {
|
||||
await this.handleArchiveAgentRequest(msg.agentId, msg.requestId);
|
||||
break;
|
||||
|
||||
case "set_voice_conversation":
|
||||
await this.handleSetVoiceConversation(msg.enabled, msg.voiceConversationId);
|
||||
case "set_voice_mode":
|
||||
await this.handleSetVoiceMode(msg.enabled, msg.voiceAgentId);
|
||||
break;
|
||||
|
||||
case "send_agent_message_request":
|
||||
@@ -1155,138 +1140,6 @@ export class Session {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a voice conversation into this session (best-effort).
|
||||
*/
|
||||
public async handleLoadVoiceConversation(
|
||||
voiceConversationId: string,
|
||||
requestId: string
|
||||
): Promise<void> {
|
||||
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
||||
voiceConversationId,
|
||||
"load_voice_conversation_request"
|
||||
);
|
||||
this.voiceAssistantAgentId = normalizedVoiceConversationId;
|
||||
|
||||
this.emit({
|
||||
type: "voice_conversation_loaded",
|
||||
payload: {
|
||||
voiceConversationId: normalizedVoiceConversationId,
|
||||
messageCount: 0,
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* List all voice conversations
|
||||
*/
|
||||
public async handleListVoiceConversations(requestId: string): Promise<void> {
|
||||
try {
|
||||
const agents = await this.agentStorage.list();
|
||||
const conversations = agents
|
||||
.filter(
|
||||
(agent) =>
|
||||
agent.labels?.surface === "voice" &&
|
||||
!agent.archivedAt &&
|
||||
!agent.internal
|
||||
)
|
||||
.map((agent) => ({
|
||||
id: agent.id,
|
||||
lastUpdated: new Date(
|
||||
agent.lastActivityAt ?? agent.updatedAt
|
||||
).toISOString(),
|
||||
messageCount: 0,
|
||||
}))
|
||||
.sort(
|
||||
(a, b) =>
|
||||
new Date(b.lastUpdated).getTime() -
|
||||
new Date(a.lastUpdated).getTime()
|
||||
);
|
||||
this.emit({
|
||||
type: "list_voice_conversations_response",
|
||||
payload: {
|
||||
conversations,
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
this.sessionLogger.error(
|
||||
{ err: error },
|
||||
"Failed to list voice conversations"
|
||||
);
|
||||
this.emit({
|
||||
type: "activity_log",
|
||||
payload: {
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: "error",
|
||||
content: `Failed to list voice conversations: ${error.message}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a voice conversation
|
||||
*/
|
||||
public async handleDeleteVoiceConversation(
|
||||
voiceConversationId: string,
|
||||
requestId: string
|
||||
): Promise<void> {
|
||||
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
||||
voiceConversationId,
|
||||
"delete_voice_conversation_request"
|
||||
);
|
||||
try {
|
||||
const record = await this.agentStorage.get(normalizedVoiceConversationId);
|
||||
if (!record || record.labels?.surface !== "voice") {
|
||||
this.emit({
|
||||
type: "delete_voice_conversation_response",
|
||||
payload: {
|
||||
voiceConversationId: normalizedVoiceConversationId,
|
||||
success: false,
|
||||
error: "Voice conversation not found",
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const live = this.agentManager.getAgent(normalizedVoiceConversationId);
|
||||
if (live) {
|
||||
await this.agentManager.closeAgent(normalizedVoiceConversationId);
|
||||
}
|
||||
await this.agentStorage.remove(normalizedVoiceConversationId);
|
||||
this.emit({
|
||||
type: "delete_voice_conversation_response",
|
||||
payload: {
|
||||
voiceConversationId: normalizedVoiceConversationId,
|
||||
success: true,
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
this.sessionLogger.info(
|
||||
{ voiceConversationId: normalizedVoiceConversationId },
|
||||
`Deleted voice conversation ${normalizedVoiceConversationId}`
|
||||
);
|
||||
} catch (error: any) {
|
||||
this.sessionLogger.error(
|
||||
{ err: error, voiceConversationId: normalizedVoiceConversationId },
|
||||
`Failed to delete voice conversation ${normalizedVoiceConversationId}`
|
||||
);
|
||||
this.emit({
|
||||
type: "delete_voice_conversation_response",
|
||||
payload: {
|
||||
voiceConversationId: normalizedVoiceConversationId,
|
||||
success: false,
|
||||
error: error.message,
|
||||
requestId,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async handleRestartServerRequest(
|
||||
requestId: string,
|
||||
reason?: string
|
||||
@@ -1410,28 +1263,30 @@ export class Session {
|
||||
/**
|
||||
* Handle voice mode toggle
|
||||
*/
|
||||
private async handleSetVoiceConversation(
|
||||
private async handleSetVoiceMode(
|
||||
enabled: boolean,
|
||||
voiceConversationId?: string
|
||||
voiceAgentId?: string
|
||||
): Promise<void> {
|
||||
if (enabled) {
|
||||
const candidateVoiceConversationId =
|
||||
voiceConversationId && voiceConversationId.trim().length > 0
|
||||
? voiceConversationId
|
||||
: uuidv4();
|
||||
const normalizedVoiceConversationId = this.parseVoiceConversationId(
|
||||
candidateVoiceConversationId,
|
||||
"set_voice_conversation"
|
||||
);
|
||||
let normalizedVoiceAgentId: string | null = null;
|
||||
if (voiceAgentId && voiceAgentId.trim().length > 0) {
|
||||
normalizedVoiceAgentId = this.parseVoiceAgentId(
|
||||
voiceAgentId,
|
||||
"set_voice_mode"
|
||||
);
|
||||
} else {
|
||||
normalizedVoiceAgentId = await this.findExistingVoiceAssistantAgentId();
|
||||
}
|
||||
|
||||
this.isVoiceMode = true;
|
||||
this.voiceAssistantAgentId = normalizedVoiceConversationId;
|
||||
this.voiceAssistantAgentId = normalizedVoiceAgentId;
|
||||
this.sessionLogger.info(
|
||||
{
|
||||
voiceAssistantAgentId: this.voiceAssistantAgentId,
|
||||
generatedConversationId: !voiceConversationId,
|
||||
resumedVoiceAgent: Boolean(normalizedVoiceAgentId),
|
||||
providedVoiceAgentId: Boolean(voiceAgentId?.trim()),
|
||||
},
|
||||
"Voice conversation enabled (agent-backed)"
|
||||
"Voice mode enabled (agent-backed)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
@@ -1439,18 +1294,45 @@ export class Session {
|
||||
this.isVoiceMode = false;
|
||||
this.sessionLogger.info(
|
||||
{ voiceAssistantAgentId: this.voiceAssistantAgentId },
|
||||
"Voice conversation disabled (agent-backed)"
|
||||
"Voice mode disabled (agent-backed)"
|
||||
);
|
||||
}
|
||||
|
||||
private parseVoiceConversationId(rawId: string, source: string): string {
|
||||
const parsed = VoiceConversationIdSchema.safeParse(rawId.trim());
|
||||
private parseVoiceAgentId(rawId: string, source: string): string {
|
||||
const parsed = VoiceAgentIdSchema.safeParse(rawId.trim());
|
||||
if (!parsed.success) {
|
||||
throw new Error(`${source}: voiceConversationId must be a UUID`);
|
||||
throw new Error(`${source}: voiceAgentId must be a UUID`);
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
private async findExistingVoiceAssistantAgentId(): Promise<string | null> {
|
||||
const scopedCandidates = await this.listAgentPayloads({
|
||||
labels: {
|
||||
surface: "voice",
|
||||
voiceClientId: this.clientId,
|
||||
},
|
||||
});
|
||||
const fallbackCandidates = await this.listAgentPayloads({
|
||||
labels: { surface: "voice" },
|
||||
});
|
||||
const available = [
|
||||
...scopedCandidates,
|
||||
...fallbackCandidates.filter(
|
||||
(candidate) => !scopedCandidates.some((scoped) => scoped.id === candidate.id)
|
||||
),
|
||||
].filter((agent) => !agent.archivedAt);
|
||||
if (available.length === 0) {
|
||||
return null;
|
||||
}
|
||||
available.sort((left, right) => {
|
||||
const leftTime = Date.parse(left.updatedAt);
|
||||
const rightTime = Date.parse(right.updatedAt);
|
||||
return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0);
|
||||
});
|
||||
return available[0]?.id ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle text message to agent (with optional image attachments)
|
||||
*/
|
||||
@@ -4562,7 +4444,7 @@ export class Session {
|
||||
},
|
||||
});
|
||||
|
||||
// Set phase to LLM and process (TTS enabled in voice mode for voice conversations)
|
||||
// Set phase to LLM and process (TTS enabled in voice mode for voice agents)
|
||||
this.clearSpeechInProgress("transcription complete");
|
||||
this.setPhase("llm");
|
||||
this.currentStreamPromise = this.processVoiceTurn(this.isVoiceMode, result.text);
|
||||
@@ -4635,6 +4517,7 @@ export class Session {
|
||||
labels: {
|
||||
surface: "voice",
|
||||
ui: "false",
|
||||
voiceClientId: this.clientId,
|
||||
},
|
||||
});
|
||||
this.voiceAssistantAgentId = created.id;
|
||||
|
||||
@@ -191,7 +191,7 @@ describe("speech models (download E2E)", () => {
|
||||
};
|
||||
});
|
||||
|
||||
await ctx.client.setVoiceConversation(true, randomUUID());
|
||||
await ctx.client.setVoiceMode(true, randomUUID());
|
||||
for (let offset = 0; offset < pcm16.length; offset += chunkBytes) {
|
||||
const chunk = pcm16.subarray(offset, Math.min(pcm16.length, offset + chunkBytes));
|
||||
const isLast = offset + chunkBytes >= pcm16.length;
|
||||
@@ -201,7 +201,7 @@ describe("speech models (download E2E)", () => {
|
||||
if (voiceText.length > 0) {
|
||||
expect(voiceText).toContain("voice note");
|
||||
}
|
||||
await ctx.client.setVoiceConversation(false);
|
||||
await ctx.client.setVoiceMode(false);
|
||||
|
||||
// Streaming TTS: generate locally from downloaded model and validate chunking.
|
||||
const ttsText = "This is a voice note.";
|
||||
|
||||
@@ -72,7 +72,7 @@ function waitForSignal<T>(
|
||||
test(
|
||||
"routes voice turns through local agent speak tool",
|
||||
async () => {
|
||||
await ctx.client.setVoiceConversation(true, randomUUID());
|
||||
await ctx.client.setVoiceMode(true, randomUUID());
|
||||
|
||||
const audioPromise = waitForSignal<{ chunkId: string }>(120000, (resolve, reject) => {
|
||||
const offAudio = ctx.client.on("audio_output", (msg) => {
|
||||
|
||||
@@ -337,23 +337,6 @@ export const UnsubscribeAgentUpdatesMessageSchema = z.object({
|
||||
subscriptionId: z.string(),
|
||||
});
|
||||
|
||||
export const LoadVoiceConversationRequestMessageSchema = z.object({
|
||||
type: z.literal("load_voice_conversation_request"),
|
||||
voiceConversationId: z.string(),
|
||||
requestId: z.string(),
|
||||
});
|
||||
|
||||
export const ListVoiceConversationsRequestMessageSchema = z.object({
|
||||
type: z.literal("list_voice_conversations_request"),
|
||||
requestId: z.string(),
|
||||
});
|
||||
|
||||
export const DeleteVoiceConversationRequestMessageSchema = z.object({
|
||||
type: z.literal("delete_voice_conversation_request"),
|
||||
voiceConversationId: z.string(),
|
||||
requestId: z.string(),
|
||||
});
|
||||
|
||||
export const DeleteAgentRequestMessageSchema = z.object({
|
||||
type: z.literal("delete_agent_request"),
|
||||
agentId: z.string(),
|
||||
@@ -366,10 +349,10 @@ export const ArchiveAgentRequestMessageSchema = z.object({
|
||||
requestId: z.string(),
|
||||
});
|
||||
|
||||
export const SetVoiceConversationMessageSchema = z.object({
|
||||
type: z.literal("set_voice_conversation"),
|
||||
export const SetVoiceModeMessageSchema = z.object({
|
||||
type: z.literal("set_voice_mode"),
|
||||
enabled: z.boolean(),
|
||||
voiceConversationId: z.string().optional(),
|
||||
voiceAgentId: z.string().optional(),
|
||||
});
|
||||
|
||||
export const SendAgentMessageSchema = z.object({
|
||||
@@ -884,12 +867,9 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
|
||||
FetchAgentRequestMessageSchema,
|
||||
SubscribeAgentUpdatesMessageSchema,
|
||||
UnsubscribeAgentUpdatesMessageSchema,
|
||||
LoadVoiceConversationRequestMessageSchema,
|
||||
ListVoiceConversationsRequestMessageSchema,
|
||||
DeleteVoiceConversationRequestMessageSchema,
|
||||
DeleteAgentRequestMessageSchema,
|
||||
ArchiveAgentRequestMessageSchema,
|
||||
SetVoiceConversationMessageSchema,
|
||||
SetVoiceModeMessageSchema,
|
||||
SendAgentMessageRequestSchema,
|
||||
WaitForFinishRequestSchema,
|
||||
DictationStreamStartMessageSchema,
|
||||
@@ -1127,15 +1107,6 @@ export const ArtifactMessageSchema = z.object({
|
||||
}),
|
||||
});
|
||||
|
||||
export const VoiceConversationLoadedMessageSchema = z.object({
|
||||
type: z.literal("voice_conversation_loaded"),
|
||||
payload: z.object({
|
||||
voiceConversationId: z.string(),
|
||||
messageCount: z.number(),
|
||||
requestId: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const AgentUpdateMessageSchema = z.object({
|
||||
type: z.literal("agent_update"),
|
||||
payload: z.discriminatedUnion("kind", [
|
||||
@@ -1225,30 +1196,6 @@ export const WaitForFinishResponseMessageSchema = z.object({
|
||||
}),
|
||||
});
|
||||
|
||||
export const ListVoiceConversationsResponseMessageSchema = z.object({
|
||||
type: z.literal("list_voice_conversations_response"),
|
||||
payload: z.object({
|
||||
conversations: z.array(
|
||||
z.object({
|
||||
id: z.string(),
|
||||
lastUpdated: z.string(),
|
||||
messageCount: z.number(),
|
||||
})
|
||||
),
|
||||
requestId: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const DeleteVoiceConversationResponseMessageSchema = z.object({
|
||||
type: z.literal("delete_voice_conversation_response"),
|
||||
payload: z.object({
|
||||
voiceConversationId: z.string(),
|
||||
success: z.boolean(),
|
||||
error: z.string().optional(),
|
||||
requestId: z.string(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const AgentPermissionRequestMessageSchema = z.object({
|
||||
type: z.literal("agent_permission_request"),
|
||||
payload: z.object({
|
||||
@@ -1679,7 +1626,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
||||
RpcErrorMessageSchema,
|
||||
InitializeAgentResponseMessageSchema,
|
||||
ArtifactMessageSchema,
|
||||
VoiceConversationLoadedMessageSchema,
|
||||
AgentUpdateMessageSchema,
|
||||
AgentStreamMessageSchema,
|
||||
AgentStreamSnapshotMessageSchema,
|
||||
@@ -1691,8 +1637,6 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
||||
SetAgentModelResponseMessageSchema,
|
||||
SetAgentThinkingResponseMessageSchema,
|
||||
WaitForFinishResponseMessageSchema,
|
||||
ListVoiceConversationsResponseMessageSchema,
|
||||
DeleteVoiceConversationResponseMessageSchema,
|
||||
AgentPermissionRequestMessageSchema,
|
||||
AgentPermissionResolvedMessageSchema,
|
||||
AgentDeletedMessageSchema,
|
||||
@@ -1737,9 +1681,6 @@ export type TranscriptionResultMessage = z.infer<typeof TranscriptionResultMessa
|
||||
export type StatusMessage = z.infer<typeof StatusMessageSchema>;
|
||||
export type RpcErrorMessage = z.infer<typeof RpcErrorMessageSchema>;
|
||||
export type ArtifactMessage = z.infer<typeof ArtifactMessageSchema>;
|
||||
export type VoiceConversationLoadedMessage = z.infer<
|
||||
typeof VoiceConversationLoadedMessageSchema
|
||||
>;
|
||||
export type AgentUpdateMessage = z.infer<typeof AgentUpdateMessageSchema>;
|
||||
export type AgentStreamMessage = z.infer<typeof AgentStreamMessageSchema>;
|
||||
export type AgentStreamSnapshotMessage = z.infer<
|
||||
@@ -1758,12 +1699,6 @@ export type SendAgentMessageResponseMessage = z.infer<
|
||||
export type WaitForFinishResponseMessage = z.infer<
|
||||
typeof WaitForFinishResponseMessageSchema
|
||||
>;
|
||||
export type ListVoiceConversationsResponseMessage = z.infer<
|
||||
typeof ListVoiceConversationsResponseMessageSchema
|
||||
>;
|
||||
export type DeleteVoiceConversationResponseMessage = z.infer<
|
||||
typeof DeleteVoiceConversationResponseMessageSchema
|
||||
>;
|
||||
export type AgentPermissionRequestMessage = z.infer<typeof AgentPermissionRequestMessageSchema>;
|
||||
export type AgentPermissionResolvedMessage = z.infer<typeof AgentPermissionResolvedMessageSchema>;
|
||||
export type AgentDeletedMessage = z.infer<typeof AgentDeletedMessageSchema>;
|
||||
|
||||
Reference in New Issue
Block a user