From a644adbf9b2091917ec9ab604c81fd4fb42c6feb Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Sat, 14 Feb 2026 13:14:57 +0700 Subject: [PATCH] feat: add voice capability status reporting to client --- packages/app/src/components/message-input.tsx | 44 +++++-- packages/app/src/contexts/session-context.tsx | 16 +-- packages/app/src/contexts/voice-context.tsx | 8 ++ .../src/dictation/dictation-stream-sender.ts | 6 +- packages/app/src/hooks/use-dictation.ts | 10 +- packages/app/src/stores/session-store.ts | 23 +++- .../utils/server-info-capabilities.test.ts | 118 ++++++++++++++++++ .../app/src/utils/server-info-capabilities.ts | 50 ++++++++ .../app/src/utils/test-daemon-connection.ts | 14 +-- packages/server/src/server/bootstrap.ts | 7 ++ .../src/server/daemon-client.e2e.test.ts | 8 +- .../src/server/speech/speech-runtime.ts | 64 +++++++++- .../websocket-server.relay-reconnect.test.ts | 117 ++++++++++++++++- .../server/src/server/websocket-server.ts | 77 +++++++++++- packages/server/src/shared/messages.ts | 67 ++++++++++ scripts/codex-refactor-loop.sh | 54 -------- 16 files changed, 585 insertions(+), 98 deletions(-) create mode 100644 packages/app/src/utils/server-info-capabilities.test.ts create mode 100644 packages/app/src/utils/server-info-capabilities.ts delete mode 100755 scripts/codex-refactor-loop.sh diff --git a/packages/app/src/components/message-input.tsx b/packages/app/src/components/message-input.tsx index 8bf1445cd..e36d22a28 100644 --- a/packages/app/src/components/message-input.tsx +++ b/packages/app/src/components/message-input.tsx @@ -32,7 +32,9 @@ import { DictationOverlay } from "./dictation-controls"; import { RealtimeVoiceOverlay } from "./realtime-voice-overlay"; import type { DaemonClient } from "@server/client/daemon-client"; import { usePanelStore } from "@/stores/panel-store"; +import { useSessionStore } from "@/stores/session-store"; import { useVoiceOptional } from "@/contexts/voice-context"; +import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { Shortcut } from "@/components/ui/shortcut"; @@ -167,6 +169,17 @@ export const MessageInput = forwardRef( const overlayTransition = useSharedValue(0); const sendAfterTranscriptRef = useRef(false); const valueRef = useRef(value); + const serverInfo = useSessionStore( + useCallback( + (state) => { + if (!voiceServerId) { + return null; + } + return state.sessions[voiceServerId]?.serverInfo ?? null; + }, + [voiceServerId] + ) + ); useEffect(() => { valueRef.current = value; @@ -212,10 +225,15 @@ export const MessageInput = forwardRef( console.error("[MessageInput] Dictation error:", error); }, []); + const dictationUnavailableMessage = resolveVoiceUnavailableMessage({ + serverInfo, + mode: "dictation", + }); + const canStartDictation = useCallback(() => { const socketConnected = client?.isConnected ?? false; - return socketConnected && !disabled; - }, [client, disabled]); + return socketConnected && !disabled && !dictationUnavailableMessage; + }, [client, disabled, dictationUnavailableMessage]); const canConfirmDictation = useCallback(() => { const socketConnected = client?.isConnected ?? false; @@ -268,6 +286,14 @@ export const MessageInput = forwardRef( sendAfterTranscriptRef.current = false; }, [dictationStatus, isDictating, isDictationProcessing]); + const startDictationIfAvailable = useCallback(async () => { + if (dictationUnavailableMessage) { + Alert.alert("Dictation unavailable", dictationUnavailableMessage); + return; + } + await startDictation(); + }, [dictationUnavailableMessage, startDictation]); + // Cmd+D to start/submit dictation, Cmd+Shift+D toggles realtime voice, Escape cancels dictation useEffect(() => { if (!IS_WEB) return; @@ -298,7 +324,7 @@ export const MessageInput = forwardRef( } void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => { console.error("[MessageInput] Failed to start realtime voice", error); - const message = error instanceof Error ? error.message : "Voice features are not available right now."; + const message = error instanceof Error ? error.message : String(error); Alert.alert("Voice unavailable", message); }); }; @@ -348,7 +374,7 @@ export const MessageInput = forwardRef( sendAfterTranscriptRef.current = true; confirmDictation(); } else { - startDictation(); + void startDictationIfAvailable(); } return; } @@ -369,7 +395,7 @@ export const MessageInput = forwardRef( isConnected, isRealtimeVoiceForCurrentAgent, isScreenFocused, - startDictation, + startDictationIfAvailable, voiceAgentId, voiceServerId, voice, @@ -400,13 +426,13 @@ export const MessageInput = forwardRef( if (isDictating) { await cancelDictation(); } else { - await startDictation(); + await startDictationIfAvailable(); } }, [ cancelDictation, isDictating, isRealtimeVoiceForCurrentAgent, - startDictation, + startDictationIfAvailable, voice, ]); @@ -464,7 +490,7 @@ export const MessageInput = forwardRef( } void voice.startVoice(voiceServerId, voiceAgentId).catch((error) => { console.error("[MessageInput] Failed to start realtime voice", error); - const message = error instanceof Error ? error.message : "Voice features are not available right now."; + const message = error instanceof Error ? error.message : String(error); Alert.alert("Voice unavailable", message); }); }, [ @@ -623,7 +649,7 @@ export const MessageInput = forwardRef( sendAfterTranscriptRef.current = true; confirmDictation(); } else { - startDictation(); + void startDictationIfAvailable(); } return; } diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index f37c4eee6..cfd760e13 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -18,6 +18,7 @@ import type { AgentStreamEventPayload, SessionOutboundMessage, } from "@server/shared/messages"; +import { parseServerInfoStatusPayload } from "@server/shared/messages"; import type { AgentLifecycleStatus } from "@server/shared/agent-lifecycle"; import type { AgentPermissionRequest } from "@server/server/agent/agent-sdk-types"; import type { DaemonClient, ConnectionState } from "@server/client/daemon-client"; @@ -984,15 +985,14 @@ export function SessionProvider({ const unsubStatus = client.on("status", (message) => { if (message.type !== "status") return; - const status = message.payload.status; - if (status === "server_info") { - const payload = message.payload as any; - const rawServerId = typeof payload.serverId === "string" ? payload.serverId.trim() : ""; - if (!rawServerId) return; - const rawHostname = typeof payload.hostname === "string" ? payload.hostname.trim() : ""; + const serverInfo = parseServerInfoStatusPayload(message.payload); + if (serverInfo) { updateSessionServerInfo(serverId, { - serverId: rawServerId, - hostname: rawHostname.length > 0 ? rawHostname : null, + serverId: serverInfo.serverId, + hostname: serverInfo.hostname, + ...(serverInfo.capabilities + ? { capabilities: serverInfo.capabilities } + : {}), }); return; } diff --git a/packages/app/src/contexts/voice-context.tsx b/packages/app/src/contexts/voice-context.tsx index 6d7995e9b..8a75afd8f 100644 --- a/packages/app/src/contexts/voice-context.tsx +++ b/packages/app/src/contexts/voice-context.tsx @@ -2,6 +2,7 @@ import { createContext, useContext, useState, ReactNode, useCallback, useEffect, import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio"; import type { SessionState } from "@/stores/session-store"; import { useSessionStore } from "@/stores/session-store"; +import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities"; import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config"; @@ -334,6 +335,13 @@ export function VoiceProvider({ children }: VoiceProviderProps) { if (!session) { throw new Error(`Host ${serverId} is not connected`); } + const unavailableMessage = resolveVoiceUnavailableMessage({ + serverInfo: session.serverInfo, + mode: "voice", + }); + if (unavailableMessage) { + throw new Error(unavailableMessage); + } setIsVoiceSwitching(true); voiceTransportReadyRef.current = false; diff --git a/packages/app/src/dictation/dictation-stream-sender.ts b/packages/app/src/dictation/dictation-stream-sender.ts index 82d11ee42..ec0dc7ae7 100644 --- a/packages/app/src/dictation/dictation-stream-sender.ts +++ b/packages/app/src/dictation/dictation-stream-sender.ts @@ -86,7 +86,11 @@ export class DictationStreamSender { } if (!this.dictationId) { - void this.restartStream("enqueue"); + if (!this.startPromise) { + void this.restartStream("enqueue").catch((error) => { + console.error("[DictationStreamSender] Failed to start stream from enqueue", error); + }); + } return; } diff --git a/packages/app/src/hooks/use-dictation.ts b/packages/app/src/hooks/use-dictation.ts index 0001f11e0..6e8da3f38 100644 --- a/packages/app/src/hooks/use-dictation.ts +++ b/packages/app/src/hooks/use-dictation.ts @@ -159,9 +159,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { if (!isRecordingRef.current) { return; } - void startNewStream("reconnect"); + void startNewStream("reconnect").catch((error) => { + reportError(error, "Failed to restart dictation stream after reconnect"); + }); }); - }, [client, startNewStream]); + }, [client, reportError, startNewStream]); useEffect(() => { if (!client) { @@ -260,7 +262,7 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { try { await audio.start(); if (client?.isConnected) { - void startNewStream("start"); + await startNewStream("start"); } isRecordingRef.current = true; setIsRecording(true); @@ -268,9 +270,11 @@ export function useDictation(options: UseDictationOptions): UseDictationResult { startDurationTracking(); } } catch (err) { + await audio.stop().catch(() => undefined); stopDurationTracking(); isRecordingRef.current = false; setIsRecording(false); + setStatus("idle"); reportError(err, "Failed to start dictation"); } finally { actionGateRef.current.starting = false; diff --git a/packages/app/src/stores/session-store.ts b/packages/app/src/stores/session-store.ts index 214354ca3..4520365d5 100644 --- a/packages/app/src/stores/session-store.ts +++ b/packages/app/src/stores/session-store.ts @@ -21,6 +21,7 @@ import type { FileDownloadTokenResponse, GitSetupOptions, ProjectPlacementPayload, + ServerCapabilities, } from "@server/shared/messages"; import { isPerfLoggingEnabled, measurePayload, perfLog } from "@/utils/perf"; @@ -155,6 +156,7 @@ export interface DaemonConnectionSnapshot { export type DaemonServerInfo = { serverId: string; hostname: string | null; + capabilities?: ServerCapabilities; }; export interface AgentTimelineCursorState { @@ -345,6 +347,13 @@ function createInitialSessionState(serverId: string, client: DaemonClient, audio }; } +function areServerCapabilitiesEqual( + current: ServerCapabilities | undefined, + next: ServerCapabilities | undefined +): boolean { + return JSON.stringify(current ?? null) === JSON.stringify(next ?? null); +} + export const useSessionStore = create()( subscribeWithSelector((set, get) => ({ sessions: {}, @@ -446,8 +455,14 @@ export const useSessionStore = create()( const nextHostname = info.hostname?.trim() || null; const prevHostname = session.serverInfo?.hostname?.trim() || null; + const nextCapabilities = info.capabilities; + const prevCapabilities = session.serverInfo?.capabilities; - if (session.serverInfo?.serverId === info.serverId && prevHostname === nextHostname) { + if ( + session.serverInfo?.serverId === info.serverId && + prevHostname === nextHostname && + areServerCapabilitiesEqual(prevCapabilities, nextCapabilities) + ) { return prev; } @@ -462,7 +477,11 @@ export const useSessionStore = create()( ...prev.sessions, [serverId]: { ...session, - serverInfo: { serverId: info.serverId, hostname: nextHostname }, + serverInfo: { + serverId: info.serverId, + hostname: nextHostname, + ...(nextCapabilities ? { capabilities: nextCapabilities } : {}), + }, }, }, }; diff --git a/packages/app/src/utils/server-info-capabilities.test.ts b/packages/app/src/utils/server-info-capabilities.test.ts new file mode 100644 index 000000000..796eb789e --- /dev/null +++ b/packages/app/src/utils/server-info-capabilities.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from "vitest"; +import type { ServerCapabilities } from "@server/shared/messages"; +import type { DaemonServerInfo } from "@/stores/session-store"; +import { + getServerCapabilities, + getVoiceReadinessState, + resolveVoiceUnavailableMessage, +} from "./server-info-capabilities"; + +function buildServerInfo(capabilities?: ServerCapabilities): DaemonServerInfo { + return { + serverId: "srv-1", + hostname: "test-host", + ...(capabilities ? { capabilities } : {}), + }; +} + +describe("server-info-capabilities", () => { + it("returns null capabilities when server_info does not include capability metadata", () => { + const serverInfo = buildServerInfo(); + expect(getServerCapabilities({ serverInfo })).toBeNull(); + }); + + it("returns the matching voice capability state by mode", () => { + const capabilities: ServerCapabilities = { + voice: { + dictation: { + enabled: true, + reason: "Dictation is warming up.", + }, + voice: { + enabled: false, + reason: "Voice is disabled in daemon config.", + }, + }, + }; + const serverInfo = buildServerInfo(capabilities); + + expect( + getVoiceReadinessState({ + serverInfo, + mode: "dictation", + }) + ).toEqual(capabilities.voice?.dictation); + expect( + getVoiceReadinessState({ + serverInfo, + mode: "voice", + }) + ).toEqual(capabilities.voice?.voice); + }); + + it("returns null when capability is enabled and has no reason", () => { + const serverInfo = buildServerInfo({ + voice: { + dictation: { + enabled: true, + reason: "", + }, + voice: { + enabled: true, + reason: "", + }, + }, + }); + + expect( + resolveVoiceUnavailableMessage({ + serverInfo, + mode: "dictation", + }) + ).toBeNull(); + }); + + it("returns capability reason when present", () => { + const serverInfo = buildServerInfo({ + voice: { + dictation: { + enabled: true, + reason: "Dictation models are still downloading.", + }, + voice: { + enabled: true, + reason: "", + }, + }, + }); + + expect( + resolveVoiceUnavailableMessage({ + serverInfo, + mode: "dictation", + }) + ).toBe("Dictation models are still downloading."); + }); + + it("returns null when capability reason is blank", () => { + const serverInfo = buildServerInfo({ + voice: { + dictation: { + enabled: false, + reason: " ", + }, + voice: { + enabled: true, + reason: "", + }, + }, + }); + + expect( + resolveVoiceUnavailableMessage({ + serverInfo, + mode: "dictation", + }) + ).toBeNull(); + }); +}); diff --git a/packages/app/src/utils/server-info-capabilities.ts b/packages/app/src/utils/server-info-capabilities.ts new file mode 100644 index 000000000..fdba66554 --- /dev/null +++ b/packages/app/src/utils/server-info-capabilities.ts @@ -0,0 +1,50 @@ +import type { ServerCapabilityState } from "@server/shared/messages"; +import type { DaemonServerInfo } from "@/stores/session-store"; + +export type VoiceReadinessMode = "dictation" | "voice"; + +export function getServerCapabilities(params: { + serverInfo: DaemonServerInfo | null | undefined; +}): DaemonServerInfo["capabilities"] | null { + const capabilities = params.serverInfo?.capabilities; + if (!capabilities) { + return null; + } + return capabilities; +} + +export function getVoiceReadinessState(params: { + serverInfo: DaemonServerInfo | null | undefined; + mode: VoiceReadinessMode; +}): ServerCapabilityState | null { + const capabilities = getServerCapabilities({ serverInfo: params.serverInfo }); + const voice = capabilities?.voice; + if (!voice) { + return null; + } + if (params.mode === "dictation") { + return voice.dictation; + } + return voice.voice; +} + +export function resolveVoiceUnavailableMessage(params: { + serverInfo: DaemonServerInfo | null | undefined; + mode: VoiceReadinessMode; +}): string | null { + const readiness = getVoiceReadinessState({ + serverInfo: params.serverInfo, + mode: params.mode, + }); + if (!readiness) { + return null; + } + if (readiness.enabled && readiness.reason.trim().length === 0) { + return null; + } + const message = readiness.reason.trim(); + if (message.length > 0) { + return message; + } + return null; +} diff --git a/packages/app/src/utils/test-daemon-connection.ts b/packages/app/src/utils/test-daemon-connection.ts index d0a1adde6..7a460be35 100644 --- a/packages/app/src/utils/test-daemon-connection.ts +++ b/packages/app/src/utils/test-daemon-connection.ts @@ -1,5 +1,6 @@ import { DaemonClient } from "@server/client/daemon-client"; import type { DaemonClientConfig } from "@server/client/daemon-client"; +import { parseServerInfoStatusPayload } from "@server/shared/messages"; import type { HostConnection } from "@/contexts/daemon-registry-context"; import { buildDaemonWebSocketUrl, buildRelayWebSocketUrl } from "./daemon-endpoints"; import { createTauriWebSocketTransportFactory } from "./tauri-daemon-transport"; @@ -116,15 +117,10 @@ function connectAndProbe( unsubscribeStatus = client.on("status", (message) => { if (message.type !== "status") return; - const payload = message.payload as { status?: unknown; serverId?: unknown; hostname?: unknown }; - if (payload?.status !== "server_info") return; - const raw = typeof payload.serverId === "string" ? payload.serverId.trim() : ""; - if (!raw) return; - serverId = raw; - hostname = typeof payload.hostname === "string" ? payload.hostname.trim() : null; - if (hostname && hostname.length === 0) { - hostname = null; - } + const payload = parseServerInfoStatusPayload(message.payload); + if (!payload) return; + serverId = payload.serverId; + hostname = payload.hostname; maybeFinishOk(); }); diff --git a/packages/server/src/server/bootstrap.ts b/packages/server/src/server/bootstrap.ts index efe0858e7..86a8ab264 100644 --- a/packages/server/src/server/bootstrap.ts +++ b/packages/server/src/server/bootstrap.ts @@ -299,6 +299,7 @@ export async function createPaseoDaemon( ); let wsServer: VoiceAssistantWebSocketServer | null = null; let voiceMcpBridgeManager: VoiceMcpSocketBridgeManager | null = null; + let unsubscribeSpeechReadiness: (() => void) | null = null; // Create in-memory transport for Session's Agent MCP client (voice assistant tools) const createInMemoryAgentMcpTransport = async (): Promise => { @@ -461,6 +462,7 @@ export async function createPaseoDaemon( resolveVoiceTts, resolveDictationStt, getSpeechReadiness, + subscribeSpeechReadiness, cleanup: cleanupSpeechRuntime, localModelConfig, } = await initializeSpeechRuntime({ @@ -503,6 +505,9 @@ export async function createPaseoDaemon( }, config.agentProviderSettings ); + unsubscribeSpeechReadiness = subscribeSpeechReadiness((snapshot) => { + wsServer?.publishSpeechReadiness(snapshot); + }); const start = async () => { // Start main HTTP server @@ -582,6 +587,8 @@ export async function createPaseoDaemon( runtimeSettings: config.agentProviderSettings, }); terminalManager.killAll(); + unsubscribeSpeechReadiness?.(); + unsubscribeSpeechReadiness = null; cleanupSpeechRuntime(); await relayTransport?.stop().catch(() => undefined); if (wsServer) { diff --git a/packages/server/src/server/daemon-client.e2e.test.ts b/packages/server/src/server/daemon-client.e2e.test.ts index 25c239fd1..81aca18aa 100644 --- a/packages/server/src/server/daemon-client.e2e.test.ts +++ b/packages/server/src/server/daemon-client.e2e.test.ts @@ -11,6 +11,7 @@ import { DaemonClient, } from "./test-utils/index.js"; import { getFullAccessConfig, getAskModeConfig } from "./daemon-e2e/agent-configs.js"; +import { parseServerInfoStatusPayload } from "./messages.js"; import { chunkPcm16, parsePcm16MonoWav, @@ -175,10 +176,9 @@ describe("daemon client E2E", () => { const infoPromise = waitForSignal<{ serverId: string }>(5000, (resolve) => { const unsubscribe = client.on("status", (message) => { if (message.type !== "status") return; - const payload = message.payload as { status?: unknown; serverId?: unknown }; - if (payload.status !== "server_info") return; - if (typeof payload.serverId !== "string" || payload.serverId.trim().length === 0) return; - resolve({ serverId: payload.serverId.trim() }); + const payload = parseServerInfoStatusPayload(message.payload); + if (!payload) return; + resolve({ serverId: payload.serverId }); }); return unsubscribe; }); diff --git a/packages/server/src/server/speech/speech-runtime.ts b/packages/server/src/server/speech/speech-runtime.ts index 6b88be981..0492fb317 100644 --- a/packages/server/src/server/speech/speech-runtime.ts +++ b/packages/server/src/server/speech/speech-runtime.ts @@ -308,6 +308,9 @@ export type InitializedSpeechRuntime = { resolveVoiceTts: () => TextToSpeechProvider | null; resolveDictationStt: () => SpeechToTextProvider | null; getSpeechReadiness: () => SpeechReadinessSnapshot; + subscribeSpeechReadiness: ( + listener: (snapshot: SpeechReadinessSnapshot) => void + ) => () => void; cleanup: () => void; localModelConfig: { modelsDir: string; @@ -359,6 +362,9 @@ export async function initializeSpeechRuntime(params: { let stopped = false; let monitorTimeout: ReturnType | null = null; let reconcileInFlight: Promise | null = null; + const readinessListeners = new Set<(snapshot: SpeechReadinessSnapshot) => void>(); + let lastReadinessFingerprint: string | null = null; + let lastPublishedReadinessSnapshot: SpeechReadinessSnapshot | null = null; const computeReadinessSnapshot = (): SpeechReadinessSnapshot => { const realtimeVoice = buildRealtimeVoiceReadiness({ @@ -398,6 +404,54 @@ export async function initializeSpeechRuntime(params: { }; }; + const readinessFingerprint = (snapshot: SpeechReadinessSnapshot): string => + JSON.stringify({ + ...snapshot, + generatedAt: "", + }); + + const publishReadinessIfChanged = (): void => { + const snapshot = computeReadinessSnapshot(); + const fingerprint = readinessFingerprint(snapshot); + if (fingerprint === lastReadinessFingerprint) { + return; + } + lastReadinessFingerprint = fingerprint; + lastPublishedReadinessSnapshot = snapshot; + for (const listener of readinessListeners) { + try { + listener(snapshot); + } catch (error) { + logger.warn( + { err: error }, + "Speech readiness listener threw an error" + ); + } + } + }; + + const subscribeSpeechReadiness = ( + listener: (snapshot: SpeechReadinessSnapshot) => void + ): (() => void) => { + readinessListeners.add(listener); + const snapshot = lastPublishedReadinessSnapshot ?? computeReadinessSnapshot(); + if (!lastPublishedReadinessSnapshot) { + lastPublishedReadinessSnapshot = snapshot; + lastReadinessFingerprint = readinessFingerprint(snapshot); + } + try { + listener(snapshot); + } catch (error) { + logger.warn( + { err: error }, + "Speech readiness listener threw an error during subscribe" + ); + } + return () => { + readinessListeners.delete(listener); + }; + }; + const refreshMissingLocalModels = async (): Promise => { missingLocalModelIds = await findMissingRequiredLocalModels({ modelsDir: localModelConfig?.modelsDir ?? null, @@ -470,12 +524,14 @@ export async function initializeSpeechRuntime(params: { const runReconcile = async (params: { modelEnsureAutoDownload: boolean }): Promise => { if (reconcileInFlight) { await reconcileInFlight; + publishReadinessIfChanged(); return; } reconcileInFlight = reconcileServices(params.modelEnsureAutoDownload).finally(() => { reconcileInFlight = null; }); await reconcileInFlight; + publishReadinessIfChanged(); }; const scheduleMonitor = (): void => { @@ -500,6 +556,7 @@ export async function initializeSpeechRuntime(params: { backgroundDownloadInProgress = true; backgroundDownloadError = null; + publishReadinessIfChanged(); logger.info( { @@ -521,6 +578,7 @@ export async function initializeSpeechRuntime(params: { backgroundDownloadError = null; } catch (error) { backgroundDownloadError = error instanceof Error ? error.message : String(error); + publishReadinessIfChanged(); logger.error( { err: error, @@ -533,6 +591,7 @@ export async function initializeSpeechRuntime(params: { await refreshMissingLocalModels().catch((error) => { logger.warn({ err: error }, "Failed to refresh local speech model status after download"); }); + publishReadinessIfChanged(); scheduleMonitor(); } })(); @@ -565,6 +624,7 @@ export async function initializeSpeechRuntime(params: { } catch (error) { logger.warn({ err: error }, "Speech runtime monitor tick failed"); } finally { + publishReadinessIfChanged(); scheduleMonitor(); } }; @@ -591,7 +651,9 @@ export async function initializeSpeechRuntime(params: { resolveVoiceStt: () => sttService, resolveVoiceTts: () => ttsService, resolveDictationStt: () => dictationSttService, - getSpeechReadiness: () => computeReadinessSnapshot(), + getSpeechReadiness: () => + lastPublishedReadinessSnapshot ?? computeReadinessSnapshot(), + subscribeSpeechReadiness, cleanup, localModelConfig, }; diff --git a/packages/server/src/server/websocket-server.relay-reconnect.test.ts b/packages/server/src/server/websocket-server.relay-reconnect.test.ts index 37eeae34a..eb2849f2f 100644 --- a/packages/server/src/server/websocket-server.relay-reconnect.test.ts +++ b/packages/server/src/server/websocket-server.relay-reconnect.test.ts @@ -76,6 +76,8 @@ import { VoiceAssistantWebSocketServer, type ExternalSocketMetadata, } from "./websocket-server"; +import { parseServerInfoStatusPayload } from "./messages.js"; +import type { SpeechReadinessSnapshot } from "./speech/speech-runtime.js"; class MockSocket { readyState = 1; @@ -133,7 +135,8 @@ function createLogger() { return logger; } -function createServer() { +function createServer(options?: { speechReadiness?: SpeechReadinessSnapshot | null }) { + const speechReadiness = options?.speechReadiness ?? null; return new VoiceAssistantWebSocketServer( {} as any, createLogger() as any, @@ -146,10 +149,54 @@ function createServer() { {} as any, "/tmp/paseo-test", async () => ({} as any), - { allowedOrigins: new Set() } + { allowedOrigins: new Set() }, + undefined, + undefined, + undefined, + speechReadiness + ? { + getSpeechReadiness: () => speechReadiness, + } + : undefined ); } +function createReadySpeechReadinessSnapshot(): SpeechReadinessSnapshot { + return { + generatedAt: "2026-02-14T00:00:00.000Z", + requiredLocalModelIds: [], + missingLocalModelIds: [], + download: { + inProgress: false, + error: null, + }, + dictation: { + enabled: true, + available: true, + reasonCode: "ready", + message: "Dictation is ready.", + retryable: false, + missingModelIds: [], + }, + realtimeVoice: { + enabled: true, + available: true, + reasonCode: "ready", + message: "Realtime voice is ready.", + retryable: false, + missingModelIds: [], + }, + voiceFeature: { + enabled: true, + available: true, + reasonCode: "ready", + message: "Voice features are ready.", + retryable: false, + missingModelIds: [], + }, + }; +} + describe("relay external socket reconnect behavior", () => { beforeEach(() => { sessionMock.instances.length = 0; @@ -205,6 +252,72 @@ describe("relay external socket reconnect behavior", () => { await server.close(); }); + test("includes voice capabilities in server_info when speech readiness exists", async () => { + const speechReadiness = createReadySpeechReadinessSnapshot(); + const server = createServer({ speechReadiness }); + const metadata: ExternalSocketMetadata = { + transport: "relay", + externalSessionKey: "relay:client-server-info-capabilities", + }; + + const socket = new MockSocket(); + await server.attachExternalSocket(socket, metadata); + + expect(socket.sent).toHaveLength(1); + expect(typeof socket.sent[0]).toBe("string"); + const envelope = JSON.parse(socket.sent[0] as string) as { + type?: unknown; + message?: { + type?: unknown; + payload?: unknown; + }; + }; + expect(envelope.type).toBe("session"); + expect(envelope.message?.type).toBe("status"); + + const payload = parseServerInfoStatusPayload(envelope.message?.payload); + expect(payload?.status).toBe("server_info"); + expect(payload?.capabilities?.voice?.dictation.enabled).toBe( + speechReadiness.dictation.enabled + ); + expect(payload?.capabilities?.voice?.dictation.reason).toBe(""); + expect(payload?.capabilities?.voice?.voice.enabled).toBe( + speechReadiness.realtimeVoice.enabled + ); + expect(payload?.capabilities?.voice?.voice.reason).toBe(""); + + await server.close(); + }); + + test("broadcasts updated server_info when capabilities change", async () => { + const server = createServer(); + const metadata: ExternalSocketMetadata = { + transport: "relay", + externalSessionKey: "relay:client-server-info-broadcast", + }; + + const socket = new MockSocket(); + await server.attachExternalSocket(socket, metadata); + expect(socket.sent).toHaveLength(1); + + const speechReadiness = createReadySpeechReadinessSnapshot(); + server.publishSpeechReadiness(speechReadiness); + expect(socket.sent).toHaveLength(2); + + const secondEnvelope = JSON.parse(socket.sent[1] as string) as { + message?: { payload?: unknown }; + }; + const secondPayload = parseServerInfoStatusPayload(secondEnvelope.message?.payload); + expect(secondPayload?.capabilities?.voice?.dictation.enabled).toBe(true); + expect(secondPayload?.capabilities?.voice?.voice.enabled).toBe(true); + + // Same readiness should not produce another server_info broadcast. + server.publishSpeechReadiness(speechReadiness); + expect(socket.sent).toHaveLength(2); + + await server.close(); + }); + test("routes inbound binary mux frames to session.handleBinaryFrame", async () => { const server = createServer(); const metadata: ExternalSocketMetadata = { diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 062527c53..cb221172c 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -9,7 +9,10 @@ import type { DownloadTokenStore } from "./file-download/token-store.js"; import type { TerminalManager } from "../terminal/terminal-manager.js"; import type pino from "pino"; import { + type ServerInfoStatusPayload, WSInboundMessageSchema, + type ServerCapabilityState, + type ServerCapabilities, type WSOutboundMessage, wrapSessionMessage, } from "./messages.js"; @@ -51,6 +54,37 @@ type WebSocketServerConfig = { allowedHosts?: AllowedHostsConfig; }; +function toServerCapabilityState( + state: SpeechReadinessSnapshot["dictation"] +): ServerCapabilityState { + return { + enabled: state.enabled, + reason: state.available ? "" : state.message, + }; +} + +function buildServerCapabilities(params: { + readiness: SpeechReadinessSnapshot | null; +}): ServerCapabilities | undefined { + const readiness = params.readiness; + if (!readiness) { + return undefined; + } + return { + voice: { + dictation: toServerCapabilityState(readiness.dictation), + voice: toServerCapabilityState(readiness.realtimeVoice), + }, + }; +} + +function areServerCapabilitiesEqual( + current: ServerCapabilities | undefined, + next: ServerCapabilities | undefined +): boolean { + return JSON.stringify(current ?? null) === JSON.stringify(next ?? null); +} + function bufferFromWsData(data: Buffer | ArrayBuffer | Buffer[] | string): Buffer { if (typeof data === "string") return Buffer.from(data, "utf8"); if (Array.isArray(data)) { @@ -123,6 +157,7 @@ export class VoiceAssistantWebSocketServer { >(); private readonly voiceCallerContexts = new Map(); private readonly agentProviderRuntimeSettings: AgentProviderRuntimeSettingsMap | undefined; + private serverCapabilities: ServerCapabilities | undefined; constructor( server: HTTPServer, @@ -168,6 +203,9 @@ export class VoiceAssistantWebSocketServer { this.voice = voice ?? null; this.dictation = dictation ?? null; this.agentProviderRuntimeSettings = agentProviderRuntimeSettings; + this.serverCapabilities = buildServerCapabilities({ + readiness: this.dictation?.getSpeechReadiness?.() ?? null, + }); const pushLogger = this.logger.child({ module: "push" }); this.pushTokenStore = new PushTokenStore( @@ -230,6 +268,21 @@ export class VoiceAssistantWebSocketServer { } } + public publishSpeechReadiness(readiness: SpeechReadinessSnapshot | null): void { + this.updateServerCapabilities(buildServerCapabilities({ readiness })); + } + + public updateServerCapabilities( + capabilities: ServerCapabilities | null | undefined + ): void { + const next = capabilities ?? undefined; + if (areServerCapabilitiesEqual(this.serverCapabilities, next)) { + return; + } + this.serverCapabilities = next; + this.broadcastServerInfo(); + } + public async attachExternalSocket( ws: WebSocketLike, metadata?: ExternalSocketMetadata @@ -410,17 +463,31 @@ export class VoiceAssistantWebSocketServer { this.bindSocketHandlers(ws, connection); } + private buildServerInfoStatusPayload(): ServerInfoStatusPayload { + return { + status: "server_info", + serverId: this.serverId, + hostname: getHostname(), + ...(this.serverCapabilities ? { capabilities: this.serverCapabilities } : {}), + }; + } + + private broadcastServerInfo(): void { + this.broadcast( + wrapSessionMessage({ + type: "status", + payload: this.buildServerInfoStatusPayload(), + }) + ); + } + private sendServerInfo(ws: WebSocketLike): void { // Advertise stable server identity immediately on connect (used for URL/shareable IDs). this.sendToClient( ws, wrapSessionMessage({ type: "status", - payload: { - status: "server_info", - serverId: this.serverId, - hostname: getHostname(), - }, + payload: this.buildServerInfoStatusPayload(), }) ); } diff --git a/packages/server/src/shared/messages.ts b/packages/server/src/shared/messages.ts index e145161f4..81d7a9058 100644 --- a/packages/server/src/shared/messages.ts +++ b/packages/server/src/shared/messages.ts @@ -1168,6 +1168,59 @@ export const DictationStreamErrorMessageSchema = z.object({ }), }); +export const ServerCapabilityStateSchema = z.object({ + enabled: z.boolean(), + reason: z.string(), +}); + +export const ServerVoiceCapabilitiesSchema = z.object({ + dictation: ServerCapabilityStateSchema, + voice: ServerCapabilityStateSchema, +}); + +export const ServerCapabilitiesSchema = z + .object({ + voice: ServerVoiceCapabilitiesSchema.optional(), + }) + .passthrough(); + +const ServerInfoHostnameSchema = z + .unknown() + .transform((value): string | null => { + if (typeof value !== "string") { + return null; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; + }); + +const ServerCapabilitiesFromUnknownSchema = z + .unknown() + .optional() + .transform((value): z.infer | undefined => { + if (value === undefined) { + return undefined; + } + const parsed = ServerCapabilitiesSchema.safeParse(value); + if (!parsed.success) { + return undefined; + } + return parsed.data; + }); + +export const ServerInfoStatusPayloadSchema = z + .object({ + status: z.literal("server_info"), + serverId: z.string().trim().min(1), + hostname: ServerInfoHostnameSchema.optional(), + capabilities: ServerCapabilitiesFromUnknownSchema, + }) + .passthrough() + .transform((payload) => ({ + ...payload, + hostname: payload.hostname ?? null, + })); + export const StatusMessageSchema = z.object({ type: z.literal("status"), payload: z @@ -1975,6 +2028,10 @@ export type AssistantChunkMessage = z.infer; export type AudioOutputMessage = z.infer; export type TranscriptionResultMessage = z.infer; export type StatusMessage = z.infer; +export type ServerCapabilityState = z.infer; +export type ServerVoiceCapabilities = z.infer; +export type ServerCapabilities = z.infer; +export type ServerInfoStatusPayload = z.infer; export type RpcErrorMessage = z.infer; export type ArtifactMessage = z.infer; export type AgentUpdateMessage = z.infer; @@ -2189,3 +2246,13 @@ export function wrapSessionMessage( message: sessionMsg, }; } + +export function parseServerInfoStatusPayload( + payload: unknown +): ServerInfoStatusPayload | null { + const parsed = ServerInfoStatusPayloadSchema.safeParse(payload); + if (!parsed.success) { + return null; + } + return parsed.data; +} diff --git a/scripts/codex-refactor-loop.sh b/scripts/codex-refactor-loop.sh deleted file mode 100755 index 7c9200aa8..000000000 --- a/scripts/codex-refactor-loop.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -cd "${REPO_ROOT}" - -SLEEP_SECONDS="${SLEEP_SECONDS:-600}" -MAX_ITERATIONS="${MAX_ITERATIONS:-0}" # 0 means run forever -CODEX_MODEL="${CODEX_MODEL:-gpt-5.3-codex}" -CODEX_REASONING_EFFORT="${CODEX_REASONING_EFFORT:-medium}" - -read -r -d '' PROMPT <<'EOF' || true -1. load the refactor skill and read its SKILL.md fully from top to bottom (do not skim or partially read) -2. check previous commits to see what other agents have worked on -3. identify and work on a single large improvement based on the refactor skill (can be a file system reorg, file splitting, refactor, add a new test, harden a test, deflakify a test, fix a test, improve control flow, remove unused code) -4. commit with a description of what was done, your reasoning, and document accomplishments or challenges for the next agent - -guidelines: -- focus on the server and app code, it's the most hairy -EOF - -iteration=1 -while true; do - if [[ "${MAX_ITERATIONS}" -gt 0 && "${iteration}" -gt "${MAX_ITERATIONS}" ]]; then - echo "Reached MAX_ITERATIONS=${MAX_ITERATIONS}; exiting." - exit 0 - fi - - echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Starting codex exec iteration ${iteration}" - - cmd=( - codex exec - --dangerously-bypass-approvals-and-sandbox - --cd "${REPO_ROOT}" - ) - - if [[ -n "${CODEX_MODEL}" ]]; then - cmd+=(--model "${CODEX_MODEL}") - fi - - cmd+=(-c "model_reasoning_effort=\"${CODEX_REASONING_EFFORT}\"") - - if "${cmd[@]}" "${PROMPT}"; then - echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Iteration ${iteration} completed successfully" - else - status=$? - echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Iteration ${iteration} failed with exit code ${status}" - fi - - echo "Sleeping ${SLEEP_SECONDS}s before next iteration..." - sleep "${SLEEP_SECONDS}" - iteration=$((iteration + 1)) -done