puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-07-29 12:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(app): replace voice hooks with voice runtime and audio engine architecture

Browse Source
This commit is contained in:
Mohamed Boudra
2026-03-13 20:53:55 +07:00
parent 730f34753a
commit 80a2f8052c
51 changed files with 4308 additions and 2380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
packages/app/assets/audio/thinking-tone.wav Normal file
View File

Binary file not shown.

2
packages/app/src/app/_layout.tsx
View File

@@ -442,6 +442,7 @@ function ProvidersWrapper({ children }: { children: ReactNode }) {
return (
<VoiceProvider>
<OfferLinkListener upsertDaemonFromOfferUrl={upsertConnectionFromOfferUrl} />
<HostSessionManager />
{children}
</VoiceProvider>
);
@@ -582,7 +583,6 @@ export default function RootLayout() {
<QueryProvider>
<HostRuntimeBootstrapProvider>
<PushNotificationRouter />
<HostSessionManager />
<ProvidersWrapper>
<SidebarAnimationProvider>
<HorizontalScrollProvider>

30
packages/app/src/components/agent-input-area.tsx
View File

@@ -103,6 +103,23 @@ export function AgentInputArea({
const { client, isConnected, snapshot } = useHostRuntimeSession(serverId)
const toast = useToast()
const voice = useVoiceOptional()
console.log("[AgentInputArea] render", {
serverId,
agentId,
isConnected,
isVoiceMode: voice?.isVoiceMode ?? false,
isVoiceSwitching: voice?.isVoiceSwitching ?? false,
isMuted: voice?.isMuted ?? false,
})
console.log("[AgentInputArea] voice_snapshot", {
serverId,
agentId,
isVoiceMode: voice?.isVoiceMode ?? false,
isVoiceSwitching: voice?.isVoiceSwitching ?? false,
isMuted: voice?.isMuted ?? false,
activeServerId: voice?.activeServerId ?? null,
activeAgentId: voice?.activeAgentId ?? null,
})
const isDictationReady =
isConnected &&
(snapshot?.agentDirectoryStatus === 'ready' ||
@@ -151,6 +168,19 @@ export function AgentInputArea({
`message-input:${serverId}:${agentId}:${Math.random().toString(36).slice(2)}`
)
useEffect(() => {
console.log("[AgentInputArea] mount", {
serverId,
agentId,
})
return () => {
console.log("[AgentInputArea] unmount", {
serverId,
agentId,
})
}
}, [agentId, serverId])
const autocomplete = useAgentAutocomplete({
userInput,
cursorIndex,

66
packages/app/src/components/left-sidebar.tsx
View File

@@ -144,6 +144,70 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
// Track user-initiated refresh to avoid showing spinner on background revalidation
const [isManualRefresh, setIsManualRefresh] = useState(false)
useEffect(() => {
console.log('[LeftSidebar] isOpen_changed', {
isOpen,
isMobile,
mobileView,
desktopAgentListOpen,
})
}, [desktopAgentListOpen, isMobile, isOpen, mobileView])
useEffect(() => {
console.log('[LeftSidebar] pathname_changed', {
pathname,
activeServerIdFromPath,
activeServerId,
})
}, [activeServerId, activeServerIdFromPath, pathname])
useEffect(() => {
console.log('[LeftSidebar] hosts_changed', {
hostCount: daemons.length,
serverIds: daemons.map((daemon) => daemon.serverId),
runtimeConnectionStatusSignature,
})
}, [daemons, runtimeConnectionStatusSignature])
useEffect(() => {
console.log('[LeftSidebar] active_host_changed', {
activeServerId,
activeHostLabel,
activeHostStatus,
})
}, [activeHostLabel, activeHostStatus, activeServerId])
useEffect(() => {
console.log('[LeftSidebar] projects_changed', {
activeServerId,
projectCount: projects.length,
projectKeys: projects.map((project) => project.projectKey),
workspaceCounts: projects.map((project) => ({
projectKey: project.projectKey,
workspaceCount: project.workspaces.length,
})),
isInitialLoad,
isRevalidating,
})
}, [activeServerId, isInitialLoad, isRevalidating, projects])
useEffect(() => {
console.log('[LeftSidebar] collapsed_or_shortcuts_changed', {
collapsedProjectKeys: Array.from(collapsedProjectKeys),
shortcutCount: shortcutIndexByWorkspaceKey.size,
})
}, [collapsedProjectKeys, shortcutIndexByWorkspaceKey])
useEffect(() => {
console.log('[LeftSidebar] animation_context_changed', {
windowWidth,
translateXValue: translateX.value,
backdropOpacityValue: backdropOpacity.value,
isGesturing: isGesturing.value,
hasCloseGestureRef: Boolean(closeGestureRef.current),
})
}, [backdropOpacity, closeGestureRef, isGesturing, translateX, windowWidth])
const handleRefresh = useCallback(() => {
setIsManualRefresh(true)
refreshAll()
@@ -354,7 +418,6 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
<SidebarAgentListSkeleton />
) : (
<SidebarWorkspaceList
isOpen={isOpen}
serverId={activeServerId}
collapsedProjectKeys={collapsedProjectKeys}
onToggleProjectCollapsed={toggleProjectCollapsed}
@@ -481,7 +544,6 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
<SidebarAgentListSkeleton />
) : (
<SidebarWorkspaceList
isOpen={isOpen}
serverId={activeServerId}
collapsedProjectKeys={collapsedProjectKeys}
onToggleProjectCollapsed={toggleProjectCollapsed}

91
packages/app/src/components/message-input.tsx
View File

@@ -101,7 +101,6 @@ export interface MessageInputRef {
const MIN_INPUT_HEIGHT = 30
const MAX_INPUT_HEIGHT = 160
const IS_WEB = Platform.OS === 'web'
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__)
type WebTextInputKeyPressEvent = NativeSyntheticEvent<
TextInputKeyPressEventData & {
@@ -125,13 +124,10 @@ type TextAreaHandle = {
}
function logWebStickyBottom(
event: string,
details: Record<string, unknown>
_event: string,
_details: Record<string, unknown>
): void {
if (!IS_DEV || !IS_WEB) {
return
}
console.log('[WebStickyBottom]', event, details)
// Intentionally disabled: this path is too noisy during voice debugging.
}
function getDebugNow(): number | null {
@@ -220,6 +216,18 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
markScrollInvestigationRender(investigationComponentId)
const toast = useToast()
const voice = useVoiceOptional()
console.log("[MessageInput] render", {
voiceServerId: voiceServerId ?? null,
voiceAgentId: voiceAgentId ?? null,
hasVoice: Boolean(voice),
isVoiceMode: voice?.isVoiceMode ?? false,
isVoiceSwitching: voice?.isVoiceSwitching ?? false,
isMuted: voice?.isMuted ?? false,
disabled,
isSubmitLoading,
valueLength: value.length,
imageCount: images.length,
})
const [inputHeight, setInputHeight] = useState(MIN_INPUT_HEIGHT)
const rootRef = useRef<View | null>(null)
const inputWrapperRef = useRef<View | null>(null)
@@ -293,6 +301,19 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
)
)
useEffect(() => {
console.log("[MessageInput] mount", {
voiceServerId: voiceServerId ?? null,
voiceAgentId: voiceAgentId ?? null,
})
return () => {
console.log("[MessageInput] unmount", {
voiceServerId: voiceServerId ?? null,
voiceAgentId: voiceAgentId ?? null,
})
}
}, [voiceAgentId, voiceServerId])
useEffect(() => {
valueRef.current = value
}, [value])
@@ -407,6 +428,26 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
const showRealtimeOverlay = isRealtimeVoiceForCurrentAgent
const showOverlay = showDictationOverlay || showRealtimeOverlay
useEffect(() => {
console.log("[MessageInput] overlay_state", {
showDictationOverlay,
showRealtimeOverlay,
showOverlay,
isDictating,
isDictationProcessing,
dictationStatus,
isRealtimeVoiceForCurrentAgent,
})
}, [
dictationStatus,
isDictating,
isDictationProcessing,
isRealtimeVoiceForCurrentAgent,
showDictationOverlay,
showOverlay,
showRealtimeOverlay,
])
useEffect(() => {
if (isDictating || isDictationProcessing) {
return
@@ -443,6 +484,11 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
}))
const handleVoicePress = useCallback(async () => {
console.log("[MessageInput] handleVoicePress", {
isRealtimeVoiceForCurrentAgent,
isDictating,
hasVoice: Boolean(voice),
})
if (isRealtimeVoiceForCurrentAgent && voice) {
voice.toggleMute()
return
@@ -462,28 +508,39 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
])
const handleCancelRecording = useCallback(async () => {
console.log("[MessageInput] handleCancelRecording")
await cancelDictation()
}, [cancelDictation])
const handleAcceptRecording = useCallback(async () => {
console.log("[MessageInput] handleAcceptRecording")
sendAfterTranscriptRef.current = false
await confirmDictation()
}, [confirmDictation])
const handleAcceptAndSendRecording = useCallback(async () => {
console.log("[MessageInput] handleAcceptAndSendRecording")
sendAfterTranscriptRef.current = true
await confirmDictation()
}, [confirmDictation])
const handleRetryFailedRecording = useCallback(() => {
console.log("[MessageInput] handleRetryFailedRecording")
void retryFailedDictation()
}, [retryFailedDictation])
const handleDiscardFailedRecording = useCallback(() => {
console.log("[MessageInput] handleDiscardFailedRecording")
discardFailedDictation()
}, [discardFailedDictation])
const handleStopRealtimeVoice = useCallback(async () => {
console.log("[MessageInput] handleStopRealtimeVoice", {
hasVoice: Boolean(voice),
isRealtimeVoiceForCurrentAgent,
isAgentRunning,
voiceAgentId: voiceAgentId ?? null,
})
if (!voice || !isRealtimeVoiceForCurrentAgent) {
return
}
@@ -503,6 +560,18 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
}, [client, isAgentRunning, isRealtimeVoiceForCurrentAgent, voice, voiceAgentId])
const handleToggleRealtimeVoiceShortcut = useCallback(() => {
console.log("[MessageInput] handleToggleRealtimeVoiceShortcut", {
hasVoice: Boolean(voice),
voiceServerId: voiceServerId ?? null,
voiceAgentId: voiceAgentId ?? null,
isConnected,
disabled,
isVoiceSwitching: voice?.isVoiceSwitching ?? false,
isVoiceModeForAgent:
voice && voiceServerId && voiceAgentId
? voice.isVoiceModeForAgent(voiceServerId, voiceAgentId)
: false,
})
if (!voice || !voiceServerId || !voiceAgentId || !isConnected || disabled) {
return
}
@@ -524,6 +593,11 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
}, [disabled, handleStopRealtimeVoice, isConnected, toast, voice, voiceAgentId, voiceServerId])
const handleSendMessage = useCallback(() => {
console.log("[MessageInput] handleSendMessage", {
valueLength: value.length,
imageCount: images.length,
isAgentRunning,
})
const trimmed = value.trim()
if (!trimmed && images.length === 0) return
const payload = {
@@ -1070,10 +1144,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
/>
) : showRealtimeOverlay && voice ? (
<RealtimeVoiceOverlay
volume={voice.volume}
isMuted={voice.isMuted}
isDetecting={voice.isDetecting}
isSpeaking={voice.isSpeaking}
isSwitching={voice.isVoiceSwitching}
onToggleMute={voice.toggleMute}
onStop={() => {

8
packages/app/src/components/message.tsx
View File

@@ -1926,9 +1926,11 @@ export const ToolCall = memo(function ToolCall({
const hasDetails =
Boolean(error) ||
(effectiveDetail
? effectiveDetail.type !== "unknown" ||
effectiveDetail.input !== null ||
effectiveDetail.output !== null
? effectiveDetail.type === "plain_text"
? Boolean(effectiveDetail.text)
: effectiveDetail.type !== "unknown" ||
effectiveDetail.input !== null ||
effectiveDetail.output !== null
: false);
const handleToggle = useCallback(() => {

23
packages/app/src/components/realtime-voice-overlay.tsx
View File

@@ -1,14 +1,13 @@
import { ActivityIndicator, Pressable, View } from "react-native";
import { useEffect } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { Mic, MicOff, Square } from "lucide-react-native";
import { FOOTER_HEIGHT } from "@/constants/layout";
import { useVoiceTelemetry } from "@/contexts/voice-context";
import { VolumeMeter } from "./volume-meter";
interface RealtimeVoiceOverlayProps {
volume: number;
isMuted: boolean;
isDetecting: boolean;
isSpeaking: boolean;
isSwitching: boolean;
onToggleMute: () => void;
onStop: () => void;
@@ -18,15 +17,27 @@ const OVERLAY_BUTTON_SIZE = 44;
const OVERLAY_VERTICAL_PADDING = (FOOTER_HEIGHT - OVERLAY_BUTTON_SIZE) / 2;
export function RealtimeVoiceOverlay({
volume,
isMuted,
isDetecting,
isSpeaking,
isSwitching,
onToggleMute,
onStop,
}: RealtimeVoiceOverlayProps) {
const { theme } = useUnistyles();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
console.log("[RealtimeVoiceOverlay] render", {
isMuted,
isSwitching,
volume,
isDetecting,
isSpeaking,
});
useEffect(() => {
console.log("[RealtimeVoiceOverlay] mount");
return () => {
console.log("[RealtimeVoiceOverlay] unmount");
};
}, []);
return (
<View style={styles.container}>

25
packages/app/src/components/sidebar-workspace-list.tsx
View File

@@ -78,8 +78,12 @@ function toProjectIconDataUri(icon: { mimeType: string; data: string } | null):
return `data:${icon.mimeType};base64,${icon.data}`
}
const workspaceKeyExtractor = (workspace: SidebarWorkspaceEntry) =>
workspace.workspaceKey
const projectKeyExtractor = (project: SidebarProjectEntry) => project.projectKey
interface SidebarWorkspaceListProps {
isOpen?: boolean
projects: SidebarProjectEntry[]
serverId: string | null
collapsedProjectKeys: ReadonlySet<string>
@@ -1156,6 +1160,13 @@ function ProjectBlock({
[renderWorkspaceRow]
)
const handleWorkspaceDragEnd = useCallback(
(workspaces: SidebarWorkspaceEntry[]) => {
onWorkspaceReorder(project.projectKey, workspaces)
},
[onWorkspaceReorder, project.projectKey]
)
return (
<View style={styles.projectBlock}>
{flattenedWorkspace ? (
@@ -1201,9 +1212,9 @@ function ProjectBlock({
<DraggableList
testID={`sidebar-workspace-list-${project.projectKey}`}
data={project.workspaces}
keyExtractor={(workspace) => workspace.workspaceKey}
keyExtractor={workspaceKeyExtractor}
renderItem={renderWorkspace}
onDragEnd={(workspaces) => onWorkspaceReorder(project.projectKey, workspaces)}
onDragEnd={handleWorkspaceDragEnd}
scrollEnabled={false}
useDragHandle
nestable={useNestable}
@@ -1218,7 +1229,6 @@ function ProjectBlock({
}
export function SidebarWorkspaceList({
isOpen = true,
projects,
serverId,
collapsedProjectKeys,
@@ -1258,7 +1268,7 @@ export function SidebarWorkspaceList({
}, [pathname])
const projectIconRequests = useMemo(() => {
if (!isOpen || !serverId) {
if (!serverId) {
return []
}
const unique = new Map<string, { serverId: string; cwd: string }>()
@@ -1270,7 +1280,7 @@ export function SidebarWorkspaceList({
unique.set(`${serverId}:${cwd}`, { serverId, cwd })
}
return Array.from(unique.values())
}, [isOpen, projects, serverId])
}, [projects, serverId])
const projectIconQueries = useQueries({
queries: projectIconRequests.map((request) => ({
@@ -1285,7 +1295,6 @@ export function SidebarWorkspaceList({
},
select: toProjectIconDataUri,
enabled: Boolean(
isOpen &&
getHostRuntimeStore().getClient(request.serverId) &&
isHostRuntimeConnected(getHostRuntimeStore().getSnapshot(request.serverId)) &&
request.cwd
@@ -1444,7 +1453,7 @@ export function SidebarWorkspaceList({
<DraggableList
testID="sidebar-project-list"
data={projects}
keyExtractor={(project) => project.projectKey}
keyExtractor={projectKeyExtractor}
renderItem={renderProject}
onDragEnd={handleProjectDragEnd}
scrollEnabled={false}

8
packages/app/src/components/stream-strategy-web.tsx
View File

@@ -24,7 +24,6 @@ const USER_SCROLL_DELTA_EPSILON = 1
const AUTO_SCROLL_BOTTOM_THRESHOLD_PX = 64
const AUTO_SCROLL_RESUME_THRESHOLD_PX = 1
const WEB_STREAM_SCROLLBAR_STYLE_ID = 'web-stream-viewport-scrollbar-style'
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__)
const WEB_STREAM_SCROLLBAR_STYLE = `
#agent-chat-scroll-web-dom-scroll,
#agent-chat-scroll-web-dom-virtualized {
@@ -40,11 +39,8 @@ const WEB_STREAM_SCROLLBAR_STYLE = `
}
`
function logWebStickyBottom(event: string, details: Record<string, unknown>): void {
if (!IS_DEV) {
return
}
console.log('[WebStickyBottom]', event, details)
function logWebStickyBottom(_event: string, _details: Record<string, unknown>): void {
// Intentionally disabled: this path is too noisy during voice debugging.
}
function getDebugNow(): number | null {

19
packages/app/src/components/ui/combobox.tsx
View File

@@ -64,6 +64,7 @@ export interface ComboboxProps {
title?: string
open?: boolean
onOpenChange?: (open: boolean) => void
enableDismissOnClose?: boolean
desktopPlacement?: 'top-start' | 'bottom-start'
/**
* Prevents an initial frame at 0,0 by hiding desktop content until floating
@@ -224,6 +225,7 @@ export function Combobox({
title = 'Select',
open,
onOpenChange,
enableDismissOnClose,
desktopPlacement = 'top-start',
desktopPreventInitialFlash = true,
anchorRef,
@@ -235,6 +237,7 @@ export function Combobox({
!isMobile && Platform.OS === 'web' && effectiveOptionsPosition === 'above-search'
const { height: windowHeight } = useWindowDimensions()
const bottomSheetRef = useRef<BottomSheetModal>(null)
const hasPresentedBottomSheetRef = useRef(false)
const snapPoints = useMemo(() => ['60%', '90%'], [])
const [availableSize, setAvailableSize] = useState<{ width?: number; height?: number } | null>(
null
@@ -379,11 +382,20 @@ export function Combobox({
useEffect(() => {
if (!isMobile) return
if (isOpen) {
bottomSheetRef.current?.present()
if (enableDismissOnClose === false && hasPresentedBottomSheetRef.current) {
bottomSheetRef.current?.snapToIndex(0)
} else {
hasPresentedBottomSheetRef.current = true
bottomSheetRef.current?.present()
}
} else {
bottomSheetRef.current?.dismiss()
if (enableDismissOnClose === false) {
bottomSheetRef.current?.close()
} else {
bottomSheetRef.current?.dismiss()
}
}
}, [isOpen, isMobile])
}, [enableDismissOnClose, isOpen, isMobile])
const handleSheetChange = useCallback(
(index: number) => {
@@ -622,6 +634,7 @@ export function Combobox({
onChange={handleSheetChange}
backdropComponent={renderBackdrop}
enablePanDownToClose
enableDismissOnClose={enableDismissOnClose}
backgroundComponent={ComboboxSheetBackground}
handleIndicatorStyle={styles.bottomSheetHandle}
keyboardBehavior="extend"

9
packages/app/src/components/use-bottom-anchor-controller.test.ts
View File

@@ -132,8 +132,6 @@ function createDriverHarness(input?: {
context.measurementState.offsetY = 720;
});
const modeChanges: BottomAnchorMode[] = [];
const warnings: Array<{ agentId: string; reason: string }> = [];
const logs: Array<{ event: string; details: Record<string, unknown> }> = [];
const driver = __private__.createBottomAnchorControllerDriver({
getAgentId: () => context.agentId,
getIsAuthoritativeHistoryReady: () => context.authoritativeReady,
@@ -145,10 +143,6 @@ function createDriverHarness(input?: {
onModeChange: (mode) => {
modeChanges.push(mode);
},
log: (event, details) => {
logs.push({ event, details });
},
warn: (details) => warnings.push(details),
scheduleFrame: (params) => scheduler.schedule(params),
cancelFrame: (handle) => scheduler.cancel(handle),
});
@@ -159,8 +153,6 @@ function createDriverHarness(input?: {
scheduler,
scrollToBottom,
modeChanges,
logs,
warnings,
};
}
@@ -367,7 +359,6 @@ describe("bottom anchor controller driver", () => {
harness.scheduler.flushAll();
expect(harness.scrollToBottom).toHaveBeenCalledTimes(1);
expect(harness.warnings).toEqual([]);
expect(harness.driver.getSnapshot().pendingRequest).toBeNull();
});

132
packages/app/src/components/use-bottom-anchor-controller.ts
View File

@@ -110,8 +110,6 @@ type CreateBottomAnchorControllerDriverInput = {
isNearBottom: () => boolean;
scrollToBottom: (animated: boolean) => void;
onModeChange: (mode: BottomAnchorMode) => void;
log: (event: BottomAnchorEvent, details: Record<string, unknown>) => void;
warn: (details: { agentId: string; reason: BottomAnchorRequestReason }) => void;
scheduleFrame: (params: {
kind: "attempt" | "verification";
callback: () => void;
@@ -123,17 +121,6 @@ type CreateBottomAnchorControllerDriverInput = {
const MAX_VERIFICATION_RETRIES = 3;
const WEB_PARTIAL_VIRTUALIZED_CONFIRMATION_DELAY_FRAMES = 1;
const USER_SCROLL_AWAY_DELTA_PX = 24;
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
function logBottomAnchorEvent(
event: BottomAnchorEvent,
details: Record<string, unknown>
): void {
if (!IS_DEV) {
return;
}
console.debug("[BottomAnchor]", event, details);
}
function scheduleAnimationFrameWithDelay(input: {
callback: () => void;
@@ -323,10 +310,6 @@ function createBottomAnchorControllerDriver(
return;
}
blockedReason = nextBlockedReason;
input.log(
"blocked_reason_changed",
getLogContext({ nextBlockedReason })
);
};
const setModeInternal = (nextMode: BottomAnchorMode) => {
@@ -367,13 +350,6 @@ function createBottomAnchorControllerDriver(
setBlockedReason(null);
return;
}
input.log(
"request_cancelled",
getLogContext({
cancelledRequestReason: currentRequest.reason,
cancelReason: reason,
})
);
pendingRequest = null;
cancelPendingAttempt();
setBlockedReason(null);
@@ -398,19 +374,6 @@ function createBottomAnchorControllerDriver(
if (verificationHandle) {
input.cancelFrame(verificationHandle);
}
input.log(
"verification_scheduled",
getLogContext({
retries: attemptContext.retries,
startedContentHeight: attemptContext.startedContentHeight ?? null,
startedOffsetY: attemptContext.startedOffsetY ?? null,
startedViewportHeight: attemptContext.startedViewportHeight ?? null,
scheduledMeasurementState:
getDetailedMeasurementState(scheduledMeasurementState),
verificationDelayFrames:
delayFramesOverride ?? input.getTransportBehavior().verificationDelayFrames,
})
);
verificationHandle = input.scheduleFrame({
kind: "verification",
delayFrames:
@@ -427,15 +390,6 @@ function createBottomAnchorControllerDriver(
});
if (verificationBlockedReason) {
input.log(
"attempt_verified",
getLogContext({
verificationPhase: "blocked",
verificationBlockedReason,
retries: attemptContext.retries,
measurementState: getDetailedMeasurementState(measurementState),
})
);
pendingVerification = attemptContext;
setBlockedReason(verificationBlockedReason);
return;
@@ -451,26 +405,6 @@ function createBottomAnchorControllerDriver(
input.getTransportBehavior().verificationRetryMode,
});
input.log(
"attempt_verified",
getLogContext({
verifiedNearBottom,
retries: attemptContext.retries,
retryDisposition,
contentHeightDeltaSinceAttempt:
measurementState.contentHeight -
(attemptContext.startedContentHeight ?? measurementState.contentHeight),
offsetDeltaSinceAttempt:
measurementState.offsetY -
(attemptContext.startedOffsetY ?? measurementState.offsetY),
viewportHeightDeltaSinceAttempt:
measurementState.viewportHeight -
(attemptContext.startedViewportHeight ??
measurementState.viewportHeight),
measurementState: getDetailedMeasurementState(measurementState),
})
);
if (verifiedNearBottom) {
if (
isRequestAttempt &&
@@ -494,7 +428,6 @@ function createBottomAnchorControllerDriver(
pendingVerification = null;
markStickyMeasurementVerified();
if (isRequestAttempt) {
input.log("request_fulfilled", getLogContext());
pendingRequest = null;
}
setBlockedReason(null);
@@ -520,21 +453,7 @@ function createBottomAnchorControllerDriver(
return;
}
input.log(
"attempt_failed",
getLogContext({
retries: attemptContext.retries,
retryDisposition,
measurementState: getDetailedMeasurementState(measurementState),
})
);
pendingVerification = null;
if (isRequestAttempt && currentRequest) {
input.warn({
agentId: input.getAgentId(),
reason: currentRequest.reason,
});
}
setBlockedReason(
isRequestAttempt ? "waiting_for_post_layout_verification" : null
);
@@ -552,14 +471,6 @@ function createBottomAnchorControllerDriver(
startedViewportHeight: measurementState.viewportHeight,
};
pendingVerification = attemptContext;
input.log(
"attempt_started",
getLogContext({
animated,
retries: attemptContext.retries,
measurementState: getDetailedMeasurementState(measurementState),
})
);
input.scrollToBottom(animated);
scheduleVerification(attemptContext);
setBlockedReason(deriveDriverBlockedReason(input.getMeasurementState()));
@@ -576,18 +487,6 @@ function createBottomAnchorControllerDriver(
| "manual_reevaluate"
| "retry_scroll"
) => {
input.log(
"evaluate_called",
getLogContext({
evaluateReason: reason,
animated,
hasAttemptHandle: attemptHandle !== null,
hasVerificationHandle: verificationHandle !== null,
pendingVerificationRequestId: pendingVerification?.requestId ?? null,
pendingVerificationRetries: pendingVerification?.retries ?? null,
measurementState: getDetailedMeasurementState(input.getMeasurementState()),
})
);
if (attemptHandle) {
return;
}
@@ -610,17 +509,6 @@ function createBottomAnchorControllerDriver(
!shouldAttemptForPendingRequest &&
!shouldAttemptForStickyVerification
) {
input.log(
"attempt_started",
getLogContext({
attemptPhase: "skipped",
evaluateReason: reason,
nextBlockedReason,
shouldAttemptForPendingRequest,
shouldAttemptForStickyVerification,
measurementState: getDetailedMeasurementState(measurementState),
})
);
return;
}
@@ -630,17 +518,6 @@ function createBottomAnchorControllerDriver(
};
const createRequest = (request: BottomAnchorRouteRequest | BottomAnchorLocalRequest) => {
const existing = pendingRequest;
if (existing) {
input.log(
"request_cancelled",
getLogContext({
cancelledRequestReason: existing.reason,
cancelReason: "replaced_by_new_request",
})
);
}
cancelPendingAttempt();
const nextRequest: BottomAnchorRequest = {
id: requestSequence + 1,
@@ -659,10 +536,6 @@ function createBottomAnchorControllerDriver(
? "sticky-bottom"
: __private__.deriveModeForLocalRequest({ reason: request.reason })
);
input.log(
"request_created",
getLogContext({ requestReason: request.reason })
);
evaluate(request.reason === "jump-to-bottom", "request_created");
};
@@ -707,7 +580,6 @@ function createBottomAnchorControllerDriver(
}
cancelPendingRequest("user_scrolled_away");
setModeInternal("detached");
input.log("detached_by_user", getLogContext());
},
handleViewportMetricsChange(params) {
if (
@@ -905,10 +777,6 @@ export function useBottomAnchorController(input: {
isNearBottom: () => isNearBottomRef.current(),
scrollToBottom: (animated) => scrollToBottomRef.current(animated),
onModeChange: (nextMode) => setMode(nextMode),
log: (event, details) => logBottomAnchorEvent(event, details),
warn: (details) => {
console.warn("[BottomAnchor] request could not be fulfilled", details);
},
scheduleFrame: ({ callback, delayFrames }) =>
scheduleAnimationFrameWithDelay({ callback, delayFrames }),
cancelFrame: (handle) => cancelScheduledAnimationFrame(handle),

22
packages/app/src/components/voice-compact-indicator.tsx
View File

@@ -1,21 +1,35 @@
import { ActivityIndicator, Alert, Pressable, View } from "react-native";
import { useEffect } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { Mic, MicOff, Square } from "lucide-react-native";
import { VolumeMeter } from "@/components/volume-meter";
import { useVoice } from "@/contexts/voice-context";
import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context";
export function VoiceCompactIndicator() {
const { theme } = useUnistyles();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const {
isVoiceMode,
isVoiceSwitching,
volume,
isMuted,
isDetecting,
isSpeaking,
toggleMute,
stopVoice,
} = useVoice();
console.log("[VoiceCompactIndicator] render", {
isVoiceMode,
isVoiceSwitching,
isMuted,
volume,
isDetecting,
isSpeaking,
});
useEffect(() => {
console.log("[VoiceCompactIndicator] mount");
return () => {
console.log("[VoiceCompactIndicator] unmount");
};
}, []);
if (!isVoiceMode) {
return null;

21
packages/app/src/components/voice-panel.tsx
View File

@@ -1,22 +1,35 @@
import { View, Pressable } from "react-native";
import { useEffect } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { MicOff, Square } from "lucide-react-native";
import { VolumeMeter } from "./volume-meter";
import { useVoice } from "@/contexts/voice-context";
import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context";
import { useHosts } from "@/runtime/host-runtime";
export function VoicePanel() {
const { theme } = useUnistyles();
const daemons = useHosts();
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
const {
volume,
isMuted,
isDetecting,
isSpeaking,
stopVoice,
toggleMute,
activeServerId,
} = useVoice();
console.log("[VoicePanel] render", {
activeServerId,
isMuted,
volume,
isDetecting,
isSpeaking,
});
useEffect(() => {
console.log("[VoicePanel] mount");
return () => {
console.log("[VoicePanel] unmount");
};
}, []);
const hostLabel = activeServerId
? daemons.find((daemon) => daemon.serverId === activeServerId)?.label ?? null

79
packages/app/src/components/volume-meter.tsx
View File

@@ -3,10 +3,9 @@ import { View } from "react-native";
import ReanimatedAnimated, {
useSharedValue,
useAnimatedStyle,
withSpring,
withTiming,
withRepeat,
withSequence,
withTiming,
Easing,
} from "react-native-reanimated";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
@@ -41,9 +40,7 @@ export function VolumeMeter({
orientation === "horizontal" ? (isCompact ? 8 : 12) : isCompact ? 14 : 20;
// Create shared values for 3 dots unconditionally
const line1Height = useSharedValue(MIN_HEIGHT);
const line2Height = useSharedValue(MIN_HEIGHT);
const line3Height = useSharedValue(MIN_HEIGHT);
const animatedVolume = useSharedValue(0);
const line1Pulse = useSharedValue(1);
const line2Pulse = useSharedValue(1);
const line3Pulse = useSharedValue(1);
@@ -90,54 +87,19 @@ export function VolumeMeter({
);
}, [isMuted]);
// Update heights based on volume with different responsiveness for all dots
// Drive a single animated volume value and derive the individual bar heights
// on the UI thread instead of scheduling three independent springs per sample.
useEffect(() => {
if (isMuted) {
// When muted, keep all lines at minimum height without animation
line1Height.value = MIN_HEIGHT;
line2Height.value = MIN_HEIGHT;
line3Height.value = MIN_HEIGHT;
animatedVolume.value = 0;
return;
}
if (volume > 0.001) {
// Active volume - animate heights based on volume
const target1 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.2);
const target2 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.05);
const target3 = MIN_HEIGHT + (MAX_HEIGHT * volume * 0.9);
line1Height.value = withSpring(target1, {
damping: 10,
stiffness: 200,
});
line2Height.value = withSpring(target2, {
damping: 12.5,
stiffness: 175,
});
line3Height.value = withSpring(target3, {
damping: 15,
stiffness: 150,
});
} else {
// No volume - return to minimum
line1Height.value = withSpring(MIN_HEIGHT, {
damping: 20,
stiffness: 150,
});
line2Height.value = withSpring(MIN_HEIGHT, {
damping: 20,
stiffness: 150,
});
line3Height.value = withSpring(MIN_HEIGHT, {
damping: 20,
stiffness: 150,
});
}
}, [volume, isMuted]);
animatedVolume.value = withTiming(volume, {
duration: volume > animatedVolume.value ? 70 : 140,
easing: Easing.out(Easing.cubic),
});
}, [animatedVolume, isMuted, volume]);
const lineColor = color ?? theme.colors.foreground;
const containerHeight =
@@ -147,9 +109,12 @@ export function VolumeMeter({
const line1Style = useAnimatedStyle(() => {
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
const currentVolume = isMuted ? 0 : animatedVolume.value;
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 1.2);
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
return {
height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value),
height:
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line1Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
@@ -157,9 +122,12 @@ export function VolumeMeter({
const line2Style = useAnimatedStyle(() => {
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
const currentVolume = isMuted ? 0 : animatedVolume.value;
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 1.05);
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
return {
height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value),
height:
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line2Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});
@@ -167,9 +135,12 @@ export function VolumeMeter({
const line3Style = useAnimatedStyle(() => {
const isActive = isSpeaking;
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
const currentVolume = isMuted ? 0 : animatedVolume.value;
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 0.9);
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
return {
height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value),
height:
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line3Pulse.value),
opacity: baseOpacity + volumeBoost,
};
});

359
packages/app/src/contexts/session-context.tsx
View File

@@ -1,7 +1,7 @@
import { useRef, ReactNode, useCallback, useEffect, useMemo } from "react";
import { Buffer } from "buffer";
import { AppState, Platform } from "react-native";
import { useQueryClient } from "@tanstack/react-query";
import { useAudioPlayer } from "@/hooks/use-audio-player";
import { useClientActivity } from "@/hooks/use-client-activity";
import { usePushTokenRegistration } from "@/hooks/use-push-token-registration";
import { clearArchiveAgentPending } from "@/hooks/use-archive-agent";
@@ -29,8 +29,13 @@ import type { DaemonClient } from "@server/client/daemon-client";
import { File } from "expo-file-system";
import {
getHostRuntimeStore,
useHostRuntimeSession,
useHostRuntimeIsConnected,
} from "@/runtime/host-runtime";
import {
useVoiceAudioEngineOptional,
useVoiceRuntimeOptional,
} from "@/contexts/voice-context";
import type { AudioPlaybackSource } from "@/voice/audio-engine-types";
import {
useSessionStore,
type Agent,
@@ -73,6 +78,66 @@ export type {
const HISTORY_STALE_AFTER_MS = 60_000;
const AUTHORITATIVE_REVALIDATION_DEBOUNCE_MS = 300;
type AudioOutputPayload = Extract<
SessionOutboundMessage,
{ type: "audio_output" }
>["payload"];
interface BufferedAudioChunk {
chunkIndex: number;
audio: string;
format: string;
id: string;
}
function decodeBase64Chunk(base64: string): Uint8Array {
return Buffer.from(base64, "base64");
}
function buildAudioPlaybackSource(
chunks: BufferedAudioChunk[]
): AudioPlaybackSource {
const decodedChunks = chunks.map((chunk) => decodeBase64Chunk(chunk.audio));
const totalSize = decodedChunks.reduce((sum, chunk) => sum + chunk.length, 0);
const output = new Uint8Array(totalSize);
let offset = 0;
for (const chunk of decodedChunks) {
output.set(chunk, offset);
offset += chunk.length;
}
const format = chunks[0]?.format ?? "pcm";
const mimeType =
format === "pcm"
? "audio/pcm;rate=24000;bits=16"
: format === "mp3"
? "audio/mpeg"
: `audio/${format}`;
const bytes = output.slice();
return {
size: bytes.byteLength,
type: mimeType,
async arrayBuffer() {
return bytes.buffer.slice(
bytes.byteOffset,
bytes.byteOffset + bytes.byteLength
);
},
};
}
function logVoiceSession(
event: string,
details?: Record<string, unknown>
): void {
if (details) {
console.log(`[VoiceSession] ${event}`, details);
return;
}
console.log(`[VoiceSession] ${event}`);
}
const findLatestAssistantMessageText = (items: StreamItem[]): string | null => {
for (let i = items.length - 1; i >= 0; i -= 1) {
const item = items[i];
@@ -206,8 +271,15 @@ function SessionProviderInternal({
serverId,
client,
}: SessionProviderClientProps) {
const voiceRuntime = useVoiceRuntimeOptional();
const voiceAudioEngine = useVoiceAudioEngineOptional();
console.log("[SessionProvider] render", {
serverId,
hasVoiceRuntime: Boolean(voiceRuntime),
hasVoiceAudioEngine: Boolean(voiceAudioEngine),
});
const queryClient = useQueryClient();
const { isConnected } = useHostRuntimeSession(serverId);
const isConnected = useHostRuntimeIsConnected(serverId);
// Zustand store actions
const initializeSession = useSessionStore((state) => state.initializeSession);
@@ -276,16 +348,6 @@ function SessionProviderInternal({
(state) => state.sessions[serverId]?.agents
);
// State for voice detection flags (will be set by RealtimeContext)
const isDetectingRef = useRef(false);
const isSpeakingRef = useRef(false);
const audioPlayer = useAudioPlayer({
isDetecting: () => isDetectingRef.current,
isSpeaking: () => isSpeakingRef.current,
});
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
const previousAgentStatusRef = useRef<Map<string, AgentLifecycleStatus>>(
new Map()
);
@@ -308,6 +370,10 @@ function SessionProviderInternal({
const revalidationInFlightRef = useRef<Promise<void> | null>(null);
const revalidationQueuedRef = useRef(false);
const wasConnectedRef = useRef(isConnected);
const audioOutputBuffersRef = useRef<Map<string, BufferedAudioChunk[]>>(
new Map()
);
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
useEffect(() => {
const subscription = AppState.addEventListener("change", (nextState) => {
@@ -617,24 +683,83 @@ function SessionProviderInternal({
[serverId]
);
// Buffer for streaming audio chunks
interface AudioChunk {
chunkIndex: number;
audio: string; // base64
format: string;
id: string;
}
const audioChunkBuffersRef = useRef<Map<string, AudioChunk[]>>(new Map());
// Initialize session in store
useEffect(() => {
initializeSession(serverId, client, audioPlayer);
}, [serverId, client, audioPlayer, initializeSession]);
console.log("[SessionProvider] mount", { serverId });
return () => {
console.log("[SessionProvider] unmount", { serverId });
};
}, [serverId]);
useEffect(() => {
initializeSession(serverId, client);
}, [serverId, client, initializeSession]);
useEffect(() => {
updateSessionClient(serverId, client);
}, [serverId, client, updateSessionClient]);
useEffect(() => {
if (!voiceRuntime) {
return;
}
console.log("[SessionProvider] register_voice_session", { serverId });
return voiceRuntime.registerSession({
serverId,
setVoiceMode: async (enabled, agentId) => {
console.log("[SessionProvider] setVoiceMode", {
serverId,
enabled,
agentId: agentId ?? null,
});
if (!client) {
throw new Error("Daemon unavailable");
}
await client.setVoiceMode(enabled, agentId);
},
sendVoiceAudioChunk: async (audioData, mimeType, isLast) => {
console.log("[SessionProvider] sendVoiceAudioChunk", {
serverId,
audioDataLength: audioData.length,
mimeType,
isLast,
});
if (!client) {
throw new Error("Daemon unavailable");
}
await client.sendVoiceAudioChunk(audioData, mimeType, isLast);
},
abortRequest: async () => {
console.log("[SessionProvider] abortRequest", { serverId });
if (!client) {
throw new Error("Daemon unavailable");
}
await client.abortRequest();
},
setAssistantAudioPlaying: (isPlaying) => {
console.log("[SessionProvider] setAssistantAudioPlaying", {
serverId,
isPlaying,
});
setIsPlayingAudio(serverId, isPlaying);
},
});
}, [
client,
serverId,
setIsPlayingAudio,
voiceRuntime,
]);
useEffect(() => {
console.log("[SessionProvider] updateSessionConnection", {
serverId,
isConnected,
});
voiceRuntime?.updateSessionConnection(serverId, isConnected);
}, [isConnected, serverId, voiceRuntime]);
// If the client drops mid-initialization, clear pending flags
useEffect(() => {
if (!isConnected) {
@@ -962,6 +1087,14 @@ function SessionProviderInternal({
const { agentId, event, timestamp, seq, epoch } = message.payload;
const parsedTimestamp = new Date(timestamp);
const streamEvent = event as AgentStreamEventPayload;
if (
event.type === "turn_started" ||
event.type === "turn_completed" ||
event.type === "turn_failed" ||
event.type === "turn_canceled"
) {
voiceRuntime?.onTurnEvent(serverId, agentId, event.type);
}
// Attention notification stays in React (not extractable to pure reducer)
if (event.type === "attention_required") {
@@ -1161,92 +1294,102 @@ function SessionProviderInternal({
const unsubAudioOutput = client.on("audio_output", async (message) => {
if (message.type !== "audio_output") return;
const data = message.payload;
const playbackGroupId = data.groupId ?? data.id;
const isFinalChunk = data.isLastChunk ?? true;
const chunkIndex = data.chunkIndex ?? 0;
if (!voiceAudioEngine) {
return;
}
const payload: AudioOutputPayload = message.payload;
console.log("[SessionProvider] audio_output", {
serverId,
id: payload.id,
groupId: payload.groupId ?? null,
chunkIndex: payload.chunkIndex ?? 0,
isLastChunk: payload.isLastChunk ?? true,
format: payload.format,
isVoiceMode: payload.isVoiceMode ?? false,
audioLength: payload.audio.length,
});
const playbackGroupId = payload.groupId ?? payload.id;
const chunkIndex = payload.chunkIndex ?? 0;
const isFinalChunk = payload.isLastChunk ?? true;
if (!audioOutputBuffersRef.current.has(playbackGroupId)) {
audioOutputBuffersRef.current.set(playbackGroupId, []);
}
const bufferedChunks = audioOutputBuffersRef.current.get(playbackGroupId)!;
bufferedChunks.push({
chunkIndex,
audio: payload.audio,
format: payload.format,
id: payload.id,
});
activeAudioGroupsRef.current.add(playbackGroupId);
setIsPlayingAudio(serverId, true);
if (!audioChunkBuffersRef.current.has(playbackGroupId)) {
audioChunkBuffersRef.current.set(playbackGroupId, []);
}
const buffer = audioChunkBuffersRef.current.get(playbackGroupId)!;
buffer.push({
chunkIndex,
audio: data.audio,
format: data.format,
id: data.id,
});
if (!isFinalChunk) {
return;
}
buffer.sort((a, b) => a.chunkIndex - b.chunkIndex);
let playbackFailed = false;
const chunkIds = buffer.map((chunk) => chunk.id);
const confirmAudioPlayed = (ids: string[]) => {
if (!client) {
console.warn("[Session] audio_played skipped: daemon unavailable");
return;
}
ids.forEach((chunkId) => {
void client.audioPlayed(chunkId).catch((error) => {
console.warn("[Session] Failed to confirm audio playback:", error);
});
bufferedChunks.sort((left, right) => left.chunkIndex - right.chunkIndex);
const chunkIds = bufferedChunks.map((chunk) => chunk.id);
const shouldPlay =
!payload.isVoiceMode ||
(voiceRuntime?.shouldPlayVoiceAudio(serverId) ?? false);
const audioBlob = buildAudioPlaybackSource(bufferedChunks);
if (payload.isVoiceMode) {
logVoiceSession("audio_output_ready", {
serverId,
playbackGroupId,
chunkCount: bufferedChunks.length,
format: payload.format,
blobBytes: audioBlob.size,
shouldPlay,
});
}
const confirmAudioPlayed = async () => {
await Promise.all(
chunkIds.map((chunkId) =>
client.audioPlayed(chunkId).catch((error) => {
console.warn("[Session] Failed to confirm audio playback:", error);
})
)
);
};
let startedVoicePlayback = false;
try {
const mimeType =
data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`;
const decodedChunks: Uint8Array[] = [];
let totalSize = 0;
for (const chunk of buffer) {
const binaryString = atob(chunk.audio);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
if (shouldPlay) {
if (payload.isVoiceMode) {
startedVoicePlayback = true;
voiceRuntime?.onAssistantAudioStarted(serverId);
}
decodedChunks.push(bytes);
totalSize += bytes.length;
await voiceAudioEngine.play(audioBlob);
if (payload.isVoiceMode) {
logVoiceSession("audio_output_played", {
serverId,
playbackGroupId,
chunkCount: bufferedChunks.length,
});
}
} else if (payload.isVoiceMode) {
logVoiceSession("audio_output_suppressed", {
serverId,
playbackGroupId,
});
}
const concatenatedBytes = new Uint8Array(totalSize);
let offset = 0;
for (const chunk of decodedChunks) {
concatenatedBytes.set(chunk, offset);
offset += chunk.length;
}
const audioBlob = {
type: mimeType,
size: totalSize,
arrayBuffer: async () => {
return concatenatedBytes.buffer;
},
} as Blob;
await audioPlayer.play(audioBlob);
confirmAudioPlayed(chunkIds);
} catch (error: any) {
playbackFailed = true;
await confirmAudioPlayed();
} catch (error) {
console.error("[Session] Audio playback error:", error);
confirmAudioPlayed(chunkIds);
await confirmAudioPlayed();
} finally {
audioChunkBuffersRef.current.delete(playbackGroupId);
audioOutputBuffersRef.current.delete(playbackGroupId);
activeAudioGroupsRef.current.delete(playbackGroupId);
setIsPlayingAudio(serverId, activeAudioGroupsRef.current.size > 0);
if (activeAudioGroupsRef.current.size === 0) {
setIsPlayingAudio(serverId, false);
if (startedVoicePlayback) {
voiceRuntime?.onAssistantAudioFinished(serverId);
}
}
});
@@ -1254,6 +1397,12 @@ function SessionProviderInternal({
const unsubActivity = client.on("activity_log", (message) => {
if (message.type !== "activity_log") return;
const data = message.payload;
console.log("[SessionProvider] activity_log", {
serverId,
type: data.type,
contentPreview:
typeof data.content === "string" ? data.content.slice(0, 80) : null,
});
if (data.type === "system" && data.content.includes("Transcribing")) {
return;
@@ -1374,13 +1523,18 @@ function SessionProviderInternal({
if (message.type !== "transcription_result") return;
const transcriptText = message.payload.text.trim();
console.log("[SessionProvider] transcription_result", {
serverId,
textLength: transcriptText.length,
textPreview: transcriptText.slice(0, 80),
});
if (!transcriptText) {
} else {
audioPlayer.stop();
setIsPlayingAudio(serverId, false);
setCurrentAssistantMessage(serverId, "");
voiceRuntime?.onTranscriptionResult(serverId, transcriptText);
return;
}
setCurrentAssistantMessage(serverId, "");
});
const unsubAgentDeleted = client.on("agent_deleted", (message) => {
@@ -1498,7 +1652,6 @@ function SessionProviderInternal({
};
}, [
client,
audioPlayer,
queryClient,
serverId,
setIsPlayingAudio,
@@ -1521,6 +1674,8 @@ function SessionProviderInternal({
requestCanonicalCatchUp,
applyAgentUpdatePayload,
applyTimelineResponse,
voiceRuntime,
voiceAudioEngine,
]);
const sendAgentMessage = useCallback(
@@ -1742,20 +1897,12 @@ function SessionProviderInternal({
[client]
);
const setVoiceDetectionFlags = useCallback(
(isDetecting: boolean, isSpeaking: boolean) => {
isDetectingRef.current = isDetecting;
isSpeakingRef.current = isSpeaking;
},
[]
);
// Cleanup on unmount
useEffect(() => {
return () => {
clearSession(serverId);
};
}, [serverId, clearSession]);
}, [clearSession, serverId]);
return children;
}

582
packages/app/src/contexts/voice-context.tsx
View File

@@ -1,41 +1,119 @@
import { createContext, useContext, useState, ReactNode, useCallback, useEffect, useRef } from "react";
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
import type { SessionState } from "@/stores/session-store";
import { useSessionStore } from "@/stores/session-store";
import { useHostRuntimeSession } from "@/runtime/host-runtime";
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import {
createContext,
useContext,
useEffect,
useRef,
useSyncExternalStore,
type ReactNode,
} from "react";
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
import { useSessionStore } from "@/stores/session-store";
import { createAudioEngine } from "@/voice/audio-engine";
import type { AudioEngine } from "@/voice/audio-engine-types";
import {
createVoiceRuntime,
type VoiceRuntime,
type VoiceRuntimeSnapshot,
type VoiceRuntimeTelemetrySnapshot,
} from "@/voice/voice-runtime";
const KEEP_AWAKE_TAG = "paseo:voice";
interface VoiceContextValue {
isVoiceMode: boolean;
isVoiceSwitching: boolean;
volume: number;
isMuted: boolean;
isDetecting: boolean;
isSpeaking: boolean;
segmentDuration: number;
interface VoiceContextValue extends VoiceRuntimeSnapshot {
startVoice: (serverId: string, agentId: string) => Promise<void>;
stopVoice: () => Promise<void>;
isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
toggleMute: () => void;
activeServerId: string | null;
activeAgentId: string | null;
}
const VoiceContext = createContext<VoiceContextValue | null>(null);
const EMPTY_SNAPSHOT: VoiceRuntimeSnapshot = {
phase: "disabled",
isVoiceMode: false,
isVoiceSwitching: false,
isMuted: false,
activeServerId: null,
activeAgentId: null,
};
const EMPTY_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
volume: 0,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
};
const VoiceRuntimeContext = createContext<VoiceRuntime | null>(null);
const VoiceAudioEngineContext = createContext<AudioEngine | null>(null);
const noopSubscribe = () => () => {};
const getEmptySnapshot = () => EMPTY_SNAPSHOT;
const getEmptyTelemetry = () => EMPTY_TELEMETRY;
let nextVoiceProviderInstanceId = 1;
function getProviderTraceStack(): string | undefined {
const stack = new Error().stack;
if (!stack) {
return undefined;
}
return stack
.split("\n")
.slice(2, 7)
.map((line) => line.trim())
.join(" | ");
}
export function useVoice() {
const context = useContext(VoiceContext);
if (!context) {
const value = useVoiceOptional();
if (!value) {
throw new Error("useVoice must be used within VoiceProvider");
}
return context;
return value;
}
export function useVoiceOptional(): VoiceContextValue | null {
return useContext(VoiceContext);
const runtime = useContext(VoiceRuntimeContext);
const snapshot = useSyncExternalStore(
runtime ? runtime.subscribe : noopSubscribe,
runtime ? runtime.getSnapshot : getEmptySnapshot,
runtime ? runtime.getSnapshot : getEmptySnapshot
);
if (!runtime) {
return null;
}
return {
...snapshot,
startVoice: runtime.startVoice,
stopVoice: runtime.stopVoice,
isVoiceModeForAgent: runtime.isVoiceModeForAgent,
toggleMute: runtime.toggleMute,
};
}
export function useVoiceTelemetry() {
const telemetry = useVoiceTelemetryOptional();
if (!telemetry) {
throw new Error("useVoiceTelemetry must be used within VoiceProvider");
}
return telemetry;
}
export function useVoiceTelemetryOptional(): VoiceRuntimeTelemetrySnapshot | null {
const runtime = useContext(VoiceRuntimeContext);
const snapshot = useSyncExternalStore(
runtime ? runtime.subscribeTelemetry : noopSubscribe,
runtime ? runtime.getTelemetrySnapshot : getEmptyTelemetry,
runtime ? runtime.getTelemetrySnapshot : getEmptyTelemetry
);
return runtime ? snapshot : null;
}
export function useVoiceRuntimeOptional(): VoiceRuntime | null {
return useContext(VoiceRuntimeContext);
}
export function useVoiceAudioEngineOptional(): AudioEngine | null {
return useContext(VoiceAudioEngineContext);
}
interface VoiceProviderProps {
@@ -43,405 +121,87 @@ interface VoiceProviderProps {
}
export function VoiceProvider({ children }: VoiceProviderProps) {
const getSession = useSessionStore((state) => state.getSession);
const [activeServerId, setActiveServerId] = useState<string | null>(null);
const [activeAgentId, setActiveAgentId] = useState<string | null>(null);
const { client: runtimeClient, isConnected: activeRuntimeConnected } = useHostRuntimeSession(
activeServerId ?? ""
);
const activeSession = useSessionStore(
useCallback(
(state: ReturnType<typeof useSessionStore.getState>) => {
if (!activeServerId) {
return null;
}
return state.sessions[activeServerId] ?? null;
const providerIdRef = useRef<number | null>(null);
if (providerIdRef.current === null) {
providerIdRef.current = nextVoiceProviderInstanceId++;
console.log("[VoiceProvider] instance_created", {
providerId: providerIdRef.current,
stack: getProviderTraceStack(),
});
}
const providerId = providerIdRef.current;
const engineRef = useRef<AudioEngine | null>(null);
const runtimeRef = useRef<VoiceRuntime | null>(null);
console.log("[VoiceProvider] render", {
providerId,
hasEngine: Boolean(engineRef.current),
hasRuntime: Boolean(runtimeRef.current),
});
if (!engineRef.current) {
let runtime: VoiceRuntime | null = null;
console.log("[VoiceProvider] create_engine_and_runtime");
const engine = createAudioEngine({
onCaptureData: (pcm) => {
console.log("[VoiceProvider] onCaptureData", {
providerId,
bytes: pcm.byteLength,
});
runtime?.handleCapturePcm(pcm);
},
[activeServerId]
)
);
const realtimeSessionRef = useRef<SessionState | null>(null);
const [isVoiceMode, setIsVoiceMode] = useState(false);
const [isVoiceSwitching, setIsVoiceSwitching] = useState(false);
const bargeInPlaybackStopRef = useRef<number | null>(null);
const wasVoiceSocketConnectedRef = useRef(false);
const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
const voiceTransportReadyRef = useRef(false);
const voiceResyncInFlightRef = useRef(false);
const silenceGraceStartMsRef = useRef<number | null>(null);
const speechInterruptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const speechInterruptStartMsRef = useRef<number | null>(null);
const speechStartInterruptSentRef = useRef(false);
const isVoiceModeRef = useRef(false);
const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({
isDetecting: false,
isSpeaking: false,
});
const clearSpeechStartInterruptTimer = useCallback((reason: string) => {
const timer = speechInterruptTimerRef.current;
if (timer) {
clearTimeout(timer);
speechInterruptTimerRef.current = null;
const startedAt = speechInterruptStartMsRef.current;
speechInterruptStartMsRef.current = null;
if (startedAt !== null) {
console.log("[Voice] Cleared speech-start interrupt timer", {
reason,
elapsedMs: Date.now() - startedAt,
onVolumeLevel: (level) => {
console.log("[VoiceProvider] onVolumeLevel", {
providerId,
level,
});
} else {
console.log("[Voice] Cleared speech-start interrupt timer", { reason });
}
return;
}
speechInterruptStartMsRef.current = null;
}, []);
runtime?.handleCaptureVolume(level);
},
onError: (error) => {
console.error("[VoiceEngine] Capture error:", error);
},
}, {
traceLabel: `voice-provider:${providerId}`,
});
const interruptActiveVoiceTurn = useCallback((source: string) => {
const session = realtimeSessionRef.current;
const sessionAudioPlayer = session?.audioPlayer ?? null;
const sessionClient = session?.client ?? null;
const sessionIsPlayingAudio = session?.isPlayingAudio ?? false;
runtime = createVoiceRuntime({
engine,
getServerInfo: (serverId) =>
useSessionStore.getState().getSession(serverId)?.serverInfo ?? null,
activateKeepAwake: async (tag) => {
await activateKeepAwakeAsync(tag);
},
deactivateKeepAwake: async (tag) => {
await deactivateKeepAwake(tag);
},
});
if (sessionIsPlayingAudio && sessionAudioPlayer) {
if (bargeInPlaybackStopRef.current === null) {
bargeInPlaybackStopRef.current = Date.now();
}
sessionAudioPlayer.stop();
}
engineRef.current = engine;
runtimeRef.current = runtime;
}
try {
if (sessionClient) {
void sessionClient.abortRequest().catch((error) => {
console.error("[Voice] Failed to send abort_request:", error);
});
}
console.log("[Voice] Sent abort_request before streaming audio", { source });
} catch (error) {
console.error("[Voice] Failed to send abort_request:", error);
}
}, []);
const realtimeAudio = useSpeechmaticsAudio({
onSpeechStart: () => {
console.log("[Voice] Segment started (speech confirmed)");
},
onSpeechEnd: () => {
const silenceMs =
silenceGraceStartMsRef.current === null
? null
: Date.now() - silenceGraceStartMsRef.current;
if (silenceMs === null) {
console.log("[Voice] Segment finalized");
} else {
console.log("[Voice] Segment finalized", { silenceMs });
}
silenceGraceStartMsRef.current = null;
clearSpeechStartInterruptTimer("speech ended");
speechStartInterruptSentRef.current = false;
},
onAudioSegment: ({ audioData, isLast }) => {
if (!voiceTransportReadyRef.current) {
console.log("[Voice] Skipping audio segment: voice transport not ready");
return;
}
console.log(
"[Voice] Sending audio segment, length:",
audioData.length,
"isLast:",
isLast
);
// Send audio segment to server (realtime always goes to orchestrator)
const session = realtimeSessionRef.current;
try {
if (session?.client) {
void session.client
.sendVoiceAudioChunk(
audioData,
"audio/pcm;rate=16000;bits=16",
isLast
)
.catch((error) => {
console.error("[Voice] Failed to send audio segment:", error);
});
}
} catch (error) {
console.error("[Voice] Failed to send audio segment:", error);
}
},
onError: (error) => {
console.error("[Voice] Audio error:", error);
const session = realtimeSessionRef.current;
if (session?.client) {
// Send error through websocket instead of directly manipulating messages
console.error("[Voice] Cannot handle error - setMessages not available from SessionState");
}
},
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
confirmedDropGracePeriod: REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
silenceDuration: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
speechConfirmationDuration: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
detectionGracePeriod: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
});
useEffect(() => {
realtimeSessionRef.current = activeSession;
}, [activeSession]);
useEffect(() => {
isVoiceModeRef.current = isVoiceMode;
}, [isVoiceMode]);
useEffect(() => {
if (!isVoiceMode) {
clearSpeechStartInterruptTimer("voice mode disabled");
speechStartInterruptSentRef.current = false;
return;
}
if (realtimeAudio.isSpeaking) {
if (
speechStartInterruptSentRef.current ||
speechInterruptTimerRef.current !== null
) {
return;
}
speechInterruptStartMsRef.current = Date.now();
speechInterruptTimerRef.current = setTimeout(() => {
speechInterruptTimerRef.current = null;
speechInterruptStartMsRef.current = null;
if (
!isVoiceModeRef.current ||
!vadStateRef.current.isSpeaking ||
speechStartInterruptSentRef.current
) {
return;
}
speechStartInterruptSentRef.current = true;
console.log("[Voice] Speech persisted beyond grace; interrupting turn", {
graceMs: REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs,
});
interruptActiveVoiceTurn("speech_start_grace_elapsed");
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
return;
}
clearSpeechStartInterruptTimer("speech stopped before grace");
}, [
clearSpeechStartInterruptTimer,
interruptActiveVoiceTurn,
isVoiceMode,
realtimeAudio.isSpeaking,
]);
const engine = engineRef.current!;
const runtime = runtimeRef.current!;
useEffect(() => {
console.log("[VoiceProvider] mount", {
providerId,
});
return () => {
clearSpeechStartInterruptTimer("voice provider unmounted");
};
}, [clearSpeechStartInterruptTimer]);
useEffect(() => {
const next = {
isDetecting: realtimeAudio.isDetecting,
isSpeaking: realtimeAudio.isSpeaking,
};
const prev = vadStateRef.current;
if (!isVoiceMode) {
silenceGraceStartMsRef.current = null;
vadStateRef.current = next;
return;
}
// Confirmed speech dropped below threshold: grace window starts.
if (prev.isSpeaking && !next.isSpeaking && next.isDetecting) {
silenceGraceStartMsRef.current = Date.now();
console.log("[Voice] Grace started (speech dropped below threshold)");
}
// User resumed speaking before grace timeout elapsed.
if (!prev.isSpeaking && next.isSpeaking && silenceGraceStartMsRef.current !== null) {
const resumedAfterMs = Date.now() - silenceGraceStartMsRef.current;
console.log("[Voice] Speech resumed during grace", { resumedAfterMs });
silenceGraceStartMsRef.current = null;
}
// Fully idle (neither detecting nor speaking).
if (!next.isDetecting && !next.isSpeaking) {
silenceGraceStartMsRef.current = null;
speechStartInterruptSentRef.current = false;
}
vadStateRef.current = next;
}, [isVoiceMode, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]);
useEffect(() => {
const connected = activeRuntimeConnected;
const client = runtimeClient;
if (!connected) {
voiceTransportReadyRef.current = false;
}
if (!isVoiceMode || !activeServerId || !activeAgentId || !client) {
wasVoiceSocketConnectedRef.current = connected;
if (!isVoiceMode) {
lastVoiceModeSyncedClientRef.current = null;
}
voiceTransportReadyRef.current = false;
return;
}
const connectionRecovered = connected && !wasVoiceSocketConnectedRef.current;
const clientChanged = lastVoiceModeSyncedClientRef.current !== client;
if (connected && (connectionRecovered || clientChanged)) {
if (!voiceResyncInFlightRef.current) {
voiceResyncInFlightRef.current = true;
voiceTransportReadyRef.current = false;
setIsVoiceSwitching(true);
void client.setVoiceMode(true, activeAgentId).then(
() => {
console.log("[Voice] Re-synced voice mode after reconnect");
lastVoiceModeSyncedClientRef.current = client;
voiceTransportReadyRef.current = true;
},
(error) => {
console.error("[Voice] Failed to re-sync voice mode:", error);
}
).finally(() => {
voiceResyncInFlightRef.current = false;
setIsVoiceSwitching(false);
});
}
}
wasVoiceSocketConnectedRef.current = connected;
}, [activeAgentId, activeRuntimeConnected, activeServerId, isVoiceMode, runtimeClient]);
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
useEffect(() => {
if (!isPlayingAudio && bargeInPlaybackStopRef.current !== null) {
const latencyMs = Date.now() - bargeInPlaybackStopRef.current;
console.log("[Telemetry] barge_in.playback_stop_latency", {
latencyMs,
startedAt: new Date(bargeInPlaybackStopRef.current).toISOString(),
completedAt: new Date().toISOString(),
console.log("[VoiceProvider] unmount", {
providerId,
});
bargeInPlaybackStopRef.current = null;
}
}, [isPlayingAudio]);
const startVoice = useCallback(
async (serverId: string, agentId: string) => {
const session = getSession(serverId) ?? null;
if (!session) {
throw new Error(`Host ${serverId} is not connected`);
}
const unavailableMessage = resolveVoiceUnavailableMessage({
serverInfo: session.serverInfo,
mode: "voice",
void runtime.destroy().catch((error) => {
console.error("[VoiceProvider] Failed to destroy voice runtime", error);
});
if (unavailableMessage) {
throw new Error(unavailableMessage);
}
setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false;
try {
const previousSession = realtimeSessionRef.current;
if (
isVoiceMode &&
previousSession?.client &&
(activeServerId !== serverId || activeAgentId !== agentId)
) {
await previousSession.client.setVoiceMode(false);
}
realtimeSessionRef.current = session;
setActiveServerId(serverId);
setActiveAgentId(agentId);
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
console.warn("[Voice] Failed to activate keep-awake:", error);
});
if (session?.client) {
await session.client.setVoiceMode(true, agentId);
} else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
await session.audioPlayer?.warmup?.();
await realtimeAudio.start();
voiceTransportReadyRef.current = true;
setIsVoiceMode(true);
lastVoiceModeSyncedClientRef.current = session.client;
console.log("[Voice] Mode enabled");
} catch (error: any) {
console.error("[Voice] Failed to start:", error);
await realtimeAudio.stop().catch(() => undefined);
setActiveServerId((current) => (current === serverId ? null : current));
setActiveAgentId((current) => (current === agentId ? null : current));
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
throw error;
} finally {
setIsVoiceSwitching(false);
}
},
[activeAgentId, activeServerId, getSession, isVoiceMode, realtimeAudio]
);
const stopVoice = useCallback(async () => {
setIsVoiceSwitching(true);
voiceTransportReadyRef.current = false;
try {
const session = realtimeSessionRef.current;
session?.audioPlayer?.stop();
if (session?.client) {
await session.client.setVoiceMode(false);
lastVoiceModeSyncedClientRef.current = session.client;
} else {
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
}
await realtimeAudio.stop();
setIsVoiceMode(false);
setActiveServerId(null);
setActiveAgentId(null);
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
console.log("[Voice] Mode disabled");
} catch (error: any) {
console.error("[Voice] Failed to stop:", error);
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
throw error;
} finally {
setIsVoiceSwitching(false);
}
}, [realtimeAudio]);
const isVoiceModeForAgent = useCallback(
(serverId: string, agentId: string) =>
isVoiceMode && activeServerId === serverId && activeAgentId === agentId,
[activeAgentId, activeServerId, isVoiceMode]
);
const value: VoiceContextValue = {
isVoiceMode,
isVoiceSwitching,
volume: realtimeAudio.volume,
isMuted: realtimeAudio.isMuted,
isDetecting: realtimeAudio.isDetecting,
isSpeaking: realtimeAudio.isSpeaking,
segmentDuration: realtimeAudio.segmentDuration,
startVoice,
stopVoice,
isVoiceModeForAgent,
toggleMute: realtimeAudio.toggleMute,
activeServerId,
activeAgentId,
};
};
}, [providerId, runtime]);
return (
<VoiceContext.Provider value={value}>
{children}
</VoiceContext.Provider>
<VoiceAudioEngineContext.Provider value={engine}>
<VoiceRuntimeContext.Provider value={runtime}>
{children}
</VoiceRuntimeContext.Provider>
</VoiceAudioEngineContext.Provider>
);
}

1
packages/app/src/hooks/use-audio-player.d.ts vendored
View File

@@ -1 +0,0 @@
export * from "./use-audio-player.native";

366
packages/app/src/hooks/use-audio-player.native.ts
View File

@@ -1,366 +0,0 @@
import { useState, useRef } from "react";
import {
initialize,
playPCMData,
stopPlayback,
pausePlayback,
resumePlayback,
} from "@boudra/expo-two-way-audio";
interface QueuedAudio {
audioData: Blob;
resolve: (duration: number) => void;
reject: (error: Error) => void;
}
/**
* Resample PCM16 audio between sample rates.
* Speechmatics expects 16kHz.
*/
function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array {
if (fromRate === toRate) {
return pcm;
}
const inputSamples = Math.floor(pcm.length / 2);
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
const out = new Uint8Array(outputSamples * 2);
const ratio = fromRate / toRate;
const readInt16 = (sampleIndex: number): number => {
const i = sampleIndex * 2;
if (i + 1 >= pcm.length) {
return 0;
}
const lo = pcm[i]!;
const hi = pcm[i + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value = value - 0x10000;
}
return value;
};
const writeInt16 = (sampleIndex: number, value: number): void => {
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
const i = sampleIndex * 2;
out[i] = clamped & 0xff;
out[i + 1] = (clamped >> 8) & 0xff;
};
for (let i = 0; i < outputSamples; i++) {
const srcPos = i * ratio;
const i0 = Math.floor(srcPos);
const frac = srcPos - i0;
const s0 = readInt16(i0);
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
writeInt16(i, s0 + (s1 - s0) * frac);
}
return out;
}
function parsePcmSampleRate(mimeType: string): number | null {
const match = /rate=(\d+)/i.exec(mimeType);
if (!match) {
return null;
}
const rate = Number(match[1]);
return Number.isFinite(rate) && rate > 0 ? rate : null;
}
export interface AudioPlayerOptions {
isDetecting?: () => boolean;
isSpeaking?: () => boolean;
}
/**
* Hook for audio playback using Speechmatics two-way audio with echo cancellation
*/
export function useAudioPlayer(options?: AudioPlayerOptions) {
const [isPlaying, setIsPlaying] = useState(false);
const [audioInitialized, setAudioInitialized] = useState(false);
const queueRef = useRef<QueuedAudio[]>([]);
const suppressedQueueRef = useRef<QueuedAudio[]>([]);
const isProcessingQueueRef = useRef(false);
const activePlaybackRef = useRef<{
resolve: (duration: number) => void;
reject: (error: Error) => void;
} | null>(null);
const playbackTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
async function play(audioData: Blob): Promise<number> {
return new Promise((resolve, reject) => {
// Check if we should suppress playback due to voice detection/speaking
const shouldSuppress =
(options?.isDetecting && options.isDetecting()) ||
(options?.isSpeaking && options.isSpeaking());
if (shouldSuppress) {
console.log("[AudioPlayer] Suppressing playback - voice detection/speaking active");
// Add to suppressed queue instead
suppressedQueueRef.current.push({ audioData, resolve, reject });
// Start checking for when flags clear
startCheckingForClearFlags();
return;
}
// Add to queue with its promise handlers
queueRef.current.push({ audioData, resolve, reject });
// Start processing queue if not already processing
if (!isProcessingQueueRef.current) {
processQueue();
}
});
}
function startCheckingForClearFlags(): void {
// Already checking
if (checkIntervalRef.current !== null) {
return;
}
console.log("[AudioPlayer] Starting to check for clear flags");
checkIntervalRef.current = setInterval(() => {
const isStillBlocked =
(options?.isDetecting && options.isDetecting()) ||
(options?.isSpeaking && options.isSpeaking());
if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
console.log("[AudioPlayer] Flags cleared - moving suppressed queue to main queue");
// Move all suppressed items to main queue
const suppressedItems = [...suppressedQueueRef.current];
suppressedQueueRef.current = [];
// Add to front of main queue (they were waiting)
queueRef.current = [...suppressedItems, ...queueRef.current];
// Stop checking
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
// Start processing if not already
if (!isProcessingQueueRef.current) {
processQueue();
}
} else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
// No more suppressed items and flags are clear - stop checking
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
}
}, 100); // Check every 100ms
}
async function processQueue(): Promise<void> {
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
return;
}
isProcessingQueueRef.current = true;
while (queueRef.current.length > 0) {
// Before processing each item, check if flags became active
const shouldSuppress =
(options?.isDetecting && options.isDetecting()) ||
(options?.isSpeaking && options.isSpeaking());
if (shouldSuppress) {
console.log("[AudioPlayer] Flags became active during processing - moving remaining queue to suppressed");
// Move remaining queue to suppressed
suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
queueRef.current = [];
startCheckingForClearFlags();
break;
}
const item = queueRef.current.shift()!;
try {
const duration = await playAudio(item.audioData);
item.resolve(duration);
} catch (error) {
item.reject(error as Error);
}
}
isProcessingQueueRef.current = false;
}
async function processNextInQueue(): Promise<void> {
if (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
try {
const duration = await playAudio(item.audioData);
item.resolve(duration);
} catch (error) {
item.reject(error as Error);
}
} else {
isProcessingQueueRef.current = false;
}
}
async function playAudio(audioData: Blob): Promise<number> {
return new Promise(async (resolve, reject) => {
activePlaybackRef.current = { resolve, reject };
try {
console.log(
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
);
// Initialize audio if not already initialized
if (!audioInitialized) {
console.log("[AudioPlayer] Initializing audio...");
await initialize();
setAudioInitialized(true);
console.log(
"[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)"
);
}
// Workaround: Resume playback before playing new audio to ensure the audio engine is ready
// This fixes the issue where playback doesn't work after calling stopPlayback()
console.log("[AudioPlayer] Resuming playback engine...");
resumePlayback();
// Get PCM data from blob (server sends PCM16)
const arrayBuffer = await audioData.arrayBuffer();
const pcm = new Uint8Array(arrayBuffer);
const inputRate = parsePcmSampleRate(audioData.type || "") ?? 24000;
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
// Calculate total duration
const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample
const durationSec = samples / 16000; // 16kHz sample rate
const audioSizeKb = (pcm16k.length / 1024).toFixed(2);
console.log(
"[AudioPlayer] 🔊 Playing audio:",
audioSizeKb,
"KB, duration:",
durationSec.toFixed(2),
"s"
);
setIsPlaying(true);
// Play entire PCM data at once through Speechmatics
playPCMData(pcm16k);
// Clear any existing timeout
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
}
// Wait for playback to finish (estimate based on duration)
playbackTimeoutRef.current = setTimeout(() => {
console.log("[AudioPlayer] ✅ Playback finished");
setIsPlaying(false);
playbackTimeoutRef.current = null;
activePlaybackRef.current = null;
resolve(durationSec);
}, durationSec * 1000);
} catch (error) {
console.error("[AudioPlayer] Error playing audio:", error);
// Clear timeout on error
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
playbackTimeoutRef.current = null;
}
setIsPlaying(false);
activePlaybackRef.current = null;
reject(error);
}
});
}
function stop(): void {
if (isPlaying) {
console.log("[AudioPlayer] 🛑 Stopping playback (interrupted)");
// Stop native playback
stopPlayback();
// Clear playback timeout
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
playbackTimeoutRef.current = null;
}
}
setIsPlaying(false);
// Reject the currently playing promise, if any.
if (activePlaybackRef.current) {
activePlaybackRef.current.reject(new Error("Playback stopped"));
activePlaybackRef.current = null;
}
// Reject all pending promises in the main queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error("Playback stopped"));
}
// Reject all pending promises in the suppressed queue
while (suppressedQueueRef.current.length > 0) {
const item = suppressedQueueRef.current.shift()!;
item.reject(new Error("Playback stopped"));
}
// Clear check interval
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
isProcessingQueueRef.current = false;
}
function clearQueue(): void {
// Reject all pending promises in the main queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error("Queue cleared"));
}
// Reject all pending promises in the suppressed queue
while (suppressedQueueRef.current.length > 0) {
const item = suppressedQueueRef.current.shift()!;
item.reject(new Error("Queue cleared"));
}
// Clear check interval
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
}
return {
play,
stop,
isPlaying: () => isPlaying,
clearQueue,
warmup: async () => {
if (!audioInitialized) {
await initialize();
setAudioInitialized(true);
}
// Ensure playback engine isn't suspended after a previous stop.
resumePlayback();
},
};
}

275
packages/app/src/hooks/use-audio-player.web.ts
View File

@@ -1,275 +0,0 @@
import { useMemo, useRef, useState } from "react";
export interface AudioPlayerOptions {
isDetecting?: () => boolean;
isSpeaking?: () => boolean;
}
/**
* Web audio player for server-provided audio chunks.
*
* Supports:
* - `audio/pcm` (assumed PCM16 LE, mono, default 24kHz unless `rate=` is present)
* - formats supported by `AudioContext.decodeAudioData` (e.g. mp3)
*/
export function useAudioPlayer(options?: AudioPlayerOptions) {
const [isPlayingState, setIsPlayingState] = useState(false);
type QueuedAudio = {
audioData: Blob;
resolve: (duration: number) => void;
reject: (error: Error) => void;
};
const audioContextRef = useRef<AudioContext | null>(null);
const queueRef = useRef<QueuedAudio[]>([]);
const suppressedQueueRef = useRef<QueuedAudio[]>([]);
const isProcessingQueueRef = useRef(false);
const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const activePlaybackRef = useRef<{
source: AudioBufferSourceNode;
resolve: (duration: number) => void;
reject: (error: Error) => void;
settled: boolean;
} | null>(null);
const decodeAudioData = async (context: AudioContext, buffer: ArrayBuffer): Promise<AudioBuffer> => {
const maybePromise = context.decodeAudioData(buffer);
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
return maybePromise as Promise<AudioBuffer>;
}
return await new Promise<AudioBuffer>((resolve, reject) => {
context.decodeAudioData(buffer, resolve, reject);
});
};
const parsePcmSampleRate = (mimeType: string): number | null => {
const match = /rate=(\\d+)/i.exec(mimeType);
if (!match) {
return null;
}
const rate = Number(match[1]);
return Number.isFinite(rate) && rate > 0 ? rate : null;
};
const pcm16LeToAudioBuffer = (context: AudioContext, bytes: Uint8Array, sampleRate: number): AudioBuffer => {
const sampleCount = Math.floor(bytes.length / 2);
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
const channel = audioBuffer.getChannelData(0);
for (let i = 0; i < sampleCount; i++) {
const lo = bytes[i * 2]!;
const hi = bytes[i * 2 + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value = value - 0x10000;
}
channel[i] = value / 0x8000;
}
return audioBuffer;
};
const ensureContext = async (): Promise<AudioContext> => {
if (audioContextRef.current) {
if (audioContextRef.current.state === "suspended") {
try {
await audioContextRef.current.resume();
} catch {
// Best effort. If this fails due to autoplay policies, a later user gesture
// (or explicit warmup call) can unlock it.
}
}
return audioContextRef.current;
}
const context = new AudioContext();
if (context.state === "suspended") {
try {
await context.resume();
} catch {
// See note above.
}
}
audioContextRef.current = context;
return context;
};
const shouldSuppressPlayback = (): boolean => {
return Boolean(options?.isDetecting?.()) || Boolean(options?.isSpeaking?.());
};
const startCheckingForClearFlags = (): void => {
if (checkIntervalRef.current !== null) {
return;
}
checkIntervalRef.current = setInterval(() => {
const isStillBlocked = shouldSuppressPlayback();
if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
const suppressedItems = [...suppressedQueueRef.current];
suppressedQueueRef.current = [];
queueRef.current = [...suppressedItems, ...queueRef.current];
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
if (!isProcessingQueueRef.current) {
void processQueue();
}
} else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
}
}, 100);
};
const playAudio = async (audioData: Blob): Promise<number> => {
const context = await ensureContext();
const arrayBuffer = await audioData.arrayBuffer();
let audioBuffer: AudioBuffer;
const type = (audioData.type || "").toLowerCase();
if (type.startsWith("audio/pcm")) {
const sampleRate = parsePcmSampleRate(type) ?? 24000;
audioBuffer = pcm16LeToAudioBuffer(context, new Uint8Array(arrayBuffer), sampleRate);
} else {
audioBuffer = await decodeAudioData(context, arrayBuffer);
}
const durationSec = audioBuffer.duration;
const source = context.createBufferSource();
source.buffer = audioBuffer;
source.connect(context.destination);
return await new Promise<number>((resolve, reject) => {
activePlaybackRef.current = { source, resolve, reject, settled: false };
setIsPlayingState(true);
const settleOnce = (fn: () => void) => {
const active = activePlaybackRef.current;
if (!active || active.source !== source || active.settled) {
return;
}
active.settled = true;
activePlaybackRef.current = null;
setIsPlayingState(false);
fn();
};
source.onended = () => {
settleOnce(() => resolve(durationSec));
};
try {
source.start();
} catch (e) {
settleOnce(() => reject(e instanceof Error ? e : new Error(String(e))));
}
});
};
const processQueue = async (): Promise<void> => {
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
return;
}
isProcessingQueueRef.current = true;
while (queueRef.current.length > 0) {
if (shouldSuppressPlayback()) {
suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
queueRef.current = [];
startCheckingForClearFlags();
break;
}
const item = queueRef.current.shift()!;
try {
const duration = await playAudio(item.audioData);
item.resolve(duration);
} catch (error) {
item.reject(error as Error);
}
}
isProcessingQueueRef.current = false;
};
const play = async (audioData: Blob): Promise<number> => {
return await new Promise<number>((resolve, reject) => {
if (shouldSuppressPlayback()) {
suppressedQueueRef.current.push({ audioData, resolve, reject });
startCheckingForClearFlags();
return;
}
queueRef.current.push({ audioData, resolve, reject });
if (!isProcessingQueueRef.current) {
void processQueue();
}
});
};
const stop = (): void => {
// Stop currently playing audio (and reject its promise)
if (activePlaybackRef.current) {
const active = activePlaybackRef.current;
activePlaybackRef.current = null;
try {
active.source.onended = null;
active.source.stop();
} catch {
// ignore
}
if (!active.settled) {
active.settled = true;
active.reject(new Error("Playback stopped"));
}
}
setIsPlayingState(false);
while (queueRef.current.length > 0) {
queueRef.current.shift()!.reject(new Error("Playback stopped"));
}
while (suppressedQueueRef.current.length > 0) {
suppressedQueueRef.current.shift()!.reject(new Error("Playback stopped"));
}
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
isProcessingQueueRef.current = false;
};
const clearQueue = (): void => {
while (queueRef.current.length > 0) {
queueRef.current.shift()!.reject(new Error("Queue cleared"));
}
while (suppressedQueueRef.current.length > 0) {
suppressedQueueRef.current.shift()!.reject(new Error("Queue cleared"));
}
if (checkIntervalRef.current !== null) {
clearInterval(checkIntervalRef.current);
checkIntervalRef.current = null;
}
};
return useMemo(
() => ({
play,
stop,
isPlaying: () => isPlayingState,
clearQueue,
warmup: async () => {
await ensureContext();
},
}),
[isPlayingState]
);
}

120
packages/app/src/hooks/use-dictation-audio-source.native.ts
View File

@@ -1,44 +1,124 @@
import { useCallback, useEffect, useRef } from "react";
import { Buffer } from "buffer";
import { useState } from "react";
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
import { createAudioEngine } from "@/voice/audio-engine";
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
let nextDictationAudioSourceInstanceId = 1;
function getDictationTraceStack(): string | undefined {
const stack = new Error().stack;
if (!stack) {
return undefined;
}
return stack
.split("\n")
.slice(2, 7)
.map((line) => line.trim())
.join(" | ");
}
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
const onPcmSegmentRef = useRef(config.onPcmSegment);
const onErrorRef = useRef(config.onError);
const [volume, setVolume] = useState(0);
const sourceIdRef = useRef<number | null>(null);
const engineRef = useRef<ReturnType<typeof createAudioEngine> | null>(null);
if (sourceIdRef.current === null) {
sourceIdRef.current = nextDictationAudioSourceInstanceId++;
console.log("[DictationAudioSource] instance_created", {
sourceId: sourceIdRef.current,
stack: getDictationTraceStack(),
});
}
const sourceId = sourceIdRef.current;
console.log("[DictationAudioSource] render", {
sourceId,
hasEngine: Boolean(engineRef.current),
});
const getOrCreateEngine = useCallback(() => {
if (engineRef.current) {
return engineRef.current;
}
console.log("[DictationAudioSource] create_engine", {
sourceId,
stack: getDictationTraceStack(),
});
engineRef.current = createAudioEngine(
{
onCaptureData: (pcm) => {
onPcmSegmentRef.current(Buffer.from(pcm).toString("base64"));
},
onVolumeLevel: (level) => {
setVolume(level);
},
onError: (error) => {
onErrorRef.current?.(error);
},
},
{
traceLabel: `dictation:${sourceId}`,
}
);
return engineRef.current;
}, [sourceId]);
useEffect(() => {
onPcmSegmentRef.current = config.onPcmSegment;
onErrorRef.current = config.onError;
}, [config.onPcmSegment, config.onError]);
const speechmatics = useSpeechmaticsAudio({
enableContinuousStreaming: true,
onAudioSegment: ({ audioData }) => {
onPcmSegmentRef.current(audioData);
},
onError: (err) => {
onErrorRef.current?.(err);
},
volumeThreshold: 0.3,
silenceDuration: 2000,
speechConfirmationDuration: 300,
detectionGracePeriod: 200,
});
useEffect(() => {
console.log("[DictationAudioSource] mount", {
sourceId,
});
return () => {
console.log("[DictationAudioSource] unmount", {
sourceId,
});
};
}, [sourceId]);
const start = useCallback(async () => {
await speechmatics.start();
}, [speechmatics]);
console.log("[DictationAudioSource] start", {
sourceId,
});
const engine = getOrCreateEngine();
await engine.initialize();
await engine.startCapture();
}, [getOrCreateEngine, sourceId]);
const stop = useCallback(async () => {
await speechmatics.stop();
}, [speechmatics]);
console.log("[DictationAudioSource] stop", {
sourceId,
hasEngine: Boolean(engineRef.current),
});
await engineRef.current?.stopCapture();
setVolume(0);
}, [sourceId]);
useEffect(() => {
return () => {
const engine = engineRef.current;
engineRef.current = null;
console.log("[DictationAudioSource] destroy_engine", {
sourceId,
hadEngine: Boolean(engine),
});
void engine?.destroy().catch(() => undefined);
};
}, [sourceId]);
return {
start,
stop,
volume: speechmatics.volume,
volume,
};
}

79
packages/app/src/hooks/use-dictation-audio-source.web.ts
View File

@@ -1,4 +1,5 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { parsePcm16Wav } from "@/utils/pcm16-wav";
import { getTauri } from "@/utils/tauri";
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
@@ -65,84 +66,6 @@ const int16ToFloat32 = (input: Int16Array): Float32Array => {
return out;
};
type Pcm16Wav = {
sampleRate: number;
samples: Int16Array;
};
const parsePcm16Wav = (buffer: ArrayBuffer): Pcm16Wav | null => {
if (buffer.byteLength < 44) {
return null;
}
const view = new DataView(buffer);
const readAscii = (offset: number, length: number): string => {
let out = "";
for (let i = 0; i < length; i += 1) {
out += String.fromCharCode(view.getUint8(offset + i));
}
return out;
};
if (readAscii(0, 4) !== "RIFF" || readAscii(8, 4) !== "WAVE") {
return null;
}
let offset = 12;
let channels = 0;
let sampleRate = 0;
let bitsPerSample = 0;
let dataOffset = 0;
let dataSize = 0;
while (offset + 8 <= buffer.byteLength) {
const chunkId = readAscii(offset, 4);
const chunkSize = view.getUint32(offset + 4, true);
const chunkDataOffset = offset + 8;
if (chunkDataOffset + chunkSize > buffer.byteLength) {
break;
}
if (chunkId === "fmt " && chunkSize >= 16) {
const audioFormat = view.getUint16(chunkDataOffset, true);
channels = view.getUint16(chunkDataOffset + 2, true);
sampleRate = view.getUint32(chunkDataOffset + 4, true);
bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
if (audioFormat !== 1) {
return null;
}
} else if (chunkId === "data") {
dataOffset = chunkDataOffset;
dataSize = chunkSize;
break;
}
offset = chunkDataOffset + chunkSize + (chunkSize % 2);
}
if (!dataOffset || !dataSize || sampleRate <= 0 || bitsPerSample !== 16 || channels <= 0) {
return null;
}
const sampleCount = Math.floor(dataSize / 2);
const interleaved = new Int16Array(buffer, dataOffset, sampleCount);
if (channels === 1) {
return { sampleRate, samples: new Int16Array(interleaved) };
}
const frameCount = Math.floor(interleaved.length / channels);
const mono = new Int16Array(frameCount);
for (let frame = 0; frame < frameCount; frame += 1) {
let sum = 0;
for (let ch = 0; ch < channels; ch += 1) {
sum += interleaved[frame * channels + ch] ?? 0;
}
mono[frame] = Math.round(sum / channels);
}
return { sampleRate, samples: mono };
};
const int16ToBase64 = (pcm: Int16Array): string => {
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
let binary = "";

43
packages/app/src/hooks/use-sidebar-workspaces-list.ts
View File

@@ -267,8 +267,6 @@ export function useSidebarWorkspacesList(options?: {
const value = options?.serverId
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null
}, [options?.serverId])
const enabled = options?.enabled ?? true
const persistedProjectOrder = useSidebarOrderStore((state) =>
serverId ? (state.projectOrderByServerId[serverId] ?? EMPTY_ORDER) : EMPTY_ORDER
)
@@ -304,6 +302,13 @@ export function useSidebarWorkspacesList(options?: {
)
const projects = useMemo(() => {
console.log('[useSidebarWorkspacesList] build_projects', {
serverId,
hasSessionWorkspaces: Boolean(sessionWorkspaces),
workspaceCount: sessionWorkspaces?.size ?? 0,
projectOrderLength: persistedProjectOrder.length,
workspaceOrderScopeCount: Object.keys(persistedWorkspaceOrderByScope).length,
})
if (!sessionWorkspaces || sessionWorkspaces.size === 0 || !serverId) {
return EMPTY_PROJECTS
}
@@ -315,6 +320,36 @@ export function useSidebarWorkspacesList(options?: {
})
}, [persistedProjectOrder, persistedWorkspaceOrderByScope, serverId, sessionWorkspaces])
useEffect(() => {
console.log('[useSidebarWorkspacesList] inputs_changed', {
serverId,
connectionStatus,
hasHydratedWorkspaces,
workspaceCount: sessionWorkspaces?.size ?? 0,
projectOrderLength: persistedProjectOrder.length,
workspaceOrderScopeCount: Object.keys(persistedWorkspaceOrderByScope).length,
})
}, [
connectionStatus,
hasHydratedWorkspaces,
persistedProjectOrder,
persistedWorkspaceOrderByScope,
serverId,
sessionWorkspaces,
])
useEffect(() => {
console.log('[useSidebarWorkspacesList] projects_changed', {
serverId,
projectCount: projects.length,
projectKeys: projects.map((project) => project.projectKey),
workspaceCounts: projects.map((project) => ({
projectKey: project.projectKey,
workspaceCount: project.workspaces.length,
})),
})
}, [projects, serverId])
useEffect(() => {
if (!serverId || projects.length === 0) {
return
@@ -345,7 +380,7 @@ export function useSidebarWorkspacesList(options?: {
}, [persistedProjectOrder, persistedWorkspaceOrderByScope, projects, serverId])
const refreshAll = useCallback(() => {
if (!isActive || !serverId || connectionStatus !== 'online' || !enabled) {
if (!isActive || !serverId || connectionStatus !== 'online') {
return
}
const client = runtime.getClient(serverId)
@@ -377,7 +412,7 @@ export function useSidebarWorkspacesList(options?: {
// ignore explicit refresh failures; hook keeps existing data
}
})()
}, [connectionStatus, enabled, isActive, runtime, serverId])
}, [connectionStatus, isActive, runtime, serverId])
const isLoading = isActive && Boolean(serverId) && connectionStatus === 'online' && !hasHydratedWorkspaces
const isInitialLoad = isLoading && projects.length === 0

1
packages/app/src/hooks/use-speechmatics-audio.d.ts vendored
View File

@@ -1 +0,0 @@
export * from "./use-speechmatics-audio.native";

274
packages/app/src/hooks/use-speechmatics-audio.native.ts
View File

@@ -1,274 +0,0 @@
import { useState, useEffect, useRef, useCallback } from "react";
import {
initialize,
useMicrophonePermissions,
toggleRecording,
tearDown,
useExpoTwoWayAudioEventListener,
type MicrophoneDataCallback,
type VolumeLevelCallback,
} from "@boudra/expo-two-way-audio";
import { SpeechSegmenter } from "@/voice/speech-segmenter";
export interface SpeechmaticsAudioConfig {
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
onSpeechStart?: () => void;
onSpeechEnd?: () => void;
onError?: (error: Error) => void;
/** When true, stream microphone PCM continuously without VAD gating. */
enableContinuousStreaming?: boolean;
volumeThreshold: number; // Volume threshold for speech detection (0-1)
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
confirmedDropGracePeriod?: number;
silenceDuration: number; // ms of silence before ending segment
speechConfirmationDuration: number; // ms of sustained speech before confirming
detectionGracePeriod: number; // ms grace period for volume dips during detection
}
export interface SpeechmaticsAudio {
start: () => Promise<void>;
stop: () => Promise<void>;
toggleMute: () => void;
isActive: boolean;
isSpeaking: boolean;
isDetecting: boolean;
isMuted: boolean;
volume: number;
segmentDuration: number;
}
/**
* Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
*/
export function useSpeechmaticsAudio(
config: SpeechmaticsAudioConfig
): SpeechmaticsAudio {
const [microphonePermission, requestMicrophonePermission] =
useMicrophonePermissions();
const [isActive, setIsActive] = useState(false);
const [isSpeaking, setIsSpeaking] = useState(false);
const [isDetecting, setIsDetecting] = useState(false);
const [audioInitialized, setAudioInitialized] = useState(false);
const [volume, setVolume] = useState(0);
const [isMuted, setIsMuted] = useState(false);
const [segmentDuration, setSegmentDuration] = useState(0);
const enableContinuousStreaming = config.enableContinuousStreaming === true;
const isActiveRef = useRef(isActive);
useEffect(() => {
isActiveRef.current = isActive;
}, [isActive]);
const isMutedRef = useRef(isMuted);
useEffect(() => {
isMutedRef.current = isMuted;
}, [isMuted]);
const callbacksRef = useRef({
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
});
useEffect(() => {
callbacksRef.current = {
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
};
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd]);
const segmenterRef = useRef<SpeechSegmenter | null>(null);
if (segmenterRef.current === null) {
segmenterRef.current = new SpeechSegmenter(
{
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
},
{
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
onDetectingChange: (next) => setIsDetecting(next),
onSpeakingChange: (next) => setIsSpeaking(next),
}
);
}
useEffect(() => {
segmenterRef.current?.updateConfig({
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
});
}, [
enableContinuousStreaming,
config.volumeThreshold,
config.confirmedDropGracePeriod,
config.silenceDuration,
config.speechConfirmationDuration,
config.detectionGracePeriod,
]);
// Update segment duration timer
useEffect(() => {
if (!isDetecting && !isSpeaking) {
setSegmentDuration(0);
return;
}
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
const interval = setInterval(() => {
const elapsed = Date.now() - startTime;
setSegmentDuration(elapsed);
}, 100);
return () => clearInterval(interval);
}, [isDetecting, isSpeaking]);
// Listen to microphone data
useExpoTwoWayAudioEventListener(
"onMicrophoneData",
useCallback<MicrophoneDataCallback>(
(event) => {
if (!isActiveRef.current || isMutedRef.current) return;
const pcmData: Uint8Array = event.data;
segmenterRef.current?.pushPcmChunk(pcmData);
},
[]
)
);
// Listen to volume level for VAD
useExpoTwoWayAudioEventListener(
"onInputVolumeLevelData",
useCallback<VolumeLevelCallback>(
(event) => {
if (!isActiveRef.current) return;
const volumeLevel: number = event.data;
setVolume(volumeLevel);
if (isMutedRef.current) return;
segmenterRef.current?.pushVolumeLevel(volumeLevel, Date.now());
},
[]
)
);
const ensureMicrophonePermission = useCallback(async () => {
let permissionStatus = microphonePermission;
if (!permissionStatus?.granted) {
try {
permissionStatus = await requestMicrophonePermission();
} catch (err) {
throw new Error("Failed to request microphone permission");
}
}
if (!permissionStatus?.granted) {
throw new Error(
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
);
}
}, [microphonePermission, requestMicrophonePermission]);
async function start(): Promise<void> {
if (isActive) {
console.log("[SpeechmaticsAudio] Already active");
return;
}
try {
await ensureMicrophonePermission();
// Initialize audio if not already initialized
if (!audioInitialized) {
console.log("[SpeechmaticsAudio] Initializing audio...");
await initialize();
setAudioInitialized(true);
console.log("[SpeechmaticsAudio] Audio initialized");
}
console.log("[SpeechmaticsAudio] Starting audio capture...");
// Start recording
toggleRecording(true);
setIsActive(true);
console.log("[SpeechmaticsAudio] Audio capture started successfully");
} catch (error) {
console.error("[SpeechmaticsAudio] Start error:", error);
const err = error instanceof Error ? error : new Error(String(error));
config.onError?.(err);
await stop();
throw error;
}
}
async function stop(): Promise<void> {
console.log("[SpeechmaticsAudio] Stopping audio capture...");
// Stop recording
if (isActive) {
toggleRecording(false);
}
segmenterRef.current?.stop(Date.now());
// Tear down audio session
if (audioInitialized) {
tearDown();
setAudioInitialized(false);
console.log("[SpeechmaticsAudio] Audio torn down");
}
// Reset state
segmenterRef.current?.reset();
setIsActive(false);
setIsSpeaking(false);
setIsDetecting(false);
setVolume(0);
setIsMuted(false);
console.log("[SpeechmaticsAudio] Audio capture stopped");
}
function toggleMute(): void {
setIsMuted((prev) => {
const newMuted = !prev;
console.log("[SpeechmaticsAudio] Mute toggled:", newMuted);
if (newMuted) {
// Clear any ongoing speech detection/speaking state
segmenterRef.current?.reset();
setIsSpeaking(false);
setIsDetecting(false);
}
return newMuted;
});
}
return {
start,
stop,
toggleMute,
isActive,
isSpeaking,
isDetecting,
isMuted,
volume,
segmentDuration,
};
}

410
packages/app/src/hooks/use-speechmatics-audio.web.ts
View File

@@ -1,410 +0,0 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { SpeechSegmenter } from "@/voice/speech-segmenter";
import { getTauri } from "@/utils/tauri";
export interface SpeechmaticsAudioConfig {
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
onSpeechStart?: () => void;
onSpeechEnd?: () => void;
onError?: (error: Error) => void;
/** When true, stream microphone PCM continuously without VAD gating. */
enableContinuousStreaming?: boolean;
volumeThreshold: number; // 0-1
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
confirmedDropGracePeriod?: number;
silenceDuration: number; // ms of silence before ending segment
speechConfirmationDuration: number; // ms of sustained speech before confirming
detectionGracePeriod: number; // ms grace period for volume dips during detection
}
export interface SpeechmaticsAudio {
start: () => Promise<void>;
stop: () => Promise<void>;
toggleMute: () => void;
isActive: boolean;
isSpeaking: boolean;
isDetecting: boolean;
isMuted: boolean;
volume: number;
segmentDuration: number;
}
const getAudioContextCtor = (): (typeof AudioContext) | null => {
if (typeof window === "undefined") {
return null;
}
const ctor =
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
return ctor ?? null;
};
const floatToInt16 = (sample: number): number => {
const clamped = Math.max(-1, Math.min(1, sample));
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
};
const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
if (input.length === 0) {
return new Int16Array(0);
}
if (inputRate === outputRate) {
const out = new Int16Array(input.length);
for (let i = 0; i < input.length; i++) {
out[i] = floatToInt16(input[i]);
}
return out;
}
const ratio = inputRate / outputRate;
const outputLength = Math.max(1, Math.round(input.length / ratio));
const out = new Int16Array(outputLength);
for (let i = 0; i < outputLength; i++) {
const sourceIndex = i * ratio;
const i0 = Math.floor(sourceIndex);
const i1 = Math.min(input.length - 1, i0 + 1);
const frac = sourceIndex - i0;
const sample = input[i0] * (1 - frac) + input[i1] * frac;
out[i] = floatToInt16(sample);
}
return out;
};
export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): SpeechmaticsAudio {
const [isActive, setIsActive] = useState(false);
const [isSpeaking, setIsSpeaking] = useState(false);
const [isDetecting, setIsDetecting] = useState(false);
const [volume, setVolume] = useState(0);
const [isMuted, setIsMuted] = useState(false);
const [segmentDuration, setSegmentDuration] = useState(0);
const enableContinuousStreaming = config.enableContinuousStreaming === true;
const callbacksRef = useRef({
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
onError: config.onError,
});
useEffect(() => {
callbacksRef.current = {
onAudioSegment: config.onAudioSegment,
onSpeechStart: config.onSpeechStart,
onSpeechEnd: config.onSpeechEnd,
onError: config.onError,
};
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd, config.onError]);
const segmenterRef = useRef<SpeechSegmenter | null>(null);
if (segmenterRef.current === null) {
segmenterRef.current = new SpeechSegmenter(
{
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
},
{
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
onDetectingChange: (next) => setIsDetecting(next),
onSpeakingChange: (next) => setIsSpeaking(next),
}
);
}
useEffect(() => {
segmenterRef.current?.updateConfig({
enableContinuousStreaming,
volumeThreshold: config.volumeThreshold,
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
silenceDurationMs: config.silenceDuration,
speechConfirmationMs: config.speechConfirmationDuration,
detectionGracePeriodMs: config.detectionGracePeriod,
});
}, [
enableContinuousStreaming,
config.volumeThreshold,
config.confirmedDropGracePeriod,
config.silenceDuration,
config.speechConfirmationDuration,
config.detectionGracePeriod,
]);
const refs = useRef<{
started: boolean;
stream: MediaStream | null;
context: AudioContext | null;
source: MediaStreamAudioSourceNode | null;
processor: ScriptProcessorNode | null;
gain: GainNode | null;
}>({
started: false,
stream: null,
context: null,
source: null,
processor: null,
gain: null,
});
const isMutedRef = useRef(isMuted);
useEffect(() => {
isMutedRef.current = isMuted;
}, [isMuted]);
const vadConfigRef = useRef({
volumeThreshold: config.volumeThreshold,
});
useEffect(() => {
vadConfigRef.current = {
volumeThreshold: config.volumeThreshold,
};
}, [config.volumeThreshold]);
const vadLogRef = useRef({
lastLogMs: 0,
lastSpeaking: false,
lastDetecting: false,
});
// Update segment duration timer
useEffect(() => {
if (!isDetecting && !isSpeaking) {
setSegmentDuration(0);
return;
}
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
const interval = setInterval(() => {
setSegmentDuration(Date.now() - startTime);
}, 100);
return () => clearInterval(interval);
}, [isDetecting, isSpeaking]);
const stopInternal = useCallback(async () => {
refs.current.started = false;
try {
refs.current.processor?.disconnect();
refs.current.source?.disconnect();
refs.current.gain?.disconnect();
} catch {
// best-effort teardown
}
if (refs.current.stream) {
for (const track of refs.current.stream.getTracks()) {
try {
track.stop();
} catch {
// ignore
}
}
}
const context = refs.current.context;
if (context && context.state !== "closed") {
try {
await context.close();
} catch {
// ignore
}
}
refs.current.stream = null;
refs.current.context = null;
refs.current.source = null;
refs.current.processor = null;
refs.current.gain = null;
segmenterRef.current?.stop(Date.now());
setIsActive(false);
setIsSpeaking(false);
setIsDetecting(false);
setVolume(0);
setIsMuted(false);
}, []);
const start = useCallback(async () => {
if (refs.current.started) {
return;
}
const missingNavigator =
typeof navigator === "undefined" ||
!navigator.mediaDevices ||
typeof navigator.mediaDevices.getUserMedia !== "function";
const secureContext =
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
? window.isSecureContext
: true;
const currentOrigin = typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
const isTauri = getTauri() !== null;
try {
if (missingNavigator) {
throw new Error("Microphone capture is not supported in this environment");
}
console.log("[Voice][Web] Microphone preflight", {
secureContext,
currentOrigin,
isTauri,
hasMediaDevices:
typeof navigator !== "undefined" &&
!!navigator.mediaDevices &&
typeof navigator.mediaDevices.getUserMedia === "function",
});
if (!secureContext && !isTauri) {
throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
}
if (!secureContext && isTauri) {
console.warn(
"[Voice][Web] Insecure context reported under Tauri; attempting getUserMedia anyway",
{ currentOrigin }
);
}
const AudioContextCtor = getAudioContextCtor();
if (!AudioContextCtor) {
throw new Error("AudioContext unavailable");
}
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
noiseSuppression: true,
echoCancellation: true,
autoGainControl: true,
},
});
const context = new AudioContextCtor();
if (context.state === "suspended") {
await context.resume();
}
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(4096, 1, 1);
const gain = context.createGain();
gain.gain.value = 0;
refs.current = {
started: true,
stream,
context,
source,
processor,
gain,
};
processor.onaudioprocess = (event) => {
if (!refs.current.started) {
return;
}
const input = event.inputBuffer.getChannelData(0);
let sumSquares = 0;
for (let i = 0; i < input.length; i++) {
const sample = input[i];
sumSquares += sample * sample;
}
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
const normalized = Math.min(1, Math.max(0, rms * 2));
setVolume(normalized);
if (isMutedRef.current) {
return;
}
const nowMs = Date.now();
segmenterRef.current?.pushVolumeLevel(normalized, nowMs);
const segmenter = segmenterRef.current;
if (segmenter) {
const speakingNow = segmenter.getIsSpeaking();
const detectingNow = segmenter.getIsDetecting();
const shouldLog =
nowMs - vadLogRef.current.lastLogMs >= 150 ||
speakingNow !== vadLogRef.current.lastSpeaking ||
detectingNow !== vadLogRef.current.lastDetecting;
if (shouldLog) {
const threshold = vadConfigRef.current.volumeThreshold;
const releaseThreshold = segmenter.getVolumeReleaseThreshold();
console.log("[Voice][Web][VAD] level", {
volume: Number(normalized.toFixed(3)),
threshold: Number(threshold.toFixed(3)),
releaseThreshold: Number(releaseThreshold.toFixed(3)),
confirmedDropGraceMs: config.confirmedDropGracePeriod ?? 250,
isSpeaking: speakingNow,
isDetecting: detectingNow,
});
vadLogRef.current.lastLogMs = nowMs;
vadLogRef.current.lastSpeaking = speakingNow;
vadLogRef.current.lastDetecting = detectingNow;
}
}
const pcm16 = resampleToPcm16(input, context.sampleRate, 16000);
const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength);
segmenterRef.current?.pushPcmChunk(pcmBytes);
};
source.connect(processor);
processor.connect(gain);
gain.connect(context.destination);
setIsActive(true);
return;
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
callbacksRef.current.onError?.(err);
await stopInternal();
throw err;
}
}, [stopInternal]);
const stop = useCallback(async () => {
await stopInternal();
}, [stopInternal]);
const toggleMute = useCallback(() => {
setIsMuted((prev) => {
const nextMuted = !prev;
if (nextMuted) {
segmenterRef.current?.reset();
setIsSpeaking(false);
setIsDetecting(false);
}
return nextMuted;
});
}, []);
useEffect(() => {
return () => {
if (refs.current.started) {
void stopInternal();
}
};
}, [stopInternal]);
return useMemo(
() => ({
start,
stop,
toggleMute,
isActive,
isSpeaking,
isDetecting,
isMuted,
volume,
segmentDuration,
}),
[start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration]
);
}

9
packages/app/src/runtime/host-runtime.test.ts
View File

@@ -922,8 +922,7 @@ describe("HostRuntimeStore", () => {
useSessionStore.getState().initializeSession(
host.serverId,
fakeClient as unknown as DaemonClient,
null as any
fakeClient as unknown as DaemonClient
);
store.syncHosts([host]);
@@ -1046,8 +1045,7 @@ describe("HostRuntimeStore", () => {
useSessionStore.getState().initializeSession(
host.serverId,
fakeClient as unknown as DaemonClient,
null as any
fakeClient as unknown as DaemonClient
);
store.syncHosts([host]);
@@ -1122,8 +1120,7 @@ describe("HostRuntimeStore", () => {
useSessionStore.getState().initializeSession(
host.serverId,
fakeClient as unknown as DaemonClient,
null as any
fakeClient as unknown as DaemonClient
);
store.syncHosts([host]);

9
packages/app/src/runtime/host-runtime.ts
View File

@@ -1822,6 +1822,15 @@ export function useHostRuntimeSession(serverId: string): {
};
}
export function useHostRuntimeIsConnected(serverId: string): boolean {
const store = getHostRuntimeStore();
return useSyncExternalStore(
(onStoreChange) => store.subscribe(serverId, onStoreChange),
() => isHostRuntimeConnected(store.getSnapshot(serverId)),
() => isHostRuntimeConnected(store.getSnapshot(serverId))
);
}
export function useHosts(): HostProfile[] {
const store = getHostRuntimeStore();
return useSyncExternalStore(

10
packages/app/src/screens/agent/agent-ready-screen.tsx
View File

@@ -69,11 +69,11 @@ function logAgentExplorer(event: string, details: Record<string, unknown>): void
console.log(`[AgentExplorer] ${event}`, details);
}
function logWebStickyBottom(event: string, details: Record<string, unknown>): void {
if (!IS_DEV || Platform.OS !== "web") {
return;
}
console.log("[WebStickyBottom]", event, details);
function logWebStickyBottom(
_event: string,
_details: Record<string, unknown>
): void {
// Intentionally disabled: this path is too noisy during voice debugging.
}
export function AgentReadyScreen({

217
packages/app/src/screens/workspace/workspace-screen.tsx
View File

@@ -1,4 +1,4 @@
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { memo, useCallback, useEffect, useMemo, useRef, useState } from "react";
import {
ActivityIndicator,
BackHandler,
@@ -25,7 +25,7 @@ import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyl
import { SidebarMenuToggle } from "@/components/headers/menu-header";
import { HeaderToggleButton } from "@/components/headers/header-toggle-button";
import { ScreenHeader } from "@/components/headers/screen-header";
import { Combobox } from "@/components/ui/combobox";
import { Combobox, type ComboboxOption } from "@/components/ui/combobox";
import { Shortcut } from "@/components/ui/shortcut";
import {
DropdownMenu,
@@ -80,6 +80,7 @@ import { WorkspaceDraftAgentTab } from "@/screens/workspace/workspace-draft-agen
import { WorkspaceDesktopTabsRow } from "@/screens/workspace/workspace-desktop-tabs-row";
import {
deriveWorkspaceTabPresentation,
type WorkspaceTabPresentation,
WorkspaceTabIcon,
WorkspaceTabOptionRow,
} from "@/screens/workspace/workspace-tab-presentation";
@@ -141,6 +142,120 @@ function buildOpenIntentKey(input: {
return `${input.serverId}:${input.workspaceId}:${openParam}`;
}
type MobileWorkspaceTabSwitcherProps = {
activeTabKey: string;
activeTabLabel: string;
activeTabPresentation: WorkspaceTabPresentation | null;
tabSwitcherOptions: ComboboxOption[];
tabByKey: Map<string, WorkspaceTabDescriptor>;
tabPresentationsByKey: Map<string, WorkspaceTabPresentation>;
onSelectSwitcherTab: (key: string) => void;
onSelectNewTabOption: (key: typeof NEW_TAB_AGENT_OPTION_ID) => void;
};
const MobileWorkspaceTabSwitcher = memo(function MobileWorkspaceTabSwitcher({
activeTabKey,
activeTabLabel,
activeTabPresentation,
tabSwitcherOptions,
tabByKey,
tabPresentationsByKey,
onSelectSwitcherTab,
onSelectNewTabOption,
}: MobileWorkspaceTabSwitcherProps) {
const { theme } = useUnistyles();
const [isOpen, setIsOpen] = useState(false);
const anchorRef = useRef<View>(null);
return (
<View style={styles.mobileTabsRow} testID="workspace-tabs-row">
<Pressable
ref={anchorRef}
style={({ hovered, pressed }) => [
styles.switcherTrigger,
(hovered || pressed || isOpen) && styles.switcherTriggerActive,
{ borderWidth: 0, borderColor: "transparent" },
Platform.OS === "web"
? {
outlineStyle: "solid",
outlineWidth: 0,
outlineColor: "transparent",
}
: null,
]}
onPress={() => setIsOpen(true)}
>
<View style={styles.switcherTriggerLeft}>
<View style={styles.switcherTriggerIcon} testID="workspace-active-tab-icon">
{activeTabPresentation ? (
<WorkspaceTabIcon presentation={activeTabPresentation} active />
) : null}
</View>
<Text style={styles.switcherTriggerText} numberOfLines={1}>
{activeTabLabel}
</Text>
</View>
<ChevronDown size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
</Pressable>
<View style={styles.mobileTabsActions}>
<Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
<TooltipTrigger
testID="workspace-new-agent-tab"
onPress={() => onSelectNewTabOption(NEW_TAB_AGENT_OPTION_ID)}
accessibilityRole="button"
accessibilityLabel="New agent tab"
style={({ hovered, pressed }) => [
styles.newTabActionButton,
(hovered || pressed) && styles.newTabActionButtonHovered,
]}
>
<Plus size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
</TooltipTrigger>
<TooltipContent side="bottom" align="end" offset={8}>
<View style={styles.newTabTooltipRow}>
<Text style={styles.newTabTooltipText}>New agent tab</Text>
<Shortcut keys={["mod", "T"]} style={styles.newTabTooltipShortcut} />
</View>
</TooltipContent>
</Tooltip>
</View>
<Combobox
options={tabSwitcherOptions}
value={activeTabKey}
onSelect={onSelectSwitcherTab}
searchable={false}
title="Switch tab"
searchPlaceholder="Search tabs"
open={isOpen}
onOpenChange={setIsOpen}
enableDismissOnClose={false}
anchorRef={anchorRef}
renderOption={({ option, selected, active, onPress }) => {
const tab = tabByKey.get(option.id);
if (!tab) {
return <View />;
}
const presentation =
tabPresentationsByKey.get(option.id) ??
deriveWorkspaceTabPresentation({ tab });
return (
<WorkspaceTabOptionRow
presentation={presentation}
selected={selected}
active={active}
onPress={onPress}
/>
);
}}
/>
</View>
);
});
export function WorkspaceScreen({
serverId,
workspaceId,
@@ -767,12 +882,10 @@ function WorkspaceScreenContent({
[handleOpenFileFromExplorer]
);
const [isTabSwitcherOpen, setIsTabSwitcherOpen] = useState(false);
const [hoveredTabKey, setHoveredTabKey] = useState<string | null>(null);
const [hoveredCloseTabKey, setHoveredCloseTabKey] = useState<string | null>(
null
);
const tabSwitcherAnchorRef = useRef<View>(null);
const tabByKey = useMemo(() => {
const map = new Map<string, WorkspaceTabDescriptor>();
@@ -837,7 +950,6 @@ function WorkspaceScreenContent({
const handleSelectSwitcherTab = useCallback(
(key: string) => {
setIsTabSwitcherOpen(false);
navigateToTabId(key);
},
[navigateToTabId]
@@ -1522,91 +1634,16 @@ function WorkspaceScreenContent({
/>
{isMobile ? (
<View style={styles.mobileTabsRow} testID="workspace-tabs-row">
<Pressable
ref={tabSwitcherAnchorRef}
style={({ hovered, pressed }) => [
styles.switcherTrigger,
(hovered || pressed || isTabSwitcherOpen) && styles.switcherTriggerActive,
{ borderWidth: 0, borderColor: "transparent" },
Platform.OS === "web"
? {
outlineStyle: "solid",
outlineWidth: 0,
outlineColor: "transparent",
}
: null,
]}
onPress={() => setIsTabSwitcherOpen(true)}
>
<View style={styles.switcherTriggerLeft}>
<View style={styles.switcherTriggerIcon} testID="workspace-active-tab-icon">
{activeTabPresentation ? (
<WorkspaceTabIcon presentation={activeTabPresentation} active />
) : null}
</View>
<Text style={styles.switcherTriggerText} numberOfLines={1}>
{activeTabLabel}
</Text>
</View>
<ChevronDown size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
</Pressable>
<View style={styles.mobileTabsActions}>
<Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
<TooltipTrigger
testID="workspace-new-agent-tab"
onPress={() => handleSelectNewTabOption(NEW_TAB_AGENT_OPTION_ID)}
accessibilityRole="button"
accessibilityLabel="New agent tab"
style={({ hovered, pressed }) => [
styles.newTabActionButton,
(hovered || pressed) && styles.newTabActionButtonHovered,
]}
>
<Plus size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
</TooltipTrigger>
<TooltipContent side="bottom" align="end" offset={8}>
<View style={styles.newTabTooltipRow}>
<Text style={styles.newTabTooltipText}>New agent tab</Text>
<Shortcut keys={["mod", "T"]} style={styles.newTabTooltipShortcut} />
</View>
</TooltipContent>
</Tooltip>
</View>
<Combobox
options={tabSwitcherOptions}
value={activeTabKey}
onSelect={handleSelectSwitcherTab}
searchable={false}
title="Switch tab"
searchPlaceholder="Search tabs"
open={isTabSwitcherOpen}
onOpenChange={setIsTabSwitcherOpen}
anchorRef={tabSwitcherAnchorRef}
renderOption={({ option, selected, active, onPress }) => {
const tab = tabByKey.get(option.id);
if (!tab) {
return <View />;
}
const presentation =
tabPresentationsByKey.get(option.id) ??
deriveWorkspaceTabPresentation({ tab });
return (
<WorkspaceTabOptionRow
presentation={presentation}
selected={selected}
active={active}
onPress={onPress}
/>
);
}}
/>
</View>
<MobileWorkspaceTabSwitcher
activeTabKey={activeTabKey}
activeTabLabel={activeTabLabel}
activeTabPresentation={activeTabPresentation}
tabSwitcherOptions={tabSwitcherOptions}
tabByKey={tabByKey}
tabPresentationsByKey={tabPresentationsByKey}
onSelectSwitcherTab={handleSelectSwitcherTab}
onSelectNewTabOption={handleSelectNewTabOption}
/>
) : (
<WorkspaceDesktopTabsRow
tabs={tabs}

16
packages/app/src/stores/session-store.ts
View File

@@ -1,7 +1,6 @@
import { create } from "zustand";
import { subscribeWithSelector } from "zustand/middleware";
import type { DaemonClient } from "@server/client/daemon-client";
import type { useAudioPlayer } from "@/hooks/use-audio-player";
import type { AgentDirectoryEntry } from "@/types/agent-directory";
import type { StreamItem } from "@/types/stream";
import type { PendingPermission } from "@/types/shared";
@@ -208,9 +207,6 @@ export interface SessionState {
// Server metadata (from server_info handshake)
serverInfo: DaemonServerInfo | null;
// Audio player (immutable reference)
audioPlayer: ReturnType<typeof useAudioPlayer> | null;
// Hydration status
hasHydratedAgents: boolean;
hasHydratedWorkspaces: boolean;
@@ -264,7 +260,7 @@ interface SessionStoreState {
// Action types
interface SessionStoreActions {
// Session management
initializeSession: (serverId: string, client: DaemonClient, audioPlayer: ReturnType<typeof useAudioPlayer>) => void;
initializeSession: (serverId: string, client: DaemonClient) => void;
clearSession: (serverId: string) => void;
getSession: (serverId: string) => SessionState | undefined;
updateSessionClient: (serverId: string, client: DaemonClient) => void;
@@ -359,12 +355,14 @@ type SessionStore = SessionStoreState & SessionStoreActions;
const agentLastActivityCoalescer = createAgentLastActivityCoalescer();
// Helper to create initial session state
function createInitialSessionState(serverId: string, client: DaemonClient, audioPlayer: ReturnType<typeof useAudioPlayer>): SessionState {
function createInitialSessionState(
serverId: string,
client: DaemonClient
): SessionState {
return {
serverId,
client,
serverInfo: null,
audioPlayer,
hasHydratedAgents: false,
hasHydratedWorkspaces: false,
isPlayingAudio: false,
@@ -424,7 +422,7 @@ export const useSessionStore = create<SessionStore>()(
agentLastActivity: new Map(),
// Session management
initializeSession: (serverId, client, audioPlayer) => {
initializeSession: (serverId, client) => {
set((prev) => {
if (prev.sessions[serverId]) {
return prev;
@@ -433,7 +431,7 @@ export const useSessionStore = create<SessionStore>()(
...prev,
sessions: {
...prev.sessions,
[serverId]: createInitialSessionState(serverId, client, audioPlayer),
[serverId]: createInitialSessionState(serverId, client),
},
};
});

88
packages/app/src/utils/pcm16-wav.ts Normal file
View File

@@ -0,0 +1,88 @@
export interface Pcm16Wav {
sampleRate: number;
samples: Int16Array;
}
export function parsePcm16Wav(buffer: ArrayBuffer): Pcm16Wav | null {
if (buffer.byteLength < 44) {
return null;
}
const view = new DataView(buffer);
function readAscii(offset: number, length: number): string {
let out = "";
for (let i = 0; i < length; i += 1) {
out += String.fromCharCode(view.getUint8(offset + i));
}
return out;
}
if (readAscii(0, 4) !== "RIFF" || readAscii(8, 4) !== "WAVE") {
return null;
}
let offset = 12;
let channels = 0;
let sampleRate = 0;
let bitsPerSample = 0;
let dataOffset = 0;
let dataSize = 0;
while (offset + 8 <= buffer.byteLength) {
const chunkId = readAscii(offset, 4);
const chunkSize = view.getUint32(offset + 4, true);
const chunkDataOffset = offset + 8;
if (chunkDataOffset + chunkSize > buffer.byteLength) {
return null;
}
if (chunkId === "fmt " && chunkSize >= 16) {
const audioFormat = view.getUint16(chunkDataOffset, true);
channels = view.getUint16(chunkDataOffset + 2, true);
sampleRate = view.getUint32(chunkDataOffset + 4, true);
bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
if (audioFormat !== 1) {
return null;
}
}
if (chunkId === "data") {
dataOffset = chunkDataOffset;
dataSize = chunkSize;
break;
}
offset = chunkDataOffset + chunkSize + (chunkSize % 2);
}
if (!dataOffset || dataSize <= 0 || sampleRate <= 0 || bitsPerSample !== 16 || channels <= 0) {
return null;
}
const sampleCount = Math.floor(dataSize / 2);
const interleaved = new Int16Array(buffer, dataOffset, sampleCount);
if (channels === 1) {
return {
sampleRate,
samples: new Int16Array(interleaved),
};
}
const frameCount = Math.floor(interleaved.length / channels);
const mono = new Int16Array(frameCount);
for (let frame = 0; frame < frameCount; frame += 1) {
let sum = 0;
for (let channel = 0; channel < channels; channel += 1) {
sum += interleaved[frame * channels + channel] ?? 0;
}
mono[frame] = Math.round(sum / channels);
}
return {
sampleRate,
samples: mono,
};
}

2
packages/app/src/utils/thinking-tone.native-pcm.ts Normal file
View File

File diff suppressed because one or more lines are too long

76
packages/app/src/utils/thinking-tone.test.ts Normal file
View File

@@ -0,0 +1,76 @@
import { describe, expect, it } from "vitest";
import { parsePcm16Wav } from "@/utils/pcm16-wav";
interface BuildWavInput {
channels?: number;
sampleRate?: number;
samples: number[];
}
function buildPcm16Wav(input: BuildWavInput): ArrayBuffer {
const channels = input.channels ?? 1;
const sampleRate = input.sampleRate ?? 24000;
const bytesPerSample = 2;
const dataSize = input.samples.length * bytesPerSample;
const buffer = new ArrayBuffer(44 + dataSize);
const view = new DataView(buffer);
function writeAscii(offset: number, value: string): void {
for (let i = 0; i < value.length; i += 1) {
view.setUint8(offset + i, value.charCodeAt(i));
}
}
writeAscii(0, "RIFF");
view.setUint32(4, 36 + dataSize, true);
writeAscii(8, "WAVE");
writeAscii(12, "fmt ");
view.setUint32(16, 16, true);
view.setUint16(20, 1, true);
view.setUint16(22, channels, true);
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * channels * bytesPerSample, true);
view.setUint16(32, channels * bytesPerSample, true);
view.setUint16(34, 16, true);
writeAscii(36, "data");
view.setUint32(40, dataSize, true);
input.samples.forEach((sample, index) => {
view.setInt16(44 + index * bytesPerSample, sample, true);
});
return buffer;
}
describe("parsePcm16Wav", () => {
it("parses mono PCM16 wav data", () => {
const buffer = buildPcm16Wav({
sampleRate: 16000,
samples: [100, -200, 300],
});
const parsed = parsePcm16Wav(buffer);
expect(parsed).not.toBeNull();
expect(parsed?.sampleRate).toBe(16000);
expect(Array.from(parsed?.samples ?? [])).toEqual([100, -200, 300]);
});
it("downmixes multichannel PCM16 wav data to mono", () => {
const buffer = buildPcm16Wav({
channels: 2,
samples: [1000, -1000, 3000, 1000],
});
const parsed = parsePcm16Wav(buffer);
expect(parsed).not.toBeNull();
expect(Array.from(parsed?.samples ?? [])).toEqual([0, 2000]);
});
it("rejects non-wav payloads", () => {
const buffer = new TextEncoder().encode("not-a-wav").buffer;
expect(parsePcm16Wav(buffer)).toBeNull();
});
});

33
packages/app/src/utils/thinking-tone.ts Normal file
View File

@@ -0,0 +1,33 @@
import { Asset } from "expo-asset";
import { File } from "expo-file-system";
import { Platform } from "react-native";
export { parsePcm16Wav, type Pcm16Wav } from "@/utils/pcm16-wav";
export const THINKING_TONE_REPEAT_GAP_MS = 350;
let thinkingToneArrayBufferPromise: Promise<ArrayBuffer> | null = null;
async function readThinkingToneArrayBuffer(): Promise<ArrayBuffer> {
const toneModule = require("../../assets/audio/thinking-tone.wav");
const asset = Asset.fromModule(toneModule);
if (Platform.OS === "web") {
const response = await fetch(asset.uri);
if (!response.ok) {
throw new Error(`Failed to fetch thinking tone asset: ${response.status}`);
}
return await response.arrayBuffer();
}
const resolvedAsset = asset.localUri ? asset : await asset.downloadAsync();
const fileUri = resolvedAsset.localUri ?? resolvedAsset.uri;
const file = new File(fileUri);
return await file.arrayBuffer();
}
export async function loadThinkingToneArrayBuffer(): Promise<ArrayBuffer> {
if (!thinkingToneArrayBufferPromise) {
thinkingToneArrayBufferPromise = readThinkingToneArrayBuffer();
}
return await thinkingToneArrayBufferPromise;
}

34
packages/app/src/utils/tool-call-display.test.ts
View File

@@ -140,4 +140,38 @@ describe("tool-call-display", () => {
expect(display.errorText).toBe('{\n "message": "boom"\n}');
});
it("shows terminal interaction with only the fixed label when no command is available", () => {
const display = buildToolCallDisplayModel({
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
icon: "square_terminal",
},
});
expect(display).toEqual({
displayName: "Interacted with terminal",
});
});
it("shows terminal interaction command as the summary when available", () => {
const display = buildToolCallDisplayModel({
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
label: "npm run test",
icon: "square_terminal",
},
});
expect(display).toEqual({
displayName: "Interacted with terminal",
summary: "npm run test",
});
});
});

29
packages/app/src/voice/audio-engine-types.ts Normal file
View File

@@ -0,0 +1,29 @@
export interface AudioEngineCallbacks {
onCaptureData(pcm: Uint8Array): void;
onVolumeLevel(level: number): void;
onError?(error: Error): void;
}
export interface AudioPlaybackSource {
arrayBuffer(): Promise<ArrayBuffer>;
size: number;
type: string;
}
export interface AudioEngine {
initialize(): Promise<void>;
destroy(): Promise<void>;
startCapture(): Promise<void>;
stopCapture(): Promise<void>;
toggleMute(): boolean;
isMuted(): boolean;
play(audio: AudioPlaybackSource): Promise<number>;
stop(): void;
clearQueue(): void;
isPlaying(): boolean;
playLooping(pcm: Uint8Array, gapMs: number): void;
stopLooping(): void;
}

1
packages/app/src/voice/audio-engine.d.ts vendored Normal file
View File

@@ -0,0 +1 @@
export * from "./audio-engine.native";

640
packages/app/src/voice/audio-engine.native.ts Normal file
View File

@@ -0,0 +1,640 @@
import {
addExpoTwoWayAudioEventListener,
getMicrophonePermissionsAsync,
initialize,
playPCMData,
requestMicrophonePermissionsAsync,
resumePlayback,
stopPlayback,
tearDown,
toggleRecording,
} from "@boudra/expo-two-way-audio";
import {
THINKING_TONE_REPEAT_GAP_MS,
} from "@/utils/thinking-tone";
import {
THINKING_TONE_NATIVE_PCM_BASE64,
THINKING_TONE_NATIVE_PCM_DURATION_MS,
} from "@/utils/thinking-tone.native-pcm";
import type {
AudioEngine,
AudioEngineCallbacks,
AudioPlaybackSource,
} from "@/voice/audio-engine-types";
import { Buffer } from "buffer";
interface QueuedAudio {
audio: AudioPlaybackSource;
resolve: (duration: number) => void;
reject: (error: Error) => void;
}
interface CuePcm {
pcm16k: Uint8Array;
durationMs: number;
}
interface StreamDebugState {
count: number;
totalValue: number;
maxValue: number;
startedAtMs: number;
lastAtMs: number;
maxGapMs: number;
nextLogAtMs: number;
}
interface AudioEngineTraceOptions {
traceLabel?: string;
}
let nextAudioEngineInstanceId = 1;
function getTraceStack(): string | undefined {
const stack = new Error().stack;
if (!stack) {
return undefined;
}
return stack
.split("\n")
.slice(2, 7)
.map((line) => line.trim())
.join(" | ");
}
function resamplePcm16(
pcm: Uint8Array,
fromRate: number,
toRate: number
): Uint8Array {
if (fromRate === toRate) {
return pcm;
}
const inputSamples = Math.floor(pcm.length / 2);
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
const output = new Uint8Array(outputSamples * 2);
const ratio = fromRate / toRate;
const readInt16 = (sampleIndex: number): number => {
const offset = sampleIndex * 2;
if (offset + 1 >= pcm.length) {
return 0;
}
const lo = pcm[offset]!;
const hi = pcm[offset + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value -= 0x10000;
}
return value;
};
const writeInt16 = (sampleIndex: number, value: number): void => {
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
const offset = sampleIndex * 2;
output[offset] = clamped & 0xff;
output[offset + 1] = (clamped >> 8) & 0xff;
};
for (let i = 0; i < outputSamples; i += 1) {
const sourceIndex = i * ratio;
const i0 = Math.floor(sourceIndex);
const frac = sourceIndex - i0;
const s0 = readInt16(i0);
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
writeInt16(i, s0 + (s1 - s0) * frac);
}
return output;
}
function parsePcmSampleRate(mimeType: string): number | null {
const match = /rate=(\d+)/i.exec(mimeType);
if (!match) {
return null;
}
const rate = Number(match[1]);
return Number.isFinite(rate) && rate > 0 ? rate : null;
}
function logVoiceNative(event: string, details?: Record<string, unknown>): void {
if (details) {
console.log(`[VoiceNative] ${event}`, details);
return;
}
console.log(`[VoiceNative] ${event}`);
}
function createStreamDebugState(nowMs: number): StreamDebugState {
return {
count: 0,
totalValue: 0,
maxValue: 0,
startedAtMs: nowMs,
lastAtMs: nowMs,
maxGapMs: 0,
nextLogAtMs: nowMs + 1000,
};
}
export function createAudioEngine(
callbacks: AudioEngineCallbacks,
options?: AudioEngineTraceOptions
): AudioEngine {
const engineId = nextAudioEngineInstanceId++;
const traceLabel = options?.traceLabel ?? "unknown";
const traceStack = getTraceStack();
const refs: {
initialized: boolean;
captureActive: boolean;
muted: boolean;
queue: QueuedAudio[];
processingQueue: boolean;
playbackTimeout: ReturnType<typeof setTimeout> | null;
activePlayback: {
resolve: (duration: number) => void;
reject: (error: Error) => void;
settled: boolean;
} | null;
looping: {
active: boolean;
token: number;
timeout: ReturnType<typeof setTimeout> | null;
};
thinkingTone: CuePcm | null;
captureDebug: StreamDebugState | null;
volumeDebug: StreamDebugState | null;
destroyed: boolean;
} = {
initialized: false,
captureActive: false,
muted: false,
queue: [],
processingQueue: false,
playbackTimeout: null,
activePlayback: null,
looping: {
active: false,
token: 0,
timeout: null,
},
thinkingTone: null,
captureDebug: null,
volumeDebug: null,
destroyed: false,
};
logVoiceNative("engine_created", {
engineId,
traceLabel,
stack: traceStack,
});
function resetCaptureDebug(): void {
refs.captureDebug = null;
}
function resetVolumeDebug(): void {
refs.volumeDebug = null;
}
function recordCaptureChunk(byteLength: number): void {
const nowMs = Date.now();
const debug = refs.captureDebug ?? createStreamDebugState(nowMs);
const gapMs = debug.count === 0 ? 0 : nowMs - debug.lastAtMs;
debug.count += 1;
debug.totalValue += byteLength;
debug.maxValue = Math.max(debug.maxValue, byteLength);
debug.maxGapMs = Math.max(debug.maxGapMs, gapMs);
debug.lastAtMs = nowMs;
if (nowMs >= debug.nextLogAtMs) {
const elapsedMs = Math.max(1, nowMs - debug.startedAtMs);
logVoiceNative("capture_summary", {
chunks: debug.count,
totalBytes: debug.totalValue,
averageChunkBytes: Math.round(debug.totalValue / debug.count),
maxChunkBytes: debug.maxValue,
chunksPerSecond: Number(((debug.count * 1000) / elapsedMs).toFixed(1)),
maxGapMs: debug.maxGapMs,
muted: refs.muted,
});
refs.captureDebug = createStreamDebugState(nowMs);
return;
}
refs.captureDebug = debug;
}
function recordVolumeLevel(level: number): void {
const nowMs = Date.now();
const debug = refs.volumeDebug ?? createStreamDebugState(nowMs);
const gapMs = debug.count === 0 ? 0 : nowMs - debug.lastAtMs;
debug.count += 1;
debug.totalValue += level;
debug.maxValue = Math.max(debug.maxValue, level);
debug.maxGapMs = Math.max(debug.maxGapMs, gapMs);
debug.lastAtMs = nowMs;
if (nowMs >= debug.nextLogAtMs) {
const elapsedMs = Math.max(1, nowMs - debug.startedAtMs);
logVoiceNative("volume_summary", {
samples: debug.count,
averageLevel: Number((debug.totalValue / debug.count).toFixed(3)),
peakLevel: Number(debug.maxValue.toFixed(3)),
samplesPerSecond: Number(((debug.count * 1000) / elapsedMs).toFixed(1)),
maxGapMs: debug.maxGapMs,
muted: refs.muted,
});
refs.volumeDebug = createStreamDebugState(nowMs);
return;
}
refs.volumeDebug = debug;
}
const microphoneSubscription = addExpoTwoWayAudioEventListener(
"onMicrophoneData",
(event) => {
console.log("[VoiceNative] onMicrophoneData", {
engineId,
traceLabel,
captureActive: refs.captureActive,
muted: refs.muted,
bytes: event.data.byteLength,
});
if (!refs.captureActive || refs.muted) {
return;
}
recordCaptureChunk(event.data.byteLength);
callbacks.onCaptureData(event.data);
}
);
logVoiceNative("listener_added", {
engineId,
traceLabel,
event: "onMicrophoneData",
});
const volumeSubscription = addExpoTwoWayAudioEventListener(
"onInputVolumeLevelData",
(event) => {
console.log("[VoiceNative] onInputVolumeLevelData", {
engineId,
traceLabel,
captureActive: refs.captureActive,
muted: refs.muted,
level: event.data,
});
if (!refs.captureActive) {
return;
}
const level = refs.muted ? 0 : event.data;
recordVolumeLevel(level);
callbacks.onVolumeLevel(level);
}
);
logVoiceNative("listener_added", {
engineId,
traceLabel,
event: "onInputVolumeLevelData",
});
async function ensureInitialized(): Promise<void> {
if (refs.initialized) {
return;
}
logVoiceNative("initialize_start");
await initialize();
refs.initialized = true;
logVoiceNative("initialize_complete");
}
async function ensureMicrophonePermission(): Promise<void> {
let permission = await getMicrophonePermissionsAsync().catch(() => null);
if (!permission?.granted) {
permission = await requestMicrophonePermissionsAsync().catch(() => null);
}
if (!permission?.granted) {
throw new Error(
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
);
}
}
async function ensureThinkingTone(): Promise<CuePcm> {
if (refs.thinkingTone) {
return refs.thinkingTone;
}
const pcm16k = Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64");
const durationMs = THINKING_TONE_NATIVE_PCM_DURATION_MS;
refs.thinkingTone = { pcm16k, durationMs };
logVoiceNative("thinking_tone_loaded", {
bytes: pcm16k.byteLength,
durationMs,
});
return refs.thinkingTone;
}
function clearPlaybackTimeout(): void {
if (refs.playbackTimeout) {
clearTimeout(refs.playbackTimeout);
refs.playbackTimeout = null;
}
}
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
await ensureInitialized();
resumePlayback();
return await new Promise<number>(async (resolve, reject) => {
refs.activePlayback = { resolve, reject, settled: false };
try {
const arrayBuffer = await audio.arrayBuffer();
const pcm = new Uint8Array(arrayBuffer);
const inputRate = parsePcmSampleRate(audio.type || "") ?? 24000;
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
const durationSec = pcm16k.length / 2 / 16000;
logVoiceNative("playback_start", {
inputBytes: pcm.byteLength,
outputBytes: pcm16k.byteLength,
inputRate,
durationMs: Math.round(durationSec * 1000),
queueLength: refs.queue.length,
});
playPCMData(pcm16k);
clearPlaybackTimeout();
refs.playbackTimeout = setTimeout(() => {
clearPlaybackTimeout();
const active = refs.activePlayback;
if (!active || active.settled) {
return;
}
active.settled = true;
refs.activePlayback = null;
logVoiceNative("playback_complete", {
durationMs: Math.round(durationSec * 1000),
});
resolve(durationSec);
}, durationSec * 1000);
} catch (error) {
clearPlaybackTimeout();
const active = refs.activePlayback;
if (active && !active.settled) {
active.settled = true;
refs.activePlayback = null;
logVoiceNative("playback_failed", {
error: error instanceof Error ? error.message : String(error),
});
reject(error instanceof Error ? error : new Error(String(error)));
}
}
});
}
async function processQueue(): Promise<void> {
if (refs.processingQueue || refs.queue.length === 0) {
return;
}
refs.processingQueue = true;
logVoiceNative("queue_drain_start", {
queueLength: refs.queue.length,
});
while (refs.queue.length > 0) {
const item = refs.queue.shift()!;
try {
const duration = await playAudio(item.audio);
item.resolve(duration);
} catch (error) {
item.reject(error instanceof Error ? error : new Error(String(error)));
}
}
refs.processingQueue = false;
logVoiceNative("queue_drain_complete");
}
function stopLooping(): void {
if (refs.looping.active) {
logVoiceNative("thinking_tone_stop");
}
refs.looping.active = false;
refs.looping.token += 1;
if (refs.looping.timeout) {
clearTimeout(refs.looping.timeout);
refs.looping.timeout = null;
}
stopPlayback();
}
return {
async initialize() {
await ensureInitialized();
await ensureThinkingTone();
},
async destroy() {
if (refs.destroyed) {
logVoiceNative("engine_destroy_skipped", {
engineId,
traceLabel,
});
return;
}
refs.destroyed = true;
logVoiceNative("engine_destroy_start", {
engineId,
traceLabel,
});
stopLooping();
this.stop();
this.clearQueue();
if (refs.captureActive) {
toggleRecording(false);
refs.captureActive = false;
}
clearPlaybackTimeout();
refs.muted = false;
resetCaptureDebug();
resetVolumeDebug();
callbacks.onVolumeLevel(0);
if (refs.initialized) {
tearDown();
refs.initialized = false;
}
microphoneSubscription.remove();
logVoiceNative("listener_removed", {
engineId,
traceLabel,
event: "onMicrophoneData",
});
volumeSubscription.remove();
logVoiceNative("listener_removed", {
engineId,
traceLabel,
event: "onInputVolumeLevelData",
});
logVoiceNative("engine_destroy_complete", {
engineId,
traceLabel,
});
},
async startCapture() {
if (refs.captureActive) {
return;
}
try {
logVoiceNative("capture_start_requested");
logVoiceNative("capture_trace", {
engineId,
traceLabel,
});
await ensureMicrophonePermission();
await ensureInitialized();
toggleRecording(true);
refs.captureActive = true;
resetCaptureDebug();
resetVolumeDebug();
logVoiceNative("capture_started");
} catch (error) {
const wrapped = error instanceof Error ? error : new Error(String(error));
logVoiceNative("capture_start_failed", {
error: wrapped.message,
});
callbacks.onError?.(wrapped);
throw wrapped;
}
},
async stopCapture() {
if (refs.captureActive) {
toggleRecording(false);
}
refs.captureActive = false;
refs.muted = false;
resetCaptureDebug();
resetVolumeDebug();
logVoiceNative("capture_stopped");
logVoiceNative("capture_trace", {
engineId,
traceLabel,
});
callbacks.onVolumeLevel(0);
},
toggleMute() {
refs.muted = !refs.muted;
logVoiceNative("mute_toggled", {
muted: refs.muted,
});
if (refs.muted) {
callbacks.onVolumeLevel(0);
}
return refs.muted;
},
isMuted() {
return refs.muted;
},
async play(audio: AudioPlaybackSource) {
return await new Promise<number>((resolve, reject) => {
refs.queue.push({ audio, resolve, reject });
logVoiceNative("queue_enqueue", {
queueLength: refs.queue.length,
blobBytes: audio.size,
mimeType: audio.type || null,
});
if (!refs.processingQueue) {
void processQueue();
}
});
},
stop() {
if (refs.activePlayback) {
logVoiceNative("playback_stopped");
}
stopPlayback();
clearPlaybackTimeout();
const active = refs.activePlayback;
refs.activePlayback = null;
if (active && !active.settled) {
active.settled = true;
active.reject(new Error("Playback stopped"));
}
},
clearQueue() {
if (refs.queue.length > 0) {
logVoiceNative("queue_cleared", {
queueLength: refs.queue.length,
});
}
while (refs.queue.length > 0) {
refs.queue.shift()!.reject(new Error("Playback stopped"));
}
refs.processingQueue = false;
},
isPlaying() {
return refs.activePlayback !== null;
},
playLooping(audio, gapMs) {
if (refs.looping.active) {
return;
}
refs.looping.active = true;
const token = refs.looping.token + 1;
refs.looping.token = token;
void (async () => {
try {
await ensureInitialized();
const cue =
audio.byteLength > 0
? {
pcm16k: audio,
durationMs: (audio.byteLength / 2 / 16000) * 1000,
}
: await ensureThinkingTone();
logVoiceNative("thinking_tone_start", {
bytes: cue.pcm16k.byteLength,
durationMs: Math.round(cue.durationMs),
gapMs: gapMs || THINKING_TONE_REPEAT_GAP_MS,
});
const loop = () => {
if (!refs.looping.active || refs.looping.token !== token) {
return;
}
resumePlayback();
playPCMData(cue.pcm16k);
logVoiceNative("thinking_tone_tick", {
durationMs: Math.round(cue.durationMs),
});
refs.looping.timeout = setTimeout(
loop,
cue.durationMs + (gapMs || THINKING_TONE_REPEAT_GAP_MS)
);
};
loop();
} catch (error) {
callbacks.onError?.(
error instanceof Error ? error : new Error(String(error))
);
}
})();
},
stopLooping,
};
}

548
packages/app/src/voice/audio-engine.web.ts Normal file
View File

@@ -0,0 +1,548 @@
import { getTauri } from "@/utils/tauri";
import {
loadThinkingToneArrayBuffer,
THINKING_TONE_REPEAT_GAP_MS,
} from "@/utils/thinking-tone";
import type {
AudioEngine,
AudioEngineCallbacks,
AudioPlaybackSource,
} from "@/voice/audio-engine-types";
interface QueuedAudio {
audio: AudioPlaybackSource;
resolve: (duration: number) => void;
reject: (error: Error) => void;
}
function getAudioContextCtor(): (typeof AudioContext) | null {
if (typeof window === "undefined") {
return null;
}
const browserWindow = window as typeof window & {
webkitAudioContext?: typeof AudioContext;
};
return browserWindow.AudioContext ?? browserWindow.webkitAudioContext ?? null;
}
function floatToInt16(sample: number): number {
const clamped = Math.max(-1, Math.min(1, sample));
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
}
function resampleToPcm16(
input: Float32Array,
inputRate: number,
outputRate: number
): Uint8Array {
if (input.length === 0) {
return new Uint8Array(0);
}
const ratio = inputRate / outputRate;
const outputLength = Math.max(1, Math.round(input.length / ratio));
const output = new Int16Array(outputLength);
for (let i = 0; i < outputLength; i += 1) {
const sourceIndex = i * ratio;
const i0 = Math.floor(sourceIndex);
const i1 = Math.min(input.length - 1, i0 + 1);
const frac = sourceIndex - i0;
const sample = input[i0]! * (1 - frac) + input[i1]! * frac;
output[i] = floatToInt16(sample);
}
return new Uint8Array(output.buffer, output.byteOffset, output.byteLength);
}
function parsePcmSampleRate(mimeType: string): number | null {
const match = /rate=(\d+)/i.exec(mimeType);
if (!match) {
return null;
}
const rate = Number(match[1]);
return Number.isFinite(rate) && rate > 0 ? rate : null;
}
function pcm16LeToAudioBuffer(
context: AudioContext,
bytes: Uint8Array,
sampleRate: number
): AudioBuffer {
const sampleCount = Math.floor(bytes.length / 2);
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
const channel = audioBuffer.getChannelData(0);
for (let i = 0; i < sampleCount; i += 1) {
const lo = bytes[i * 2]!;
const hi = bytes[i * 2 + 1]!;
let value = (hi << 8) | lo;
if (value & 0x8000) {
value -= 0x10000;
}
channel[i] = value / 0x8000;
}
return audioBuffer;
}
async function decodeAudioData(
context: AudioContext,
buffer: ArrayBuffer
): Promise<AudioBuffer> {
const maybePromise = context.decodeAudioData(buffer.slice(0));
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
return maybePromise as Promise<AudioBuffer>;
}
return await new Promise<AudioBuffer>((resolve, reject) => {
context.decodeAudioData(buffer.slice(0), resolve, reject);
});
}
export function createAudioEngine(
callbacks: AudioEngineCallbacks,
_options?: { traceLabel?: string }
): AudioEngine {
const refs: {
playbackContext: AudioContext | null;
captureContext: AudioContext | null;
stream: MediaStream | null;
source: MediaStreamAudioSourceNode | null;
processor: ScriptProcessorNode | null;
gain: GainNode | null;
started: boolean;
muted: boolean;
queue: QueuedAudio[];
processingQueue: boolean;
activePlayback: {
source: AudioBufferSourceNode;
resolve: (duration: number) => void;
reject: (error: Error) => void;
settled: boolean;
} | null;
looping: {
token: number;
source: AudioBufferSourceNode | null;
gain: GainNode | null;
timeout: ReturnType<typeof setTimeout> | null;
};
thinkingTone: {
bytes: Uint8Array | null;
buffer: AudioBuffer | null;
};
} = {
playbackContext: null,
captureContext: null,
stream: null,
source: null,
processor: null,
gain: null,
started: false,
muted: false,
queue: [],
processingQueue: false,
activePlayback: null,
looping: {
token: 0,
source: null,
gain: null,
timeout: null,
},
thinkingTone: {
bytes: null,
buffer: null,
},
};
async function ensurePlaybackContext(): Promise<AudioContext> {
if (refs.playbackContext) {
if (refs.playbackContext.state === "suspended") {
await refs.playbackContext.resume().catch(() => undefined);
}
return refs.playbackContext;
}
const AudioContextCtor = getAudioContextCtor();
if (!AudioContextCtor) {
throw new Error("AudioContext unavailable");
}
const context = new AudioContextCtor();
if (context.state === "suspended") {
await context.resume().catch(() => undefined);
}
refs.playbackContext = context;
return context;
}
async function ensureCaptureContext(): Promise<AudioContext> {
if (refs.captureContext) {
if (refs.captureContext.state === "suspended") {
await refs.captureContext.resume().catch(() => undefined);
}
return refs.captureContext;
}
const AudioContextCtor = getAudioContextCtor();
if (!AudioContextCtor) {
throw new Error("AudioContext unavailable");
}
const context = new AudioContextCtor();
if (context.state === "suspended") {
await context.resume().catch(() => undefined);
}
refs.captureContext = context;
return context;
}
async function ensureThinkingToneBytes(): Promise<Uint8Array> {
if (refs.thinkingTone.bytes) {
return refs.thinkingTone.bytes;
}
const wav = await loadThinkingToneArrayBuffer();
refs.thinkingTone.bytes = new Uint8Array(wav);
return refs.thinkingTone.bytes;
}
async function ensureThinkingToneBuffer(): Promise<AudioBuffer> {
if (refs.thinkingTone.buffer) {
return refs.thinkingTone.buffer;
}
const context = await ensurePlaybackContext();
const wav = await loadThinkingToneArrayBuffer();
const audioBuffer = await decodeAudioData(context, wav);
refs.thinkingTone.buffer = audioBuffer;
return audioBuffer;
}
function stopLooping(): void {
refs.looping.token += 1;
if (refs.looping.timeout) {
clearTimeout(refs.looping.timeout);
refs.looping.timeout = null;
}
if (refs.looping.source) {
try {
refs.looping.source.onended = null;
refs.looping.source.stop();
} catch {
// Ignore best-effort stop errors.
}
refs.looping.source.disconnect();
refs.looping.source = null;
}
if (refs.looping.gain) {
refs.looping.gain.disconnect();
refs.looping.gain = null;
}
}
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
const context = await ensurePlaybackContext();
const arrayBuffer = await audio.arrayBuffer();
const type = (audio.type || "").toLowerCase();
const audioBuffer = type.startsWith("audio/pcm")
? pcm16LeToAudioBuffer(
context,
new Uint8Array(arrayBuffer),
parsePcmSampleRate(type) ?? 24000
)
: await decodeAudioData(context, arrayBuffer);
const durationSec = audioBuffer.duration;
const source = context.createBufferSource();
source.buffer = audioBuffer;
source.connect(context.destination);
return await new Promise<number>((resolve, reject) => {
refs.activePlayback = { source, resolve, reject, settled: false };
const settle = (fn: () => void) => {
const active = refs.activePlayback;
if (!active || active.source !== source || active.settled) {
return;
}
active.settled = true;
refs.activePlayback = null;
fn();
};
source.onended = () => {
settle(() => resolve(durationSec));
};
try {
source.start();
} catch (error) {
settle(() =>
reject(error instanceof Error ? error : new Error(String(error)))
);
}
});
}
async function processQueue(): Promise<void> {
if (refs.processingQueue || refs.queue.length === 0) {
return;
}
refs.processingQueue = true;
while (refs.queue.length > 0) {
const item = refs.queue.shift()!;
try {
const duration = await playAudio(item.audio);
item.resolve(duration);
} catch (error) {
item.reject(error instanceof Error ? error : new Error(String(error)));
}
}
refs.processingQueue = false;
}
async function stopCapture(): Promise<void> {
refs.started = false;
try {
refs.processor?.disconnect();
refs.source?.disconnect();
refs.gain?.disconnect();
} catch {
// Ignore best-effort teardown errors.
}
if (refs.stream) {
for (const track of refs.stream.getTracks()) {
try {
track.stop();
} catch {
// Ignore best-effort teardown errors.
}
}
}
refs.stream = null;
refs.source = null;
refs.processor = null;
refs.gain = null;
refs.muted = false;
callbacks.onVolumeLevel(0);
const captureContext = refs.captureContext;
refs.captureContext = null;
if (captureContext && captureContext.state !== "closed") {
await captureContext.close().catch(() => undefined);
}
}
return {
async initialize() {
await ensurePlaybackContext();
await Promise.all([ensureThinkingToneBytes(), ensureThinkingToneBuffer()]);
},
async destroy() {
stopLooping();
this.stop();
this.clearQueue();
await stopCapture();
const playbackContext = refs.playbackContext;
refs.playbackContext = null;
refs.thinkingTone.buffer = null;
if (playbackContext && playbackContext.state !== "closed") {
await playbackContext.close().catch(() => undefined);
}
},
async startCapture() {
if (refs.started) {
return;
}
const missingNavigator =
typeof navigator === "undefined" ||
!navigator.mediaDevices ||
typeof navigator.mediaDevices.getUserMedia !== "function";
const secureContext =
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
? window.isSecureContext
: true;
const currentOrigin =
typeof window !== "undefined" && window.location
? window.location.origin
: "unknown";
const isTauri = getTauri() !== null;
if (missingNavigator) {
throw new Error("Microphone capture is not supported in this environment");
}
if (!secureContext && !isTauri) {
throw new Error(
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`
);
}
try {
const context = await ensureCaptureContext();
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
noiseSuppression: true,
echoCancellation: true,
autoGainControl: true,
},
});
const source = context.createMediaStreamSource(stream);
const processor = context.createScriptProcessor(4096, 1, 1);
const gain = context.createGain();
gain.gain.value = 0;
processor.onaudioprocess = (event) => {
if (!refs.started) {
return;
}
const input = event.inputBuffer.getChannelData(0);
let sumSquares = 0;
for (let i = 0; i < input.length; i += 1) {
const sample = input[i]!;
sumSquares += sample * sample;
}
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
const normalized = Math.min(1, Math.max(0, rms * 2));
callbacks.onVolumeLevel(normalized);
if (refs.muted) {
return;
}
callbacks.onCaptureData(resampleToPcm16(input, context.sampleRate, 16000));
};
source.connect(processor);
processor.connect(gain);
gain.connect(context.destination);
refs.started = true;
refs.stream = stream;
refs.source = source;
refs.processor = processor;
refs.gain = gain;
} catch (error) {
await stopCapture();
const wrapped = error instanceof Error ? error : new Error(String(error));
callbacks.onError?.(wrapped);
throw wrapped;
}
},
async stopCapture() {
await stopCapture();
},
toggleMute() {
refs.muted = !refs.muted;
if (refs.muted) {
callbacks.onVolumeLevel(0);
}
return refs.muted;
},
isMuted() {
return refs.muted;
},
async play(audio: AudioPlaybackSource) {
return await new Promise<number>((resolve, reject) => {
refs.queue.push({ audio, resolve, reject });
if (!refs.processingQueue) {
void processQueue();
}
});
},
stop() {
if (refs.activePlayback) {
const active = refs.activePlayback;
refs.activePlayback = null;
try {
active.source.onended = null;
active.source.stop();
} catch {
// Ignore best-effort stop errors.
}
if (!active.settled) {
active.settled = true;
active.reject(new Error("Playback stopped"));
}
}
},
clearQueue() {
while (refs.queue.length > 0) {
refs.queue.shift()!.reject(new Error("Playback stopped"));
}
refs.processingQueue = false;
},
isPlaying() {
return refs.activePlayback !== null;
},
playLooping(audio, gapMs) {
if (refs.looping.source || refs.looping.timeout) {
return;
}
const token = refs.looping.token + 1;
refs.looping.token = token;
void (async () => {
try {
const context = await ensurePlaybackContext();
const audioBuffer =
audio.byteLength > 0
? pcm16LeToAudioBuffer(context, audio, 16000)
: await ensureThinkingToneBuffer();
if (refs.looping.token !== token) {
return;
}
const gain = context.createGain();
gain.gain.value = 1;
gain.connect(context.destination);
refs.looping.gain = gain;
const playOnce = () => {
if (refs.looping.token !== token) {
return;
}
const source = context.createBufferSource();
source.buffer = audioBuffer;
source.connect(gain);
refs.looping.source = source;
source.onended = () => {
if (refs.looping.source === source) {
refs.looping.source = null;
}
if (refs.looping.token !== token) {
return;
}
refs.looping.timeout = setTimeout(
playOnce,
gapMs || THINKING_TONE_REPEAT_GAP_MS
);
};
source.start();
};
playOnce();
} catch (error) {
callbacks.onError?.(
error instanceof Error ? error : new Error(String(error))
);
}
})();
},
stopLooping,
};
}

22
packages/app/src/voice/speech-segmenter.ts
View File

@@ -139,6 +139,11 @@ export class SpeechSegmenter {
}
flush(isLast: boolean): void {
console.log("[SpeechSegmenter] flush", {
isLast,
audioBufferLength: this.audioBuffer.length,
bufferedBytes: this.bufferedBytes,
});
if (this.audioBuffer.length === 0) {
this.bufferedBytes = 0;
return;
@@ -157,6 +162,13 @@ export class SpeechSegmenter {
* In non-continuous mode, chunk buffering is gated by the VAD state.
*/
pushPcmChunk(chunk: Uint8Array): void {
console.log("[SpeechSegmenter] pushPcmChunk", {
bytes: chunk.length,
speechDetectionStartMs: this.speechDetectionStartMs,
isSpeaking: this.isSpeaking,
speechConfirmed: this.speechConfirmed,
bufferedBytes: this.bufferedBytes,
});
if (chunk.length === 0) {
return;
}
@@ -186,6 +198,16 @@ export class SpeechSegmenter {
* Called with a normalized volume level (0..1). Drives VAD transitions.
*/
pushVolumeLevel(volume: number, nowMs: number): void {
console.log("[SpeechSegmenter] pushVolumeLevel", {
volume,
nowMs,
speechDetectionStartMs: this.speechDetectionStartMs,
isSpeaking: this.isSpeaking,
isDetecting: this.isDetecting,
speechConfirmed: this.speechConfirmed,
silenceStartMs: this.silenceStartMs,
confirmedDropStartMs: this.confirmedDropStartMs,
});
if (this.config.enableContinuousStreaming) {
return;
}

243
packages/app/src/voice/voice-runtime.test.ts Normal file
View File

@@ -0,0 +1,243 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { DaemonServerInfo } from "@/stores/session-store";
import type { AudioEngine } from "@/voice/audio-engine-types";
import {
createVoiceRuntime,
type VoiceRuntime,
type VoiceSessionAdapter,
} from "@/voice/voice-runtime";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
function createAudioEngineMock(): AudioEngine {
return {
initialize: vi.fn().mockResolvedValue(undefined),
destroy: vi.fn().mockResolvedValue(undefined),
startCapture: vi.fn().mockResolvedValue(undefined),
stopCapture: vi.fn().mockResolvedValue(undefined),
toggleMute: vi.fn().mockReturnValue(true),
isMuted: vi.fn().mockReturnValue(false),
play: vi.fn().mockResolvedValue(0.1),
stop: vi.fn(),
clearQueue: vi.fn(),
isPlaying: vi.fn().mockReturnValue(false),
playLooping: vi.fn(),
stopLooping: vi.fn(),
};
}
function createSessionAdapter(serverId = "server-1"): VoiceSessionAdapter {
return {
serverId,
setVoiceMode: vi.fn().mockResolvedValue(undefined),
sendVoiceAudioChunk: vi.fn().mockResolvedValue(undefined),
abortRequest: vi.fn().mockResolvedValue(undefined),
setAssistantAudioPlaying: vi.fn(),
};
}
function createServerInfo(): DaemonServerInfo {
return {
serverId: "server-1",
hostname: "host",
version: "1.0.0",
capabilities: {
voice: {
dictation: { enabled: true, reason: "" },
voice: { enabled: true, reason: "" },
},
},
};
}
function createRuntime(options?: {
engine?: AudioEngine;
getServerInfo?: (serverId: string) => DaemonServerInfo | null;
}) {
const engine = options?.engine ?? createAudioEngineMock();
const runtime = createVoiceRuntime({
engine,
getServerInfo: options?.getServerInfo ?? (() => createServerInfo()),
activateKeepAwake: vi.fn().mockResolvedValue(undefined),
deactivateKeepAwake: vi.fn().mockResolvedValue(undefined),
});
return { runtime, engine };
}
describe("voice runtime", () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
it("starts voice when adapter is ready", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
expect(engine.initialize).toHaveBeenCalled();
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
expect(engine.startCapture).toHaveBeenCalled();
expect(runtime.getSnapshot()).toMatchObject({
phase: "listening",
isVoiceMode: true,
activeServerId: "server-1",
activeAgentId: "agent-1",
});
});
it("transitions from capturing to submitting to waiting on the final chunk", async () => {
const adapter = createSessionAdapter();
const { runtime } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.handleCaptureVolume(0.4);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
);
runtime.handleCaptureVolume(0.4);
runtime.handleCapturePcm(new Uint8Array(32000));
expect(runtime.getSnapshot().phase).toBe("capturing");
runtime.handleCapturePcm(new Uint8Array(4000));
runtime.handleCaptureVolume(0);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs + 10
);
runtime.handleCaptureVolume(0);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.silenceDurationMs + 10
);
runtime.handleCaptureVolume(0);
await Promise.resolve();
expect(adapter.sendVoiceAudioChunk).toHaveBeenLastCalledWith(
expect.any(String),
"audio/pcm;rate=16000;bits=16",
true
);
expect(runtime.getSnapshot().phase).toBe("waiting");
});
it("moves from waiting to playing on the first assistant audio", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onAssistantAudioStarted("server-1");
expect(runtime.getSnapshot().phase).toBe("playing");
expect(engine.stopLooping).toHaveBeenCalled();
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(true);
});
it("returns to waiting after assistant playback when the turn is still active", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
runtime.onAssistantAudioStarted("server-1");
runtime.onAssistantAudioFinished("server-1");
expect(runtime.getSnapshot().phase).toBe("waiting");
expect(engine.playLooping).toHaveBeenCalled();
});
it("returns to listening after assistant playback once the turn is complete", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
runtime.onAssistantAudioStarted("server-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_completed");
runtime.onAssistantAudioFinished("server-1");
expect(runtime.getSnapshot().phase).toBe("listening");
expect(engine.playLooping).not.toHaveBeenCalled();
});
it("interrupts the active turn on barge-in after the grace period", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
runtime.onAssistantAudioStarted("server-1");
runtime.handleCaptureVolume(0.5);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
);
await vi.advanceTimersByTimeAsync(
REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs
);
expect(adapter.abortRequest).toHaveBeenCalled();
expect(engine.stop).toHaveBeenCalled();
expect(runtime.getSnapshot().phase).toBe("capturing");
});
it("authoritatively stops and suppresses later voice audio", async () => {
const adapter = createSessionAdapter();
const { runtime, engine } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
await runtime.stopVoice();
expect(adapter.setVoiceMode).toHaveBeenLastCalledWith(false);
expect(engine.stopCapture).toHaveBeenCalled();
expect(runtime.getSnapshot().phase).toBe("disabled");
expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false);
});
it("returns an explicit not-ready error when the adapter is missing", async () => {
const { runtime } = createRuntime();
await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
"Voice runtime is not ready for host server-1"
);
});
it("resyncs voice mode after connection recovers", async () => {
const adapter = createSessionAdapter();
const { runtime } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
vi.mocked(adapter.setVoiceMode).mockClear();
runtime.updateSessionConnection("server-1", false);
runtime.updateSessionConnection("server-1", true);
await Promise.resolve();
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
});
it("does not emit when the snapshot is unchanged", async () => {
const adapter = createSessionAdapter();
const { runtime } = createRuntime();
runtime.registerSession(adapter);
await runtime.startVoice("server-1", "agent-1");
const listener = vi.fn();
const unsubscribe = runtime.subscribe(listener);
runtime.handleCaptureVolume(0);
runtime.handleCaptureVolume(0);
expect(listener).not.toHaveBeenCalled();
unsubscribe();
});
});

859
packages/app/src/voice/voice-runtime.ts Normal file
View File

@@ -0,0 +1,859 @@
import type { AgentStreamEventPayload } from "@server/shared/messages";
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
import type { DaemonServerInfo } from "@/stores/session-store";
import type { AudioEngine } from "@/voice/audio-engine-types";
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
import { SpeechSegmenter } from "@/voice/speech-segmenter";
const PCM_MIME_TYPE = "audio/pcm;rate=16000;bits=16";
const KEEP_AWAKE_TAG = "paseo:voice";
const THINKING_TONE_REPEAT_GAP_MS = 350;
const DISPLAY_VOLUME_PUBLISH_INTERVAL_MS = 120;
const DISPLAY_VOLUME_CHANGE_EPSILON = 0.02;
const DISPLAY_VOLUME_ATTACK = 0.35;
const DISPLAY_VOLUME_RELEASE = 0.18;
type TurnEventType = Extract<
AgentStreamEventPayload["type"],
"turn_started" | "turn_completed" | "turn_failed" | "turn_canceled"
>;
export type VoiceRuntimePhase =
| "disabled"
| "starting"
| "listening"
| "capturing"
| "submitting"
| "waiting"
| "playing"
| "stopping";
export interface VoiceRuntimeSnapshot {
phase: VoiceRuntimePhase;
isVoiceMode: boolean;
isVoiceSwitching: boolean;
isMuted: boolean;
activeServerId: string | null;
activeAgentId: string | null;
}
export interface VoiceRuntimeTelemetrySnapshot {
volume: number;
isDetecting: boolean;
isSpeaking: boolean;
segmentDuration: number;
}
export interface VoiceSessionAdapter {
serverId: string;
setVoiceMode(enabled: boolean, agentId?: string): Promise<void>;
sendVoiceAudioChunk(
audioData: string,
mimeType: string,
isLast: boolean
): Promise<void>;
abortRequest(): Promise<void>;
setAssistantAudioPlaying(isPlaying: boolean): void;
}
export interface VoiceRuntimeDeps {
engine: AudioEngine;
getServerInfo(serverId: string): DaemonServerInfo | null;
activateKeepAwake(tag: string): Promise<void>;
deactivateKeepAwake(tag: string): Promise<void>;
}
interface RuntimeSessionState {
adapter: VoiceSessionAdapter;
connected: boolean;
}
interface RuntimeState {
snapshot: VoiceRuntimeSnapshot;
telemetry: VoiceRuntimeTelemetrySnapshot;
turnInProgress: boolean;
transportReady: boolean;
generation: number;
speechInterruptTimer: ReturnType<typeof setTimeout> | null;
speechInterruptSent: boolean;
segmentDurationTimer: ReturnType<typeof setInterval> | null;
lastDisplayVolumePublishMs: number;
}
const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
phase: "disabled",
isVoiceMode: false,
isVoiceSwitching: false,
isMuted: false,
activeServerId: null,
activeAgentId: null,
};
const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
volume: 0,
isDetecting: false,
isSpeaking: false,
segmentDuration: 0,
};
function logVoiceRuntime(
event: string,
details?: Record<string, unknown>
): void {
if (details) {
console.log(`[VoiceRuntime] ${event}`, details);
return;
}
console.log(`[VoiceRuntime] ${event}`);
}
function snapshotsEqual(
left: VoiceRuntimeSnapshot,
right: VoiceRuntimeSnapshot
): boolean {
return (
left.phase === right.phase &&
left.isVoiceMode === right.isVoiceMode &&
left.isVoiceSwitching === right.isVoiceSwitching &&
left.isMuted === right.isMuted &&
left.activeServerId === right.activeServerId &&
left.activeAgentId === right.activeAgentId
);
}
function telemetryEqual(
left: VoiceRuntimeTelemetrySnapshot,
right: VoiceRuntimeTelemetrySnapshot
): boolean {
return (
left.volume === right.volume &&
left.isDetecting === right.isDetecting &&
left.isSpeaking === right.isSpeaking &&
left.segmentDuration === right.segmentDuration
);
}
export interface VoiceRuntime {
subscribe(listener: () => void): () => void;
getSnapshot(): VoiceRuntimeSnapshot;
subscribeTelemetry(listener: () => void): () => void;
getTelemetrySnapshot(): VoiceRuntimeTelemetrySnapshot;
registerSession(adapter: VoiceSessionAdapter): () => void;
updateSessionConnection(serverId: string, connected: boolean): void;
handleCapturePcm(chunk: Uint8Array): void;
handleCaptureVolume(level: number): void;
startVoice(serverId: string, agentId: string): Promise<void>;
stopVoice(): Promise<void>;
destroy(): Promise<void>;
toggleMute(): void;
isVoiceModeForAgent(serverId: string, agentId: string): boolean;
shouldPlayVoiceAudio(serverId: string): boolean;
onAssistantAudioStarted(serverId: string): void;
onAssistantAudioFinished(serverId: string): void;
onTranscriptionResult(serverId: string, text: string): void;
onTurnEvent(serverId: string, agentId: string, eventType: TurnEventType): void;
}
export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
const listeners = new Set<() => void>();
const telemetryListeners = new Set<() => void>();
const sessions = new Map<string, RuntimeSessionState>();
const state: RuntimeState = {
snapshot: INITIAL_SNAPSHOT,
telemetry: INITIAL_TELEMETRY,
turnInProgress: false,
transportReady: false,
generation: 0,
speechInterruptTimer: null,
speechInterruptSent: false,
segmentDurationTimer: null,
lastDisplayVolumePublishMs: 0,
};
function emit(): void {
for (const listener of listeners) {
listener();
}
}
function emitTelemetry(): void {
for (const listener of telemetryListeners) {
listener();
}
}
function patchSnapshot(
patch:
| Partial<VoiceRuntimeSnapshot>
| ((previous: VoiceRuntimeSnapshot) => VoiceRuntimeSnapshot)
): void {
const next =
typeof patch === "function"
? patch(state.snapshot)
: { ...state.snapshot, ...patch };
if (snapshotsEqual(next, state.snapshot)) {
return;
}
const previous = state.snapshot;
state.snapshot = next;
logVoiceRuntime("snapshot_changed", {
previousPhase: previous.phase,
phase: next.phase,
isVoiceMode: next.isVoiceMode,
isVoiceSwitching: next.isVoiceSwitching,
isMuted: next.isMuted,
activeServerId: next.activeServerId,
activeAgentId: next.activeAgentId,
});
emit();
}
function patchTelemetry(
patch:
| Partial<VoiceRuntimeTelemetrySnapshot>
| ((
previous: VoiceRuntimeTelemetrySnapshot
) => VoiceRuntimeTelemetrySnapshot)
): void {
const next =
typeof patch === "function"
? patch(state.telemetry)
: { ...state.telemetry, ...patch };
if (telemetryEqual(next, state.telemetry)) {
return;
}
state.telemetry = next;
emitTelemetry();
}
function getActiveSession(): RuntimeSessionState | null {
if (!state.snapshot.activeServerId) {
return null;
}
return sessions.get(state.snapshot.activeServerId) ?? null;
}
function clearSpeechInterruptTimer(): void {
if (state.speechInterruptTimer) {
clearTimeout(state.speechInterruptTimer);
state.speechInterruptTimer = null;
}
}
function clearSegmentDurationTimer(): void {
if (state.segmentDurationTimer) {
clearInterval(state.segmentDurationTimer);
state.segmentDurationTimer = null;
}
}
function reconcileSegmentDurationTimer(
segmenter: SpeechSegmenter
): void {
if (!state.telemetry.isDetecting && !state.telemetry.isSpeaking) {
clearSegmentDurationTimer();
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
return;
}
if (state.segmentDurationTimer) {
return;
}
state.segmentDurationTimer = setInterval(() => {
const startedAt = segmenter.getSpeechDetectionStartMs();
patchTelemetry((prev) => ({
...prev,
segmentDuration: startedAt ? Date.now() - startedAt : 0,
}));
}, 100);
}
function canPlayCue(): boolean {
return (
state.snapshot.isVoiceMode &&
state.snapshot.phase === "waiting" &&
!state.telemetry.isDetecting &&
!state.telemetry.isSpeaking
);
}
function stopCue(): void {
deps.engine.stopLooping();
}
function resetSegmenter(segmenter: SpeechSegmenter): void {
segmenter.reset();
clearSpeechInterruptTimer();
clearSegmentDurationTimer();
patchTelemetry({ ...INITIAL_TELEMETRY });
}
function reconcileCue(): void {
if (!canPlayCue()) {
stopCue();
return;
}
logVoiceRuntime("thinking_tone_requested", {
phase: state.snapshot.phase,
isDetecting: state.telemetry.isDetecting,
isSpeaking: state.telemetry.isSpeaking,
});
deps.engine.playLooping(new Uint8Array(0), THINKING_TONE_REPEAT_GAP_MS);
}
async function interruptActiveTurn(): Promise<void> {
const activeSession = getActiveSession();
if (!activeSession) {
return;
}
stopCue();
deps.engine.stop();
deps.engine.clearQueue();
activeSession.adapter.setAssistantAudioPlaying(false);
patchSnapshot((prev) => ({ ...prev, phase: "capturing" }));
try {
await activeSession.adapter.abortRequest();
} catch (error) {
console.error("[VoiceRuntime] Failed to abort active turn:", error);
}
}
const segmenter = new SpeechSegmenter(
{
enableContinuousStreaming: false,
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
confirmedDropGracePeriodMs:
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
silenceDurationMs: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
speechConfirmationMs: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
detectionGracePeriodMs: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
},
{
onAudioSegment: ({ audioData, isLast }) => {
const activeSession = getActiveSession();
if (!activeSession || !state.transportReady || !state.snapshot.isVoiceMode) {
return;
}
const generation = state.generation;
patchSnapshot((prev) => ({
...prev,
phase: isLast ? "submitting" : "capturing",
}));
if (isLast) {
state.turnInProgress = true;
}
logVoiceRuntime("audio_segment_ready", {
isLast,
base64Bytes: audioData.length,
phase: state.snapshot.phase,
serverId: activeSession.adapter.serverId,
});
void activeSession.adapter
.sendVoiceAudioChunk(audioData, PCM_MIME_TYPE, isLast)
.then(() => {
if (
!isLast ||
generation !== state.generation ||
!state.snapshot.isVoiceMode ||
state.snapshot.activeServerId !== activeSession.adapter.serverId ||
state.snapshot.phase !== "submitting"
) {
return;
}
logVoiceRuntime("audio_segment_sent", {
isLast,
serverId: activeSession.adapter.serverId,
});
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
reconcileCue();
})
.catch((error) => {
console.error("[VoiceRuntime] Failed to send audio segment:", error);
});
},
onSpeechStart: () => {
logVoiceRuntime("speech_started");
stopCue();
},
onSpeechEnd: () => {
logVoiceRuntime("speech_ended");
clearSpeechInterruptTimer();
state.speechInterruptSent = false;
reconcileCue();
},
onDetectingChange: (isDetecting) => {
const previous = state.telemetry;
patchTelemetry((prev) => ({ ...prev, isDetecting }));
if (previous.isDetecting !== isDetecting) {
logVoiceRuntime("detecting_changed", {
isDetecting,
volume: Number(state.telemetry.volume.toFixed(3)),
});
}
if (!previous.isDetecting && isDetecting) {
stopCue();
}
reconcileSegmentDurationTimer(segmenter);
reconcileCue();
},
onSpeakingChange: (isSpeaking) => {
const previous = state.telemetry;
patchTelemetry((prev) => ({ ...prev, isSpeaking }));
if (previous.isSpeaking !== isSpeaking) {
logVoiceRuntime("speaking_changed", {
isSpeaking,
phase: state.snapshot.phase,
volume: Number(state.telemetry.volume.toFixed(3)),
});
}
if (!previous.isSpeaking && isSpeaking) {
stopCue();
if (
state.snapshot.phase === "waiting" ||
state.snapshot.phase === "playing"
) {
clearSpeechInterruptTimer();
state.speechInterruptTimer = setTimeout(() => {
state.speechInterruptTimer = null;
if (
state.speechInterruptSent ||
!state.snapshot.isVoiceMode ||
!state.telemetry.isSpeaking
) {
return;
}
state.speechInterruptSent = true;
logVoiceRuntime("barge_in_interrupt_triggered", {
phase: state.snapshot.phase,
});
void interruptActiveTurn();
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
logVoiceRuntime("barge_in_interrupt_scheduled", {
gracePeriodMs: REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs,
phase: state.snapshot.phase,
});
}
}
if (!isSpeaking) {
clearSpeechInterruptTimer();
state.speechInterruptSent = false;
}
reconcileSegmentDurationTimer(segmenter);
reconcileCue();
},
}
);
function resetToDisabledState(): void {
state.transportReady = false;
state.turnInProgress = false;
state.speechInterruptSent = false;
state.lastDisplayVolumePublishMs = 0;
resetSegmenter(segmenter);
patchSnapshot({ ...INITIAL_SNAPSHOT });
}
function publishDisplayVolume(level: number, nowMs: number): void {
const previousVolume = state.telemetry.volume;
const smoothing =
level >= previousVolume ? DISPLAY_VOLUME_ATTACK : DISPLAY_VOLUME_RELEASE;
const nextVolume = Math.max(
0,
Math.min(1, previousVolume + (level - previousVolume) * smoothing)
);
const enoughTimeElapsed =
nowMs - state.lastDisplayVolumePublishMs >=
DISPLAY_VOLUME_PUBLISH_INTERVAL_MS;
const enoughChange =
Math.abs(nextVolume - previousVolume) >= DISPLAY_VOLUME_CHANGE_EPSILON;
if (!enoughTimeElapsed && !enoughChange) {
return;
}
state.lastDisplayVolumePublishMs = nowMs;
patchTelemetry((prev) => ({
...prev,
volume: Number(nextVolume.toFixed(3)),
}));
}
async function performLocalStop(): Promise<void> {
stopCue();
deps.engine.stop();
deps.engine.clearQueue();
await deps.engine.stopCapture().catch(() => undefined);
await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
resetToDisabledState();
}
async function resyncVoiceMode(serverId: string): Promise<void> {
if (
!state.snapshot.isVoiceMode ||
state.snapshot.activeServerId !== serverId ||
!state.snapshot.activeAgentId
) {
return;
}
const activeSession = getActiveSession();
if (!activeSession || !activeSession.connected) {
return;
}
patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: true }));
try {
await activeSession.adapter.setVoiceMode(true, state.snapshot.activeAgentId);
state.transportReady = true;
} finally {
patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: false }));
}
}
return {
subscribe(listener) {
listeners.add(listener);
return () => {
listeners.delete(listener);
};
},
getSnapshot() {
return state.snapshot;
},
subscribeTelemetry(listener) {
telemetryListeners.add(listener);
return () => {
telemetryListeners.delete(listener);
};
},
getTelemetrySnapshot() {
return state.telemetry;
},
registerSession(adapter) {
logVoiceRuntime("session_registered", {
serverId: adapter.serverId,
});
sessions.set(adapter.serverId, {
adapter,
connected: true,
});
return () => {
const activeServerId = state.snapshot.activeServerId;
sessions.delete(adapter.serverId);
if (activeServerId === adapter.serverId) {
void performLocalStop();
}
};
},
updateSessionConnection(serverId, connected) {
const session = sessions.get(serverId);
if (!session) {
return;
}
session.connected = connected;
logVoiceRuntime("session_connection_changed", {
serverId,
connected,
});
if (state.snapshot.activeServerId !== serverId) {
return;
}
if (!connected) {
state.transportReady = false;
return;
}
void resyncVoiceMode(serverId);
},
handleCapturePcm(chunk) {
console.log("[VoiceRuntime] handleCapturePcm", {
bytes: chunk.byteLength,
isVoiceMode: state.snapshot.isVoiceMode,
isMuted: state.snapshot.isMuted,
phase: state.snapshot.phase,
});
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
return;
}
segmenter.pushPcmChunk(chunk);
},
handleCaptureVolume(level) {
const nowMs = Date.now();
console.log("[VoiceRuntime] handleCaptureVolume", {
level,
isVoiceMode: state.snapshot.isVoiceMode,
isMuted: state.snapshot.isMuted,
phase: state.snapshot.phase,
});
const displayLevel = state.snapshot.isMuted ? 0 : level;
publishDisplayVolume(displayLevel, nowMs);
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
return;
}
segmenter.pushVolumeLevel(level, nowMs);
},
async startVoice(serverId, agentId) {
logVoiceRuntime("start_requested", {
serverId,
agentId,
});
const session = sessions.get(serverId);
if (!session) {
throw new Error(`Voice runtime is not ready for host ${serverId}`);
}
if (!session.connected) {
throw new Error(`Host ${serverId} is not connected`);
}
const serverInfo = deps.getServerInfo(serverId);
const unavailableMessage = resolveVoiceUnavailableMessage({
serverInfo,
mode: "voice",
});
if (unavailableMessage) {
throw new Error(unavailableMessage);
}
const previousServerId = state.snapshot.activeServerId;
const previousAgentId = state.snapshot.activeAgentId;
const generation = state.generation + 1;
state.generation = generation;
state.transportReady = false;
patchSnapshot((prev) => ({
...prev,
isVoiceSwitching: true,
phase: "starting",
activeServerId: serverId,
activeAgentId: agentId,
}));
try {
if (
state.snapshot.isVoiceMode &&
previousServerId &&
(previousServerId !== serverId || previousAgentId !== agentId)
) {
const previousSession = sessions.get(previousServerId);
if (previousSession) {
previousSession.adapter.setAssistantAudioPlaying(false);
await previousSession.adapter.setVoiceMode(false);
}
}
await deps.activateKeepAwake(KEEP_AWAKE_TAG).catch((error) => {
console.warn("[VoiceRuntime] Failed to activate keep-awake:", error);
});
await deps.engine.initialize();
await session.adapter.setVoiceMode(true, agentId);
await deps.engine.startCapture();
if (state.generation !== generation) {
return;
}
state.transportReady = true;
state.turnInProgress = false;
resetSegmenter(segmenter);
patchSnapshot((prev) => ({
...prev,
isVoiceMode: true,
isVoiceSwitching: false,
phase: "listening",
isMuted: deps.engine.isMuted(),
}));
logVoiceRuntime("start_completed", {
serverId,
agentId,
});
} catch (error) {
logVoiceRuntime("start_failed", {
serverId,
agentId,
error: error instanceof Error ? error.message : String(error),
});
await performLocalStop();
throw error;
}
},
async stopVoice() {
logVoiceRuntime("stop_requested", {
activeServerId: state.snapshot.activeServerId,
activeAgentId: state.snapshot.activeAgentId,
});
const activeSession = getActiveSession();
const generation = state.generation + 1;
state.generation = generation;
patchSnapshot((prev) => ({
...prev,
isVoiceSwitching: true,
phase: "stopping",
}));
try {
state.transportReady = false;
stopCue();
deps.engine.stop();
deps.engine.clearQueue();
activeSession?.adapter.setAssistantAudioPlaying(false);
if (activeSession) {
await activeSession.adapter.setVoiceMode(false);
}
await deps.engine.stopCapture();
await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
} finally {
if (state.generation === generation) {
resetToDisabledState();
}
logVoiceRuntime("stop_completed");
}
},
async destroy() {
await this.stopVoice().catch(() => undefined);
await deps.engine.destroy();
listeners.clear();
telemetryListeners.clear();
sessions.clear();
},
toggleMute() {
const nextMuted = deps.engine.toggleMute();
logVoiceRuntime("mute_toggled", {
muted: nextMuted,
});
if (nextMuted) {
resetSegmenter(segmenter);
patchSnapshot((prev) => ({
...prev,
isMuted: true,
}));
reconcileCue();
return;
}
patchSnapshot((prev) => ({ ...prev, isMuted: false }));
},
isVoiceModeForAgent(serverId, agentId) {
return (
state.snapshot.isVoiceMode &&
state.snapshot.activeServerId === serverId &&
state.snapshot.activeAgentId === agentId
);
},
shouldPlayVoiceAudio(serverId) {
return (
state.snapshot.isVoiceMode &&
state.snapshot.activeServerId === serverId &&
state.snapshot.phase !== "stopping" &&
state.snapshot.phase !== "disabled" &&
!state.telemetry.isDetecting &&
!state.telemetry.isSpeaking
);
},
onAssistantAudioStarted(serverId) {
if (
!state.snapshot.isVoiceMode ||
state.snapshot.activeServerId !== serverId
) {
return;
}
logVoiceRuntime("assistant_audio_started", {
serverId,
});
stopCue();
getActiveSession()?.adapter.setAssistantAudioPlaying(true);
patchSnapshot((prev) => ({ ...prev, phase: "playing" }));
},
onAssistantAudioFinished(serverId) {
if (state.snapshot.activeServerId !== serverId) {
return;
}
logVoiceRuntime("assistant_audio_finished", {
serverId,
turnInProgress: state.turnInProgress,
});
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
if (!state.snapshot.isVoiceMode) {
return;
}
if (state.turnInProgress) {
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
reconcileCue();
return;
}
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
reconcileCue();
},
onTranscriptionResult(serverId, text) {
if (
serverId !== state.snapshot.activeServerId ||
!state.snapshot.isVoiceMode
) {
return;
}
if (text.trim()) {
return;
}
logVoiceRuntime("empty_transcription_result", {
serverId,
});
state.turnInProgress = false;
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
stopCue();
},
onTurnEvent(serverId, agentId, eventType) {
if (
!state.snapshot.isVoiceMode ||
state.snapshot.activeServerId !== serverId ||
state.snapshot.activeAgentId !== agentId
) {
return;
}
logVoiceRuntime("turn_event", {
serverId,
agentId,
eventType,
});
if (eventType === "turn_started") {
state.turnInProgress = true;
return;
}
state.turnInProgress = false;
if (state.snapshot.phase !== "playing") {
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
}
stopCue();
},
};
}

142
packages/server/src/server/agent/providers/codex-app-server-agent.ts
View File

@@ -44,6 +44,7 @@ import {
resolveProviderCommandPrefix,
type ProviderRuntimeSettings,
} from "../provider-launch-config.js";
import { extractCodexTerminalSessionId, nonEmptyString } from "./tool-call-mapper-utils.js";
const DEFAULT_TIMEOUT_MS = 14 * 24 * 60 * 60 * 1000;
@@ -1054,6 +1055,32 @@ function mapCodexPatchNotificationToToolCall(params: {
return params.running ? toRunningToolCall(mapped) : mapped;
}
function mapCodexTerminalInteractionToToolCall(params: {
processId?: string | null;
fallbackCallId?: string | null;
command?: string | null;
}): ToolCallTimelineItem {
const processId = nonEmptyString(params.processId ?? undefined);
const callId =
processId
? `terminal-session-${processId}`
: nonEmptyString(params.fallbackCallId ?? undefined) ?? "terminal-interaction";
const label = nonEmptyString(params.command ?? undefined);
return {
type: "tool_call",
callId,
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
...(label ? { label } : {}),
icon: "square_terminal",
},
...(processId ? { metadata: { processId } } : {}),
};
}
function threadItemToTimeline(
item: any,
options?: { includeUserMessage?: boolean; cwd?: string | null }
@@ -1278,6 +1305,23 @@ const CodexEventExecCommandOutputDeltaNotificationSchema = z.object({
.passthrough(),
}).passthrough();
const CodexEventTerminalInteractionNotificationSchema = z.object({
msg: z
.object({
type: z.literal("terminal_interaction"),
call_id: z.string().optional(),
process_id: z.union([z.string(), z.number()]).optional(),
stdin: z.string().optional(),
})
.passthrough(),
}).passthrough();
const ItemCommandExecutionTerminalInteractionNotificationSchema = z.object({
itemId: z.string().optional(),
processId: z.union([z.string(), z.number()]).optional(),
stdin: z.string().optional(),
}).passthrough();
const CodexEventPatchApplyBeginNotificationSchema = z.object({
msg: z
.object({
@@ -1358,6 +1402,13 @@ type ParsedCodexNotification =
stream: string | null;
chunk: string | null;
}
| {
kind: "terminal_interaction";
source: "item" | "codex_event";
callId: string | null;
processId: string | null;
stdin: string | null;
}
| {
kind: "patch_apply_started";
callId: string | null;
@@ -1544,6 +1595,45 @@ const CodexNotificationSchema = z.union([
}).transform(
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
),
z.object({
method: z.literal("codex/event/terminal_interaction"),
params: CodexEventTerminalInteractionNotificationSchema,
}).transform(
({ params }): ParsedCodexNotification => ({
kind: "terminal_interaction",
source: "codex_event",
callId: params.msg.call_id ?? null,
processId:
typeof params.msg.process_id === "number"
? String(params.msg.process_id)
: params.msg.process_id ?? null,
stdin: params.msg.stdin ?? null,
})
),
z.object({ method: z.literal("codex/event/terminal_interaction"), params: z.unknown() }).transform(
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
),
z.object({
method: z.literal("item/commandExecution/terminalInteraction"),
params: ItemCommandExecutionTerminalInteractionNotificationSchema,
}).transform(
({ params }): ParsedCodexNotification => ({
kind: "terminal_interaction",
source: "item",
callId: params.itemId ?? null,
processId:
typeof params.processId === "number"
? String(params.processId)
: params.processId ?? null,
stdin: params.stdin ?? null,
})
),
z.object({
method: z.literal("item/commandExecution/terminalInteraction"),
params: z.unknown(),
}).transform(
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
),
z.object({
method: z.literal("codex/event/patch_apply_begin"),
params: CodexEventPatchApplyBeginNotificationSchema,
@@ -1758,6 +1848,8 @@ class CodexAppServerAgentSession implements AgentSession {
private pendingReasoning = new Map<string, string[]>();
private pendingCommandOutputDeltas = new Map<string, string[]>();
private pendingFileChangeOutputDeltas = new Map<string, string[]>();
private terminalCommandByProcessId = new Map<string, string>();
private emittedTerminalInteractionKeys = new Set<string>();
private emittedExecCommandStartedCallIds = new Set<string>();
private emittedExecCommandCompletedCallIds = new Set<string>();
private emittedItemStartedIds = new Set<string>();
@@ -2667,11 +2759,13 @@ class CodexAppServerAgentSession implements AgentSession {
this.pendingCommandOutputDeltas,
parsed.callId
);
const resolvedOutput = parsed.output ?? bufferedOutput;
this.rememberTerminalProcessForCommand(parsed.command, resolvedOutput);
const timelineItem = mapCodexExecNotificationToToolCall({
callId: parsed.callId,
command: parsed.command,
cwd: parsed.cwd ?? this.config.cwd ?? null,
output: parsed.output ?? bufferedOutput,
output: resolvedOutput,
exitCode: parsed.exitCode,
success: parsed.success,
stderr: parsed.stderr,
@@ -2684,6 +2778,25 @@ class CodexAppServerAgentSession implements AgentSession {
return;
}
if (parsed.kind === "terminal_interaction") {
const interactionKey = [
parsed.processId ?? "",
parsed.stdin ?? "",
].join("\u0000");
if (!this.shouldEmitTerminalInteractionKey(interactionKey)) {
return;
}
const timelineItem = mapCodexTerminalInteractionToToolCall({
processId: parsed.processId,
fallbackCallId: parsed.callId,
command:
(parsed.processId ? this.terminalCommandByProcessId.get(parsed.processId) : undefined) ??
null,
});
this.emitEvent({ type: "timeline", provider: CODEX_PROVIDER, item: timelineItem });
return;
}
if (parsed.kind === "patch_apply_started") {
if (parsed.callId) {
this.pendingFileChangeOutputDeltas.delete(parsed.callId);
@@ -2876,6 +2989,33 @@ class CodexAppServerAgentSession implements AgentSession {
return buffered.join("");
}
private rememberTerminalProcessForCommand(command: unknown, output: string | null): void {
const normalizedCommand = normalizeCodexCommandValue(command);
if (!normalizedCommand) {
return;
}
const displayCommand =
typeof normalizedCommand === "string"
? normalizedCommand
: normalizedCommand.join(" ").trim();
if (!displayCommand) {
return;
}
const processId = extractCodexTerminalSessionId(output ?? undefined);
if (!processId) {
return;
}
this.terminalCommandByProcessId.set(processId, displayCommand);
}
private shouldEmitTerminalInteractionKey(key: string): boolean {
if (this.emittedTerminalInteractionKeys.has(key)) {
return false;
}
this.emittedTerminalInteractionKeys.add(key);
return true;
}
private warnOnIncompleteEditToolCall(
item: ToolCallTimelineItem,
source: string,

62
packages/server/src/server/agent/providers/codex-rollout-parsing.test.ts
View File

@@ -86,7 +86,7 @@ describe("codex rollout parsing", () => {
});
});
test("skips write_stdin function calls (polling)", async () => {
test("maps write_stdin polling calls into terminal interaction tool calls", async () => {
const rolloutPath = join(tmpDir, "rollout.jsonl");
const lines = [
JSON.stringify({
@@ -99,6 +99,16 @@ describe("codex rollout parsing", () => {
call_id: "call_real",
},
}),
JSON.stringify({
timestamp: "2026-01-22T07:09:01.785Z",
type: "response_item",
payload: {
type: "function_call_output",
call_id: "call_real",
output:
"Chunk ID: 13d232\nWall time: 0.2667 seconds\nProcess exited with code 0\nOriginal token count: 10\nOutput:\nProcess running with session ID 7144",
},
}),
JSON.stringify({
timestamp: "2026-01-22T07:28:16.497Z",
type: "response_item",
@@ -116,8 +126,54 @@ describe("codex rollout parsing", () => {
const timeline = await parseRolloutFile(rolloutPath);
const toolCalls = timeline.filter((i) => i.type === "tool_call");
expect(toolCalls.length).toBe(1);
expect(toolCalls[0].name).toBe("Bash");
expect(toolCalls.length).toBe(2);
expect(toolCalls[0]).toMatchObject({
type: "tool_call",
name: "Bash",
});
expect(toolCalls[1]).toMatchObject({
type: "tool_call",
name: "terminal",
callId: "terminal-session-7144",
detail: {
type: "plain_text",
icon: "square_terminal",
},
metadata: {
processId: "7144",
},
});
});
test("falls back to generic terminal interaction label when command cannot be resolved", async () => {
const rolloutPath = join(tmpDir, "rollout.jsonl");
const lines = [
JSON.stringify({
timestamp: "2026-01-22T07:28:16.497Z",
type: "response_item",
payload: {
type: "function_call",
name: "write_stdin",
arguments:
'{"session_id":7144,"chars":"","yield_time_ms":1000,"max_output_tokens":6000}',
call_id: "call_polling",
},
}),
];
writeFileSync(rolloutPath, lines.join("\n") + "\n");
const timeline = await parseRolloutFile(rolloutPath);
const toolCalls = timeline.filter((i) => i.type === "tool_call");
expect(toolCalls).toHaveLength(1);
expect(toolCalls[0]).toMatchObject({
type: "tool_call",
name: "terminal",
callId: "terminal-session-7144",
detail: {
type: "plain_text",
icon: "square_terminal",
},
});
});
});

101
packages/server/src/server/agent/providers/codex-rollout-timeline.ts
View File

@@ -7,6 +7,7 @@ import type { Logger } from "pino";
import type { AgentTimelineItem } from "../agent-sdk-types.js";
import { mapCodexRolloutToolCall } from "./codex/tool-call-mapper.js";
import { extractCodexTerminalSessionId, nonEmptyString } from "./tool-call-mapper-utils.js";
const MAX_ROLLOUT_SEARCH_DEPTH = 4;
@@ -272,6 +273,74 @@ function parseJsonLikeValue(value: unknown): unknown {
return value;
}
function readTerminalSessionId(value: unknown): string | undefined {
if (typeof value === "number" && Number.isFinite(value)) {
return String(value);
}
if (typeof value === "string") {
return nonEmptyString(value.trim());
}
return undefined;
}
function mapCodexTerminalInteractionToToolCall(params: {
processId?: string | undefined;
fallbackCallId?: string | undefined;
command?: string | undefined;
}): Extract<AgentTimelineItem, { type: "tool_call" }> {
const processId = nonEmptyString(params.processId);
const callId = processId
? `terminal-session-${processId}`
: nonEmptyString(params.fallbackCallId) ?? "terminal-interaction";
const label = nonEmptyString(params.command);
return {
type: "tool_call",
callId,
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
...(label ? { label } : {}),
icon: "square_terminal",
},
metadata: processId ? { processId } : undefined,
};
}
function buildTerminalCommandBySessionId(
parsedRecords: ParsedRolloutRecord[]
): Map<string, string> {
const outputsByCallId = parsedRecords
.filter((record): record is Extract<ParsedRolloutRecord, { kind: "output" }> => record.kind === "output")
.reduce((map, record) => map.set(record.callId, record.output), new Map<string, unknown>());
const commandsBySessionId = new Map<string, string>();
for (const record of parsedRecords) {
if (record.kind !== "call" || record.name === "write_stdin") {
continue;
}
const mapped = mapCodexRolloutToolCall({
callId: record.callId ?? null,
name: record.name,
input: record.input ?? null,
output: record.callId ? outputsByCallId.get(record.callId) ?? null : null,
});
if (!mapped || mapped.detail.type !== "shell") {
continue;
}
const sessionId = extractCodexTerminalSessionId(mapped.detail.output);
if (!sessionId) {
continue;
}
commandsBySessionId.set(sessionId, mapped.detail.command);
}
return commandsBySessionId;
}
function readOutputPayloadValue(payload: Record<string, unknown>): unknown {
if (payload.output !== undefined) {
return parseJsonLikeValue(payload.output);
@@ -319,15 +388,12 @@ const RolloutResponseRecordSchema = z
rawName === "exec_command" || rawName === "shell"
? FunctionCallInputNormalizationSchema.parse(parsedArguments)
: { name: rawName, input: parsedArguments };
const skip = rawName === "write_stdin";
return skip
? { kind: "ignore" }
: {
kind: "call",
name: normalized.name,
callId: payload.call_id,
input: normalized.input,
};
return {
kind: "call",
name: normalized.name,
callId: payload.call_id,
input: normalized.input,
};
}),
z
.union([
@@ -498,12 +564,29 @@ export async function parseRolloutFile(
const outputsByCallId = parsedRecords
.filter((record): record is Extract<ParsedRolloutRecord, { kind: "output" }> => record.kind === "output")
.reduce((map, record) => map.set(record.callId, record.output), new Map<string, unknown>());
const terminalCommandsBySessionId = buildTerminalCommandBySessionId(parsedRecords);
const timeline = parsedRecords.flatMap((record): AgentTimelineItem[] =>
record.kind === "timeline"
? [record.item]
: record.kind === "call"
? (() => {
if (record.name === "write_stdin") {
const input =
record.input && typeof record.input === "object"
? (record.input as { session_id?: unknown; sessionId?: unknown })
: null;
const sessionId =
readTerminalSessionId(input?.session_id) ??
readTerminalSessionId(input?.sessionId);
return [
mapCodexTerminalInteractionToToolCall({
processId: sessionId,
fallbackCallId: record.callId,
command: sessionId ? terminalCommandsBySessionId.get(sessionId) : undefined,
}),
];
}
const mapped = mapCodexRolloutToolCall({
callId: record.callId ?? null,
name: record.name,

10
packages/server/src/server/agent/providers/tool-call-mapper-utils.ts
View File

@@ -72,6 +72,16 @@ export function extractCodexShellOutput(value: string | undefined): string | und
return nonEmptyString(lines.slice(firstBodyLineIndex).join("\n"));
}
export function extractCodexTerminalSessionId(value: string | undefined): string | undefined {
const text = nonEmptyString(value);
if (!text) {
return undefined;
}
const match = text.match(/process running with session id\s+([A-Za-z0-9._:-]+)/i);
return nonEmptyString(match?.[1]);
}
export function flattenReadContent<Chunk extends ReadChunkLike>(
value: string | Chunk | Chunk[] | undefined
): string | undefined {

232
packages/server/src/server/session.ts
View File

@@ -1255,6 +1255,132 @@ export class Session {
})
}
private buildPersistedProjectRecord(input: {
workspaceId: string
placement: ProjectPlacementPayload
createdAt: string
updatedAt: string
}): PersistedProjectRecord {
return createPersistedProjectRecord({
projectId: input.placement.projectKey,
rootPath: deriveProjectRootPath({
cwd: input.workspaceId,
checkout: input.placement.checkout,
}),
kind: deriveProjectKind(input.placement.checkout),
displayName: input.placement.projectName,
createdAt: input.createdAt,
updatedAt: input.updatedAt,
archivedAt: null,
})
}
private buildPersistedWorkspaceRecord(input: {
workspaceId: string
placement: ProjectPlacementPayload
createdAt: string
updatedAt: string
}): PersistedWorkspaceRecord {
return createPersistedWorkspaceRecord({
workspaceId: input.workspaceId,
projectId: input.placement.projectKey,
cwd: input.workspaceId,
kind: deriveWorkspaceKind(input.placement.checkout),
displayName: deriveWorkspaceDisplayName({
cwd: input.workspaceId,
checkout: input.placement.checkout,
}),
createdAt: input.createdAt,
updatedAt: input.updatedAt,
archivedAt: null,
})
}
private async archiveProjectRecordIfEmpty(projectId: string, archivedAt: string): Promise<void> {
const siblingWorkspaces = (await this.workspaceRegistry.list()).filter(
(workspace) => workspace.projectId === projectId && !workspace.archivedAt
)
if (siblingWorkspaces.length === 0) {
await this.projectRegistry.archive(projectId, archivedAt)
}
}
private async reconcileWorkspaceRecord(workspaceId: string): Promise<{
workspace: PersistedWorkspaceRecord
changed: boolean
}> {
const normalizedWorkspaceId = normalizePersistedWorkspaceId(workspaceId)
const existing = await this.workspaceRegistry.get(normalizedWorkspaceId)
const placement = await this.buildProjectPlacement(normalizedWorkspaceId)
const now = new Date().toISOString()
const nextProjectCreatedAt = existing?.createdAt ?? now
const nextWorkspaceCreatedAt = existing?.createdAt ?? now
const currentProjectRecord = await this.projectRegistry.get(placement.projectKey)
const nextProjectRecord = this.buildPersistedProjectRecord({
workspaceId: normalizedWorkspaceId,
placement,
createdAt: currentProjectRecord?.createdAt ?? nextProjectCreatedAt,
updatedAt: now,
})
const nextWorkspaceRecord = this.buildPersistedWorkspaceRecord({
workspaceId: normalizedWorkspaceId,
placement,
createdAt: nextWorkspaceCreatedAt,
updatedAt: now,
})
const needsWorkspaceUpdate =
!existing ||
existing.archivedAt ||
existing.projectId !== nextWorkspaceRecord.projectId ||
existing.kind !== nextWorkspaceRecord.kind ||
existing.displayName !== nextWorkspaceRecord.displayName
const needsProjectUpdate =
!currentProjectRecord ||
currentProjectRecord.archivedAt ||
currentProjectRecord.rootPath !== nextProjectRecord.rootPath ||
currentProjectRecord.kind !== nextProjectRecord.kind ||
currentProjectRecord.displayName !== nextProjectRecord.displayName
if (!needsWorkspaceUpdate && !needsProjectUpdate) {
return {
workspace: existing!,
changed: false,
}
}
await this.projectRegistry.upsert(nextProjectRecord)
await this.workspaceRegistry.upsert(nextWorkspaceRecord)
if (
existing &&
!existing.archivedAt &&
existing.projectId !== nextWorkspaceRecord.projectId
) {
await this.archiveProjectRecordIfEmpty(existing.projectId, now)
}
return {
workspace: nextWorkspaceRecord,
changed: true,
}
}
private async reconcileActiveWorkspaceRecords(): Promise<Set<string>> {
const changedWorkspaceIds = new Set<string>()
const activeWorkspaces = (await this.workspaceRegistry.list()).filter((workspace) => !workspace.archivedAt)
for (const workspace of activeWorkspaces) {
const result = await this.reconcileWorkspaceRecord(workspace.workspaceId)
if (result.changed) {
changedWorkspaceIds.add(result.workspace.workspaceId)
}
}
return changedWorkspaceIds
}
private async forwardAgentUpdate(agent: ManagedAgent): Promise<void> {
try {
await this.ensureWorkspaceRegistered(agent.cwd)
@@ -5275,7 +5401,7 @@ export class Session {
}
}
private async listWorkspaceDescriptors(): Promise<WorkspaceDescriptorPayload[]> {
private async listWorkspaceDescriptorsSnapshot(): Promise<WorkspaceDescriptorPayload[]> {
const [agents, persistedWorkspaces, persistedProjects] = await Promise.all([
this.listAgentPayloads(),
this.workspaceRegistry.list(),
@@ -5318,6 +5444,11 @@ export class Session {
return Array.from(descriptorsByWorkspaceId.values())
}
private async listWorkspaceDescriptors(): Promise<WorkspaceDescriptorPayload[]> {
await this.reconcileActiveWorkspaceRecords()
return this.listWorkspaceDescriptorsSnapshot()
}
private normalizeFetchWorkspacesSort(
sort: FetchWorkspacesRequestSort[] | undefined
): FetchWorkspacesRequestSort[] {
@@ -5597,43 +5728,7 @@ export class Session {
private async ensureWorkspaceRegistered(cwd: string): Promise<PersistedWorkspaceRecord> {
const workspaceId = normalizePersistedWorkspaceId(cwd)
const existing = await this.workspaceRegistry.get(workspaceId)
if (existing && !existing.archivedAt) {
return existing
}
const placement = await this.buildProjectPlacement(workspaceId)
const now = new Date().toISOString()
const projectExisting = await this.projectRegistry.get(placement.projectKey)
const projectRecord: PersistedProjectRecord = createPersistedProjectRecord({
projectId: placement.projectKey,
rootPath: deriveProjectRootPath({
cwd: workspaceId,
checkout: placement.checkout,
}),
kind: deriveProjectKind(placement.checkout),
displayName: placement.projectName,
createdAt: projectExisting?.createdAt ?? now,
updatedAt: now,
archivedAt: null,
})
await this.projectRegistry.upsert(projectRecord)
const workspaceRecord = createPersistedWorkspaceRecord({
workspaceId,
projectId: placement.projectKey,
cwd: workspaceId,
kind: deriveWorkspaceKind(placement.checkout),
displayName: deriveWorkspaceDisplayName({
cwd: workspaceId,
checkout: placement.checkout,
}),
createdAt: existing?.createdAt ?? now,
updatedAt: now,
archivedAt: null,
})
await this.workspaceRegistry.upsert(workspaceRecord)
return workspaceRecord
return (await this.reconcileWorkspaceRecord(workspaceId)).workspace
}
private async archiveWorkspaceRecord(workspaceId: string, archivedAt?: string): Promise<void> {
@@ -5660,28 +5755,26 @@ export class Session {
}
const workspaceId = normalizePersistedWorkspaceId(cwd)
const all = await this.listWorkspaceDescriptors()
const workspace = all.find((entry) => entry.id === workspaceId)
if (!workspace) {
this.bufferOrEmitWorkspaceUpdate(subscription, {
kind: 'remove',
id: workspaceId,
})
return
}
const changedWorkspaceIds = await this.reconcileActiveWorkspaceRecords()
const all = await this.listWorkspaceDescriptorsSnapshot()
const descriptorsByWorkspaceId = new Map(all.map((entry) => [entry.id, entry] as const))
const workspaceIdsToEmit = new Set<string>([workspaceId, ...changedWorkspaceIds])
if (!this.matchesWorkspaceFilter({ workspace, filter: subscription.filter })) {
this.bufferOrEmitWorkspaceUpdate(subscription, {
kind: 'remove',
id: workspaceId,
})
return
}
for (const nextWorkspaceId of workspaceIdsToEmit) {
const workspace = descriptorsByWorkspaceId.get(nextWorkspaceId)
if (!workspace || !this.matchesWorkspaceFilter({ workspace, filter: subscription.filter })) {
this.bufferOrEmitWorkspaceUpdate(subscription, {
kind: 'remove',
id: nextWorkspaceId,
})
continue
}
this.bufferOrEmitWorkspaceUpdate(subscription, {
kind: 'upsert',
workspace,
})
this.bufferOrEmitWorkspaceUpdate(subscription, {
kind: 'upsert',
workspace,
})
}
}
private async emitWorkspaceUpdatesForCwds(cwds: Iterable<string>): Promise<void> {
@@ -5689,7 +5782,8 @@ export class Session {
return
}
const uniqueWorkspaceCwds = new Set<string>()
const changedWorkspaceIds = await this.reconcileActiveWorkspaceRecords()
const uniqueWorkspaceCwds = new Set<string>(changedWorkspaceIds)
for (const cwd of cwds) {
const normalized = normalizePersistedWorkspaceId(cwd)
if (!normalized) {
@@ -5698,8 +5792,24 @@ export class Session {
uniqueWorkspaceCwds.add(normalized)
}
for (const workspaceCwd of uniqueWorkspaceCwds) {
await this.emitWorkspaceUpdateForCwd(workspaceCwd)
const subscription = this.workspaceUpdatesSubscription
const all = await this.listWorkspaceDescriptorsSnapshot()
const descriptorsByWorkspaceId = new Map(all.map((entry) => [entry.id, entry] as const))
for (const workspaceId of uniqueWorkspaceCwds) {
const workspace = descriptorsByWorkspaceId.get(workspaceId)
if (!workspace || !this.matchesWorkspaceFilter({ workspace, filter: subscription.filter })) {
this.bufferOrEmitWorkspaceUpdate(subscription, {
kind: 'remove',
id: workspaceId,
})
continue
}
this.bufferOrEmitWorkspaceUpdate(subscription, {
kind: 'upsert',
workspace,
})
}
}

220
packages/server/src/server/session.workspaces.test.ts
View File

@@ -284,8 +284,9 @@ describe('workspace aggregation', () => {
isBootstrapping: false,
pendingUpdatesByWorkspaceId: new Map(),
}
session.reconcileActiveWorkspaceRecords = async () => new Set()
session.listWorkspaceDescriptors = async () => [
session.listWorkspaceDescriptorsSnapshot = async () => [
{
id: '/tmp/repo',
projectId: '/tmp/repo',
@@ -300,7 +301,7 @@ describe('workspace aggregation', () => {
]
await session.emitWorkspaceUpdateForCwd('/tmp/repo')
session.listWorkspaceDescriptors = async () => [
session.listWorkspaceDescriptorsSnapshot = async () => [
{
id: '/tmp/repo',
projectId: '/tmp/repo',
@@ -335,16 +336,40 @@ describe('workspace aggregation', () => {
})
test('workspace update fanout for multiple cwd values is deduplicated', async () => {
const emitted: Array<{ type: string; payload: unknown }> = []
const session = createSessionForWorkspaceTests() as any
session.emit = (message: any) => emitted.push(message)
session.workspaceUpdatesSubscription = {
subscriptionId: 'sub-dedupe',
filter: undefined,
isBootstrapping: false,
pendingUpdatesByWorkspaceId: new Map(),
}
const emitWorkspaceUpdateForCwd = vi.fn(async () => {})
session.emitWorkspaceUpdateForCwd = emitWorkspaceUpdateForCwd
session.reconcileActiveWorkspaceRecords = async () => new Set()
session.listWorkspaceDescriptorsSnapshot = async () => [
{
id: '/tmp/repo',
projectId: '/tmp/repo',
projectDisplayName: 'repo',
projectRootPath: '/tmp/repo',
projectKind: 'non_git',
workspaceKind: 'directory',
name: 'repo',
status: 'done',
activityAt: null,
},
{
id: '/tmp/repo/sub',
projectId: '/tmp/repo',
projectDisplayName: 'repo',
projectRootPath: '/tmp/repo',
projectKind: 'non_git',
workspaceKind: 'directory',
name: 'sub',
status: 'done',
activityAt: null,
},
]
await session.emitWorkspaceUpdatesForCwds([
'/tmp/repo',
@@ -354,9 +379,12 @@ describe('workspace aggregation', () => {
'/tmp/repo/sub/',
])
expect(emitWorkspaceUpdateForCwd).toHaveBeenCalledTimes(2)
expect(emitWorkspaceUpdateForCwd).toHaveBeenNthCalledWith(1, '/tmp/repo')
expect(emitWorkspaceUpdateForCwd).toHaveBeenNthCalledWith(2, '/tmp/repo/sub')
const workspaceUpdates = emitted.filter((message) => message.type === 'workspace_update') as any[]
expect(workspaceUpdates).toHaveLength(2)
expect(workspaceUpdates.map((message) => message.payload.workspace.id)).toEqual([
'/tmp/repo',
'/tmp/repo/sub',
])
})
test('open_project_request registers a workspace before any agent exists', async () => {
@@ -434,4 +462,180 @@ describe('workspace aggregation', () => {
const response = emitted.find((message) => message.type === 'archive_workspace_response') as any
expect(response?.payload.error).toBeNull()
})
test('opening a new worktree reconciles older local workspaces into the remote project', async () => {
const emitted: Array<{ type: string; payload: unknown }> = []
const session = createSessionForWorkspaceTests() as any
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>()
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>()
const mainWorkspaceId = '/tmp/inkwell'
const worktreeWorkspaceId = '/tmp/inkwell/.paseo/worktrees/feature-a'
const localProjectId = mainWorkspaceId
const remoteProjectId = 'remote:github.com/zimakki/inkwell'
projects.set(
localProjectId,
createPersistedProjectRecord({
projectId: localProjectId,
rootPath: mainWorkspaceId,
kind: 'git',
displayName: 'inkwell',
createdAt: '2026-03-01T12:00:00.000Z',
updatedAt: '2026-03-01T12:00:00.000Z',
})
)
workspaces.set(
mainWorkspaceId,
createPersistedWorkspaceRecord({
workspaceId: mainWorkspaceId,
projectId: localProjectId,
cwd: mainWorkspaceId,
kind: 'local_checkout',
displayName: 'main',
createdAt: '2026-03-01T12:00:00.000Z',
updatedAt: '2026-03-01T12:00:00.000Z',
})
)
session.emit = (message: any) => emitted.push(message)
session.workspaceUpdatesSubscription = {
subscriptionId: 'sub-reconcile',
filter: undefined,
isBootstrapping: false,
pendingUpdatesByWorkspaceId: new Map(),
}
session.listAgentPayloads = async () => []
session.projectRegistry.get = async (projectId: string) => projects.get(projectId) ?? null
session.projectRegistry.list = async () => Array.from(projects.values())
session.projectRegistry.upsert = async (record: ReturnType<typeof createPersistedProjectRecord>) => {
projects.set(record.projectId, record)
}
session.projectRegistry.archive = async (projectId: string, archivedAt: string) => {
const existing = projects.get(projectId)
if (!existing) return
projects.set(projectId, { ...existing, archivedAt, updatedAt: archivedAt })
}
session.workspaceRegistry.get = async (workspaceId: string) => workspaces.get(workspaceId) ?? null
session.workspaceRegistry.list = async () => Array.from(workspaces.values())
session.workspaceRegistry.upsert = async (
record: ReturnType<typeof createPersistedWorkspaceRecord>
) => {
workspaces.set(record.workspaceId, record)
}
session.buildProjectPlacement = async (cwd: string) => ({
projectKey: remoteProjectId,
projectName: 'zimakki/inkwell',
checkout: {
cwd,
isGit: true,
currentBranch: cwd === mainWorkspaceId ? 'main' : 'feature-a',
remoteUrl: 'https://github.com/zimakki/inkwell.git',
isPaseoOwnedWorktree: cwd !== mainWorkspaceId,
mainRepoRoot: cwd === mainWorkspaceId ? null : mainWorkspaceId,
},
})
await session.handleMessage({
type: 'open_project_request',
cwd: worktreeWorkspaceId,
requestId: 'req-open-worktree',
})
expect(workspaces.get(mainWorkspaceId)?.projectId).toBe(remoteProjectId)
expect(workspaces.get(worktreeWorkspaceId)?.projectId).toBe(remoteProjectId)
expect(projects.get(localProjectId)?.archivedAt).toBeTruthy()
const workspaceUpdates = emitted.filter((message) => message.type === 'workspace_update') as any[]
expect(workspaceUpdates).toHaveLength(2)
expect(workspaceUpdates.map((message) => message.payload.workspace.id).sort()).toEqual([
mainWorkspaceId,
worktreeWorkspaceId,
])
expect(
workspaceUpdates.every((message) => message.payload.workspace.projectId === remoteProjectId)
).toBe(true)
})
test('fetch_workspaces_request reconciles remote URL changes for existing workspaces', async () => {
const session = createSessionForWorkspaceTests() as any
const projects = new Map<string, ReturnType<typeof createPersistedProjectRecord>>()
const workspaces = new Map<string, ReturnType<typeof createPersistedWorkspaceRecord>>()
const mainWorkspaceId = '/tmp/inkwell'
const worktreeWorkspaceId = '/tmp/inkwell/.paseo/worktrees/feature-a'
const oldProjectId = 'remote:github.com/old-owner/inkwell'
const newProjectId = 'remote:github.com/new-owner/inkwell'
projects.set(
oldProjectId,
createPersistedProjectRecord({
projectId: oldProjectId,
rootPath: mainWorkspaceId,
kind: 'git',
displayName: 'old-owner/inkwell',
createdAt: '2026-03-01T12:00:00.000Z',
updatedAt: '2026-03-01T12:00:00.000Z',
})
)
for (const [workspaceId, displayName] of [
[mainWorkspaceId, 'main'],
[worktreeWorkspaceId, 'feature-a'],
] as const) {
workspaces.set(
workspaceId,
createPersistedWorkspaceRecord({
workspaceId,
projectId: oldProjectId,
cwd: workspaceId,
kind: workspaceId === mainWorkspaceId ? 'local_checkout' : 'worktree',
displayName,
createdAt: '2026-03-01T12:00:00.000Z',
updatedAt: '2026-03-01T12:00:00.000Z',
})
)
}
session.listAgentPayloads = async () => []
session.projectRegistry.get = async (projectId: string) => projects.get(projectId) ?? null
session.projectRegistry.list = async () => Array.from(projects.values())
session.projectRegistry.upsert = async (record: ReturnType<typeof createPersistedProjectRecord>) => {
projects.set(record.projectId, record)
}
session.projectRegistry.archive = async (projectId: string, archivedAt: string) => {
const existing = projects.get(projectId)
if (!existing) return
projects.set(projectId, { ...existing, archivedAt, updatedAt: archivedAt })
}
session.workspaceRegistry.get = async (workspaceId: string) => workspaces.get(workspaceId) ?? null
session.workspaceRegistry.list = async () => Array.from(workspaces.values())
session.workspaceRegistry.upsert = async (
record: ReturnType<typeof createPersistedWorkspaceRecord>
) => {
workspaces.set(record.workspaceId, record)
}
session.buildProjectPlacement = async (cwd: string) => ({
projectKey: newProjectId,
projectName: 'new-owner/inkwell',
checkout: {
cwd,
isGit: true,
currentBranch: cwd === mainWorkspaceId ? 'main' : 'feature-a',
remoteUrl: 'https://github.com/new-owner/inkwell.git',
isPaseoOwnedWorktree: cwd !== mainWorkspaceId,
mainRepoRoot: cwd === mainWorkspaceId ? null : mainWorkspaceId,
},
})
const result = await session.listFetchWorkspacesEntries({
type: 'fetch_workspaces_request',
requestId: 'req-fetch-reconcile',
})
expect(result.entries.map((entry: any) => entry.projectId)).toEqual([newProjectId, newProjectId])
expect(workspaces.get(mainWorkspaceId)?.projectId).toBe(newProjectId)
expect(workspaces.get(worktreeWorkspaceId)?.projectId).toBe(newProjectId)
expect(projects.get(oldProjectId)?.archivedAt).toBeTruthy()
})
})

34
packages/server/src/shared/tool-call-display.test.ts
View File

@@ -106,4 +106,38 @@ describe("shared tool-call display mapping", () => {
expect(display.errorText).toBe('{\n "message": "boom"\n}');
});
it("labels terminal interaction rows without a summary when no command is available", () => {
const display = buildToolCallDisplayModel({
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
icon: "square_terminal",
},
});
expect(display).toEqual({
displayName: "Interacted with terminal",
});
});
it("uses the command as terminal interaction summary when available", () => {
const display = buildToolCallDisplayModel({
name: "terminal",
status: "completed",
error: null,
detail: {
type: "plain_text",
label: "npm run test",
icon: "square_terminal",
},
});
expect(display).toEqual({
displayName: "Interacted with terminal",
summary: "npm run test",
});
});
});

6
packages/server/src/shared/tool-call-display.ts
View File

@@ -103,6 +103,12 @@ export function buildToolCallDisplayModel(input: ToolCallDisplayInput): ToolCall
summary = isRecord(input.metadata) ? readString(input.metadata.subAgentActivity) : undefined;
} else if (lowerName === "thinking" && input.detail.type === "unknown") {
displayName = "Thinking";
} else if (lowerName === "terminal") {
displayName = "Interacted with terminal";
summary =
input.detail.type === "plain_text"
? readString(input.detail.label)
: undefined;
}
const errorText = input.status === "failed" ? formatErrorText(input.error) : undefined;