diff --git a/packages/app/README.md b/packages/app/README.md index 48dd63ff3..80374dcab 100644 --- a/packages/app/README.md +++ b/packages/app/README.md @@ -48,3 +48,7 @@ Join our community of developers creating universal apps. - [Expo on GitHub](https://github.com/expo/expo): View our open source platform and contribute. - [Discord community](https://chat.expo.dev): Chat with Expo users and ask questions. + +## Dictation debugging + +Set `EXPO_PUBLIC_ENABLE_AUDIO_DEBUG=1` before running `npx expo start` to render the in-app audio debug card. Pair it with the server-side `STT_DEBUG_AUDIO_DIR` flag so every dictation includes a copyable path to the saved raw audio file. diff --git a/packages/app/src/app/agent/[id].tsx b/packages/app/src/app/agent/[id].tsx index 5b4aca215..175bd0e5e 100644 --- a/packages/app/src/app/agent/[id].tsx +++ b/packages/app/src/app/agent/[id].tsx @@ -13,10 +13,11 @@ import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; import ReanimatedAnimated, { useAnimatedStyle, useSharedValue } from "react-native-reanimated"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; -import { MoreVertical, GitBranch, Folder, RotateCcw } from "lucide-react-native"; +import { MoreVertical, GitBranch, Folder, RotateCcw, PlusCircle } from "lucide-react-native"; import { BackHeader } from "@/components/headers/back-header"; import { AgentStreamView } from "@/components/agent-stream-view"; import { AgentInputArea } from "@/components/agent-input-area"; +import { CreateAgentModal, type CreateAgentInitialValues } from "@/components/create-agent-modal"; import { useSession } from "@/contexts/session-context"; import type { Agent } from "@/contexts/session-context"; import { useFooterControls } from "@/contexts/footer-controls-context"; @@ -92,7 +93,16 @@ export default function AgentScreen() { const [branchStatus, setBranchStatus] = useState("idle"); const [branchLabel, setBranchLabel] = useState(null); const [branchError, setBranchError] = useState(null); + const [showCreateAgentModal, setShowCreateAgentModal] = useState(false); + const [createAgentInitialValues, setCreateAgentInitialValues] = + useState(); const repoInfoRequestIdRef = useRef(null); + const hasPendingRepoRequest = repoInfoRequestIdRef.current !== null; + const branchDisplayValue = + branchStatus === "error" + ? branchError ?? "Unavailable" + : branchLabel ?? "Unknown"; + const shouldListenForBranchInfo = menuVisible || hasPendingRepoRequest; // Keyboard animation const { height: keyboardHeight } = useReanimatedKeyboardAnimation(); @@ -161,6 +171,9 @@ export default function AgentScreen() { }, [agent?.cwd, resetBranchState, sendGitRepoInfoRequest]); useEffect(() => { + if (!shouldListenForBranchInfo) { + return; + } const unsubscribe = ws.on("git_repo_info_response", (message) => { if (message.type !== "git_repo_info_response") { return; @@ -195,7 +208,7 @@ export default function AgentScreen() { return () => { unsubscribe(); }; - }, [agent?.cwd, ws]); + }, [agent?.cwd, shouldListenForBranchInfo, ws]); useEffect(() => { if (!id) { @@ -352,164 +365,196 @@ export default function AgentScreen() { refreshAgent({ agentId: id }); }, [handleCloseMenu, id, refreshAgent]); - const branchDisplayValue = - branchStatus === "error" - ? branchError ?? "Unavailable" - : branchLabel ?? "Unknown"; + const handleCreateNewAgent = useCallback(() => { + if (!agent) { + return; + } + handleCloseMenu(); + setCreateAgentInitialValues({ + workingDir: agent.cwd, + provider: agent.provider, + modeId: agent.currentModeId, + model: agentModel ?? undefined, + }); + setShowCreateAgentModal(true); + }, [agent, agentModel, handleCloseMenu]); + + const handleCloseCreateAgentModal = useCallback(() => { + setShowCreateAgentModal(false); + }, []); + + const createAgentModal = ( + + ); + if (!agent) { return ( - - - - Agent not found + <> + + + + Agent not found + - + {createAgentModal} + ); } return ( - - {/* Header */} - - - - - - } - /> - - {/* Content Area with Keyboard Animation */} - - - {isInitializing ? ( - - - Loading agent... + <> + + {/* Header */} + + + + - ) : ( - - respondToPermission(agentId, requestId, response) - } - /> - )} - - + } + /> - {/* Dropdown Menu */} - - - - - - - Directory - - {agent.cwd} - + {/* Content Area with Keyboard Animation */} + + + {isInitializing ? ( + + + Loading agent... + ) : ( + + respondToPermission(agentId, requestId, response) + } + /> + )} + + - - Model - - {modelDisplayValue} - - + {/* Dropdown Menu */} + + + + + + + Directory + + {agent.cwd} + + - - Branch - - {branchStatus === "loading" ? ( - <> - - Fetching… - - ) : ( - - {branchDisplayValue} - - )} + + Model + + {modelDisplayValue} + + + + + Branch + + {branchStatus === "loading" ? ( + <> + + Fetching… + + ) : ( + + {branchDisplayValue} + + )} + + + + + + + View Changes + + + + Browse Files + + + + New Agent + + + + + {isInitializing ? "Refreshing..." : "Refresh"} + + {isInitializing && ( + + )} + - - - - - - View Changes - - - - Browse Files - - - - - {isInitializing ? "Refreshing..." : "Refresh"} - - {isInitializing && ( - - )} - - - - + + + {createAgentModal} + ); } diff --git a/packages/app/src/app/index.tsx b/packages/app/src/app/index.tsx index 30e393f03..de72edbc0 100644 --- a/packages/app/src/app/index.tsx +++ b/packages/app/src/app/index.tsx @@ -1,5 +1,5 @@ import { View } from "react-native"; -import { useState } from "react"; +import { useState, useCallback } from "react"; import { useSafeAreaInsets } from "react-native-safe-area-context"; import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller"; import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated"; @@ -7,7 +7,7 @@ import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { HomeHeader } from "@/components/headers/home-header"; import { EmptyState } from "@/components/empty-state"; import { AgentList } from "@/components/agent-list"; -import { CreateAgentModal } from "@/components/create-agent-modal"; +import { CreateAgentModal, ImportAgentModal } from "@/components/create-agent-modal"; import { useSession } from "@/contexts/session-context"; export default function HomeScreen() { @@ -15,6 +15,9 @@ export default function HomeScreen() { const insets = useSafeAreaInsets(); const { agents } = useSession(); const [showCreateModal, setShowCreateModal] = useState(false); + const [showImportModal, setShowImportModal] = useState(false); + const [createModalMounted, setCreateModalMounted] = useState(false); + const [importModalMounted, setImportModalMounted] = useState(false); // Keyboard animation const { height: keyboardHeight } = useReanimatedKeyboardAnimation(); @@ -29,14 +32,31 @@ export default function HomeScreen() { const hasAgents = agents.size > 0; - function handleCreateAgent() { + const handleCreateAgent = useCallback(() => { + setCreateModalMounted(true); setShowCreateModal(true); - } + }, []); + + const handleImportAgent = useCallback(() => { + setImportModalMounted(true); + setShowImportModal(true); + }, []); + + const handleCloseCreateModal = useCallback(() => { + setShowCreateModal(false); + }, []); + + const handleCloseImportModal = useCallback(() => { + setShowImportModal(false); + }, []); return ( {/* Header */} - + {/* Content Area with Keyboard Animation */} @@ -48,10 +68,12 @@ export default function HomeScreen() { {/* Create Agent Modal */} - setShowCreateModal(false)} - /> + {createModalMounted ? ( + + ) : null} + {importModalMounted ? ( + + ) : null} ); } diff --git a/packages/app/src/components/agent-input-area.tsx b/packages/app/src/components/agent-input-area.tsx index 4fcafb759..94d17e81d 100644 --- a/packages/app/src/components/agent-input-area.tsx +++ b/packages/app/src/components/agent-input-area.tsx @@ -10,7 +10,7 @@ import { Text, ActivityIndicator, } from "react-native"; -import { useState, useEffect, useRef, useLayoutEffect } from "react"; +import { useState, useEffect, useRef, useLayoutEffect, useCallback } from "react"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; import { Mic, ArrowUp, AudioLines, Square, Paperclip, X, Pencil } from "lucide-react-native"; import Animated, { @@ -29,6 +29,14 @@ import { generateMessageId } from "@/types/stream"; import { AgentStatusBar } from "./agent-status-bar"; import { RealtimeControls } from "./realtime-controls"; import { useImageAttachmentPicker } from "@/hooks/use-image-attachment-picker"; +import { AUDIO_DEBUG_ENABLED } from "@/config/audio-debug"; +import { AudioDebugNotice, type AudioDebugInfo } from "./audio-debug-notice"; + +type QueuedMessage = { + id: string; + text: string; + images?: Array<{ uri: string; mimeType: string }>; +}; interface AgentInputAreaProps { agentId: string; @@ -68,8 +76,8 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { sendAgentAudio, agents, cancelAgentRun, - draftInputs, - setDraftInputs, + getDraftInput, + saveDraftInput, queuedMessages: queuedMessagesByAgent, setQueuedMessages: setQueuedMessagesByAgent, } = useSession(); @@ -84,6 +92,7 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { const [recordingDuration, setRecordingDuration] = useState(0); const [transcribingRequestId, setTranscribingRequestId] = useState(null); const [isCancellingAgent, setIsCancellingAgent] = useState(false); + const [audioDebugInfo, setAudioDebugInfo] = useState(null); const recordingIntervalRef = useRef | null>(null); const textInputRef = useRef unknown }) | null>(null); @@ -91,7 +100,15 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { const baselineInputHeightRef = useRef(null); const overlayTransition = useSharedValue(0); const { pickImages } = useImageAttachmentPicker(); - + const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED; + const pendingTranscriptionRef = useRef<{ requestId: string } | null>(null); + const agentIdRef = useRef(agentId); + const sendAgentMessageRef = useRef(sendAgentMessage); + const agentStatusRef = useRef(undefined); + const updateQueueRef = useRef< + ((updater: (current: QueuedMessage[]) => QueuedMessage[]) => void) | null + >(null); + const audioRecorder = useAudioRecorder({ onAudioLevel: (level) => { setRecordingVolume(level); @@ -108,6 +125,14 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { inputHeightRef.current = inputHeight; }, [inputHeight]); + useEffect(() => { + agentIdRef.current = agentId; + }, [agentId]); + + useEffect(() => { + sendAgentMessageRef.current = sendAgentMessage; + }, [sendAgentMessage]); + async function handleSendMessage() { if (!userInput.trim() || !ws.isConnected) return; @@ -153,7 +178,11 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { if (isRealtimeMode) { return; } - + + if (shouldShowAudioDebug) { + setAudioDebugInfo(null); + } + // Start recording try { await audioRecorder.start(); @@ -208,16 +237,20 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { // Generate request ID for tracking transcription const requestId = generateMessageId(); setTranscribingRequestId(requestId); + pendingTranscriptionRef.current = { + requestId, + }; console.log("[AgentInput] Audio recorded:", audioData.size, "bytes", "requestId:", requestId); try { // Send audio to agent for transcription and processing - await sendAgentAudio(agentId, audioData, requestId); + await sendAgentAudio(agentId, audioData, requestId, { mode: "transcribe_only" }); console.log("[AgentInput] Audio sent to agent"); } catch (error) { console.error("[AgentInput] Failed to send audio:", error); // Clear transcribing state on error setTranscribingRequestId(null); + pendingTranscriptionRef.current = null; overlayTransition.value = withTiming(0, { duration: 250 }); } } else { @@ -228,29 +261,81 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { console.error("[AgentInput] Failed to stop recording:", error); setIsRecording(false); setTranscribingRequestId(null); + pendingTranscriptionRef.current = null; overlayTransition.value = withTiming(0, { duration: 250 }); } } - // Listen for transcription completion useEffect(() => { - if (!transcribingRequestId) return; - const unsubscribe = ws.on("transcription_result", (message) => { - if (message.type !== "transcription_result") return; - - // Check if this transcription result matches our request - if (message.payload.requestId === transcribingRequestId) { - console.log("[AgentInput] Transcription completed for requestId:", transcribingRequestId); - setTranscribingRequestId(null); - overlayTransition.value = withTiming(0, { duration: 250 }); + if (message.type !== "transcription_result") { + return; } + + const pending = pendingTranscriptionRef.current; + if (!pending || !message.payload.requestId) { + return; + } + + if (message.payload.requestId !== pending.requestId) { + return; + } + + console.log("[AgentInput] Transcription completed for requestId:", pending.requestId); + pendingTranscriptionRef.current = null; + setTranscribingRequestId(null); + overlayTransition.value = withTiming(0, { duration: 250 }); + + if (shouldShowAudioDebug) { + setAudioDebugInfo({ + requestId: pending.requestId, + transcript: message.payload.text?.trim(), + debugRecordingPath: message.payload.debugRecordingPath ?? undefined, + format: message.payload.format, + byteLength: message.payload.byteLength, + duration: message.payload.duration, + avgLogprob: message.payload.avgLogprob, + isLowConfidence: message.payload.isLowConfidence, + }); + } + + const transcriptText = message.payload.text?.trim(); + if (!transcriptText) { + return; + } + + const shouldQueue = agentStatusRef.current === "running"; + if (shouldQueue) { + updateQueueRef.current?.((current) => [ + ...current, + { + id: generateMessageId(), + text: transcriptText, + }, + ]); + return; + } + + void (async () => { + try { + await sendAgentMessageRef.current?.(agentIdRef.current, transcriptText); + } catch (error) { + console.error("[AgentInput] Failed to send transcribed message:", error); + updateQueueRef.current?.((current) => [ + ...current, + { + id: generateMessageId(), + text: transcriptText, + }, + ]); + } + })(); }); return () => { unsubscribe(); }; - }, [transcribingRequestId, ws, overlayTransition]); + }, [ws, overlayTransition, shouldShowAudioDebug]); // Cleanup timer on unmount useEffect(() => { @@ -412,14 +497,30 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { const agent = agents.get(agentId); const isAgentRunning = agent?.status === "running"; + agentStatusRef.current = agent?.status; const hasText = userInput.trim().length > 0; const hasImages = selectedImages.length > 0; const hasSendableContent = hasText || hasImages; const shouldShowSendButton = !isAgentRunning && hasSendableContent; - const shouldShowVoiceControls = !isAgentRunning && !hasSendableContent; + const shouldShowVoiceControls = !hasSendableContent; const queuedMessages = queuedMessagesByAgent.get(agentId) ?? []; const shouldHandleDesktopSubmit = IS_WEB; + const updateQueue = useCallback( + (updater: (current: QueuedMessage[]) => QueuedMessage[]) => { + setQueuedMessagesByAgent((prev) => { + const next = new Map(prev); + next.set(agentId, updater(prev.get(agentId) ?? [])); + return next; + }); + }, + [agentId, setQueuedMessagesByAgent], + ); + useEffect(() => { + updateQueueRef.current = updateQueue; + }, [updateQueue]); + + useLayoutEffect(() => { if (!IS_WEB) { return; @@ -509,7 +610,7 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { // Hydrate draft only when switching agents useEffect(() => { - const draft = draftInputs.get(agentId); + const draft = getDraftInput(agentId); if (!draft) { setUserInput(""); setSelectedImages([]); @@ -518,28 +619,23 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { setUserInput(draft.text); setSelectedImages(draft.images); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [agentId]); + }, [agentId, getDraftInput]); - // Persist drafts into context with change detection to avoid render loops + // Persist drafts into the shared session store with change detection to avoid redundant work useEffect(() => { - setDraftInputs((prev) => { - const existing = prev.get(agentId); - const isSameText = existing?.text === userInput; - const existingImages = existing?.images ?? []; - const isSameImages = - existingImages.length === selectedImages.length && - existingImages.every((img, idx) => img.uri === selectedImages[idx]?.uri && img.mimeType === selectedImages[idx]?.mimeType); + const existing = getDraftInput(agentId); + const isSameText = existing?.text === userInput; + const existingImages = existing?.images ?? []; + const isSameImages = + existingImages.length === selectedImages.length && + existingImages.every((img, idx) => img.uri === selectedImages[idx]?.uri && img.mimeType === selectedImages[idx]?.mimeType); - if (isSameText && isSameImages) { - return prev; - } + if (isSameText && isSameImages) { + return; + } - const next = new Map(prev); - next.set(agentId, { text: userInput, images: selectedImages }); - return next; - }); - }, [agentId, userInput, selectedImages, setDraftInputs]); + saveDraftInput(agentId, { text: userInput, images: selectedImages }); + }, [agentId, userInput, selectedImages, getDraftInput, saveDraftInput]); const overlayAnimatedStyle = useAnimatedStyle(() => { return { @@ -582,14 +678,6 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { cancelAgentRun(agentId); } - function updateQueue(updater: (current: typeof queuedMessages) => typeof queuedMessages) { - setQueuedMessagesByAgent((prev) => { - const next = new Map(prev); - next.set(agentId, updater(prev.get(agentId) ?? [])); - return next; - }); - } - function handleQueueCurrentInput() { if (!hasSendableContent) return; @@ -643,6 +731,24 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { ); + const voiceButton = ( + + {isRecording ? ( + + ) : ( + + )} + + ); + return ( {/* Border separator */} @@ -664,6 +770,12 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { {/* Regular input controls */} + {shouldShowAudioDebug && audioDebugInfo ? ( + setAudioDebugInfo(null)} + /> + ) : null} {/* Queue list */} {queuedMessages.length > 0 && ( @@ -741,6 +853,7 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) { {isAgentRunning ? ( <> + {shouldShowVoiceControls && voiceButton} {hasSendableContent && ( - ) : shouldShowVoiceControls ? ( - <> - - {isRecording ? ( - - ) : ( - - )} - - {realtimeButton} - - ) : null} + ) : shouldShowVoiceControls ? ( + <> + {voiceButton} + {realtimeButton} + + ) : null} diff --git a/packages/app/src/components/audio-debug-notice.tsx b/packages/app/src/components/audio-debug-notice.tsx new file mode 100644 index 000000000..42b4be6a2 --- /dev/null +++ b/packages/app/src/components/audio-debug-notice.tsx @@ -0,0 +1,187 @@ +import { useState, useMemo } from "react"; +import { View, Text, Pressable } from "react-native"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import * as Clipboard from "expo-clipboard"; +import { Check, Copy, X } from "lucide-react-native"; + +export interface AudioDebugInfo { + requestId?: string | null; + transcript?: string; + debugRecordingPath?: string; + format?: string; + byteLength?: number; + duration?: number; + avgLogprob?: number; + isLowConfidence?: boolean; +} + +interface AudioDebugNoticeProps { + info: AudioDebugInfo | null; + onDismiss?: () => void; + title?: string; +} + +function formatBytes(bytes?: number): string | null { + if (!bytes || bytes <= 0) { + return null; + } + if (bytes < 1024) { + return `${bytes} B`; + } + const units = ["KB", "MB", "GB"] as const; + let value = bytes; + let unitIndex = -1; + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024; + unitIndex += 1; + } + return `${value.toFixed(value >= 100 ? 0 : 1)} ${units[unitIndex]}`; +} + +function formatDuration(duration?: number): string | null { + if (!duration || duration <= 0) { + return null; + } + const seconds = duration / 1000; + if (seconds < 1) { + return `${(seconds * 1000).toFixed(0)} ms`; + } + return `${seconds.toFixed(seconds >= 10 ? 0 : 1)} s`; +} + +export function AudioDebugNotice({ info, onDismiss, title = "Dictation Debug" }: AudioDebugNoticeProps) { + const { theme } = useUnistyles(); + const [copied, setCopied] = useState(false); + + const stats = useMemo(() => { + if (!info) { + return null; + } + const parts: string[] = []; + if (info.format) { + parts.push(info.format); + } + const size = formatBytes(info.byteLength); + if (size) { + parts.push(size); + } + const duration = formatDuration(info.duration); + if (duration) { + parts.push(duration); + } + if (info.avgLogprob !== undefined) { + const label = `${info.avgLogprob.toFixed(2)} avg logprob`; + parts.push(info.isLowConfidence ? `${label} (low confidence)` : label); + } + return parts.join(" · "); + }, [info]); + + if (!info) { + return null; + } + + const handleCopyPath = async () => { + if (!info.debugRecordingPath) { + return; + } + try { + await Clipboard.setStringAsync(info.debugRecordingPath); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + } catch (error) { + console.warn("[AudioDebug] Failed to copy path", error); + } + }; + + const pathMissing = !info.debugRecordingPath; + + return ( + + + {title} + {onDismiss ? ( + + + + ) : null} + + + {info.debugRecordingPath ? ( + + + {info.debugRecordingPath} + + + {copied ? ( + + ) : ( + + )} + + + ) : ( + + Raw audio path unavailable. Set STT_DEBUG_AUDIO_DIR on the server to persist recordings. + + )} + + {stats ? ( + + {stats} + + ) : null} + + ); +} + +const styles = StyleSheet.create({ + container: { + borderRadius: 10, + borderWidth: StyleSheet.hairlineWidth, + padding: 12, + gap: 8, + }, + headerRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + }, + title: { + fontSize: 12, + fontWeight: "600", + textTransform: "uppercase", + }, + pathRow: { + flexDirection: "row", + alignItems: "center", + gap: 8, + }, + pathText: { + flex: 1, + fontSize: 13, + }, + copyPill: { + width: 28, + height: 28, + borderRadius: 14, + alignItems: "center", + justifyContent: "center", + }, + stats: { + fontSize: 12, + }, + hint: { + fontSize: 12, + }, +}); diff --git a/packages/app/src/components/create-agent-modal.tsx b/packages/app/src/components/create-agent-modal.tsx index 401fd194a..5051ccd67 100644 --- a/packages/app/src/components/create-agent-modal.tsx +++ b/packages/app/src/components/create-agent-modal.tsx @@ -35,13 +35,17 @@ import Animated, { Easing, runOnJS, } from "react-native-reanimated"; -import { StyleSheet } from "react-native-unistyles"; -import { X, ChevronDown } from "lucide-react-native"; +import { StyleSheet, useUnistyles } from "react-native-unistyles"; +import { Mic, Check, X, ChevronDown } from "lucide-react-native"; import { theme as defaultTheme } from "@/styles/theme"; import { useRecentPaths } from "@/hooks/use-recent-paths"; import { useSession } from "@/contexts/session-context"; import { useRouter } from "expo-router"; import { generateMessageId } from "@/types/stream"; +import { useAudioRecorder } from "@/hooks/use-audio-recorder"; +import { VolumeMeter } from "@/components/volume-meter"; +import { AUDIO_DEBUG_ENABLED } from "@/config/audio-debug"; +import { AudioDebugNotice, type AudioDebugInfo } from "./audio-debug-notice"; import { AGENT_PROVIDER_DEFINITIONS, type AgentProviderDefinition, @@ -56,9 +60,24 @@ import type { } from "@server/server/agent/agent-sdk-types"; import type { WSInboundMessage } from "@server/server/messages"; -interface CreateAgentModalProps { +export type CreateAgentInitialValues = { + workingDir?: string; + provider?: AgentProvider; + modeId?: string | null; + model?: string | null; +}; + +interface AgentFlowModalProps { isVisible: boolean; onClose: () => void; + flow: "create" | "import"; + initialValues?: CreateAgentInitialValues; +} + +interface ModalWrapperProps { + isVisible: boolean; + onClose: () => void; + initialValues?: CreateAgentInitialValues; } const providerDefinitions = AGENT_PROVIDER_DEFINITIONS; @@ -71,11 +90,12 @@ const DEFAULT_PROVIDER: AgentProvider = fallbackDefinition?.id ?? "claude"; const DEFAULT_MODE_FOR_DEFAULT_PROVIDER = fallbackDefinition?.defaultModeId ?? ""; const BACKDROP_OPACITY = 0.55; -const RESUME_PAGE_SIZE = 20; +const IMPORT_PAGE_SIZE = 20; const PROMPT_MIN_HEIGHT = 64; const PROMPT_MAX_HEIGHT = 200; const FORM_PREFERENCES_STORAGE_KEY = "@paseo:create-agent-preferences"; const IS_WEB = Platform.OS === "web"; +const DICTATION_AGENT_ID = "__dictation__"; type WebTextInputKeyPressEvent = NativeSyntheticEvent< TextInputKeyPressEventData & { @@ -109,7 +129,7 @@ const isTextAreaLike = (node: unknown): node is TextAreaHandle => { return true; }; -type ResumeCandidate = { +type ImportCandidate = { provider: AgentProvider; sessionId: string; cwd: string; @@ -119,7 +139,6 @@ type ResumeCandidate = { timeline: AgentTimelineItem[]; }; -type ResumeTab = "new" | "resume"; type ProviderFilter = "all" | AgentProvider; type DropdownKey = "assistant" | "permissions" | "model" | "workingDir" | "baseBranch"; @@ -152,7 +171,7 @@ function formatRelativeTime(date: Date): string { return `${days}d ago`; } -function getResumePreview(candidate: ResumeCandidate): string { +function getImportPreview(candidate: ImportCandidate): string { for (const item of candidate.timeline) { if (item.type === "user_message") { const text = item.text.trim(); @@ -164,10 +183,12 @@ function getResumePreview(candidate: ResumeCandidate): string { return candidate.title || candidate.cwd; } -export function CreateAgentModal({ +function AgentFlowModal({ isVisible, onClose, -}: CreateAgentModalProps) { + flow, + initialValues, +}: AgentFlowModalProps) { const insets = useSafeAreaInsets(); const { height: screenHeight, width: screenWidth } = useWindowDimensions(); const slideOffset = useSharedValue(screenHeight); @@ -176,16 +197,20 @@ export function CreateAgentModal({ const isCompactLayout = screenWidth < 720; const shouldHandlePromptDesktopSubmit = IS_WEB; const shouldAutoFocusPrompt = IS_WEB; + const isImportFlow = flow === "import"; + const isCreateFlow = !isImportFlow; const { recentPaths, addRecentPath } = useRecentPaths(); const { ws, createAgent, resumeAgent, + sendAgentAudio, agents, providerModels, requestProviderModels, } = useSession(); + const isWsConnected = ws.isConnected; const router = useRouter(); const [isMounted, setIsMounted] = useState(isVisible); @@ -207,15 +232,18 @@ export function CreateAgentModal({ const [worktreeSlugEdited, setWorktreeSlugEdited] = useState(false); const [errorMessage, setErrorMessage] = useState(""); const [isLoading, setIsLoading] = useState(false); - const [activeTab, setActiveTab] = useState("new"); - const [resumeProviderFilter, setResumeProviderFilter] = + const [importProviderFilter, setImportProviderFilter] = useState("all"); - const [resumeSearchQuery, setResumeSearchQuery] = useState(""); - const [resumeCandidates, setResumeCandidates] = useState( + const [importSearchQuery, setImportSearchQuery] = useState(""); + const [importCandidates, setImportCandidates] = useState( [] ); - const [isResumeLoading, setIsResumeLoading] = useState(false); - const [resumeError, setResumeError] = useState(null); + const [isImportLoading, setIsImportLoading] = useState(false); + const [importError, setImportError] = useState(null); + const [isDictating, setIsDictating] = useState(false); + const [isDictationProcessing, setIsDictationProcessing] = useState(false); + const [dictationVolume, setDictationVolume] = useState(0); + const [dictationDebugInfo, setDictationDebugInfo] = useState(null); const [openDropdown, setOpenDropdown] = useState(null); const [repoInfo, setRepoInfo] = useState(null); const [repoInfoStatus, setRepoInfoStatus] = useState< @@ -237,6 +265,19 @@ export function CreateAgentModal({ model: false, workingDir: false, }); + const prevVisibilityRef = useRef(isVisible); + const dictationRequestIdRef = useRef(null); + const dictationRecorder = useAudioRecorder({ + onAudioLevel: (level) => { + setDictationVolume(level); + }, + }); + const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED; + const hasPendingCreateOrResume = pendingRequestIdRef.current !== null; + const hasPendingDictation = dictationRequestIdRef.current !== null; + const shouldListenForStatus = isVisible || hasPendingCreateOrResume; + const shouldListenForDictation = + isCreateFlow && (isVisible || hasPendingDictation); const pendingNavigationAgentIdRef = useRef(null); const openDropdownSheet = useCallback((key: DropdownKey) => { @@ -276,6 +317,41 @@ export function CreateAgentModal({ [setWorkingDirFromUser] ); + const applyInitialValues = useCallback(() => { + if (!isCreateFlow || !initialValues) { + return; + } + + if (Object.prototype.hasOwnProperty.call(initialValues, "workingDir")) { + const providedWorkingDir = initialValues.workingDir ?? ""; + userEditedPreferencesRef.current.workingDir = true; + setWorkingDir(providedWorkingDir); + } + + if (initialValues.provider && providerDefinitionMap.has(initialValues.provider)) { + userEditedPreferencesRef.current.provider = true; + setSelectedProvider(initialValues.provider); + } + + if (typeof initialValues.modeId === "string" && initialValues.modeId.length > 0) { + userEditedPreferencesRef.current.mode = true; + setSelectedMode(initialValues.modeId); + } + + if (typeof initialValues.model === "string" && initialValues.model.length > 0) { + userEditedPreferencesRef.current.model = true; + setSelectedModel(initialValues.model); + } + }, [initialValues, isCreateFlow]); + + useEffect(() => { + const wasVisible = prevVisibilityRef.current; + if (isVisible && !wasVisible) { + applyInitialValues(); + } + prevVisibilityRef.current = isVisible; + }, [applyInitialValues, isVisible]); + const refreshProviderModels = useCallback(() => { const trimmed = workingDir.trim(); requestProviderModels(selectedProvider, { @@ -373,13 +449,6 @@ export function CreateAgentModal({ void persist(); }, [selectedMode, selectedProvider, workingDir]); - const tabOptions = useMemo( - () => [ - { id: "new" as ResumeTab, label: "New Agent" }, - { id: "resume" as ResumeTab, label: "Resume Agent" }, - ], - [] - ); const providerFilterOptions = useMemo( () => [ { id: "all" as ProviderFilter, label: "All" }, @@ -403,6 +472,9 @@ export function CreateAgentModal({ const modelError = modelState?.error ?? null; useEffect(() => { + if (!isVisible) { + return; + } const trimmed = workingDir.trim(); const currentState = providerModels.get(selectedProvider); if (currentState?.models?.length || currentState?.isLoading) { @@ -411,7 +483,7 @@ export function CreateAgentModal({ requestProviderModels(selectedProvider, { cwd: trimmed.length > 0 ? trimmed : undefined, }); - }, [providerModels, requestProviderModels, selectedProvider, workingDir]); + }, [isVisible, providerModels, requestProviderModels, selectedProvider, workingDir]); const setPromptHeight = useCallback((nextHeight: number) => { const bounded = Math.min( PROMPT_MAX_HEIGHT, @@ -538,10 +610,10 @@ export function CreateAgentModal({ }); return ids; }, [agents]); - const filteredResumeCandidates = useMemo(() => { - const providerFilter = resumeProviderFilter; - const query = resumeSearchQuery.trim().toLowerCase(); - return resumeCandidates + const filteredImportCandidates = useMemo(() => { + const providerFilter = importProviderFilter; + const query = importSearchQuery.trim().toLowerCase(); + return importCandidates .filter((candidate) => !activeSessionIds.has(candidate.sessionId)) .filter( (candidate) => @@ -558,9 +630,9 @@ export function CreateAgentModal({ .sort((a, b) => b.lastActivityAt.getTime() - a.lastActivityAt.getTime()); }, [ activeSessionIds, - resumeCandidates, - resumeProviderFilter, - resumeSearchQuery, + importCandidates, + importProviderFilter, + importSearchQuery, ]); useEffect(() => { @@ -595,11 +667,11 @@ export function CreateAgentModal({ if (!shouldAutoFocusPrompt) { return; } - if (!isVisible || activeTab !== "new") { + if (!isVisible || !isCreateFlow) { return; } focusPromptInput(); - }, [activeTab, focusPromptInput, isVisible, shouldAutoFocusPrompt]); + }, [focusPromptInput, isCreateFlow, isVisible, shouldAutoFocusPrompt]); const resetFormState = useCallback(() => { setInitialPrompt(""); @@ -622,8 +694,74 @@ export function CreateAgentModal({ pendingRequestIdRef.current = null; repoInfoRequestIdRef.current = null; shouldSyncBaseBranchRef.current = true; + dictationRequestIdRef.current = null; + setIsDictating(false); + setIsDictationProcessing(false); + setDictationVolume(0); + setDictationDebugInfo(null); + if (dictationRecorder.isRecording?.()) { + void dictationRecorder.stop().catch(() => {}); + } }, [setPromptHeight]); + const handleDictationStart = useCallback(async () => { + if (!isCreateFlow || isLoading || !isWsConnected || isDictating || isDictationProcessing) { + return; + } + try { + if (shouldShowAudioDebug) { + setDictationDebugInfo(null); + } + await dictationRecorder.start(); + setIsDictating(true); + setDictationVolume(0); + setErrorMessage(""); + } catch (error) { + console.error("[CreateAgentModal] Failed to start dictation:", error); + } + }, [dictationRecorder, isCreateFlow, isDictating, isDictationProcessing, isLoading, isWsConnected, shouldShowAudioDebug]); + + const handleDictationCancel = useCallback(async () => { + if (!isDictating) { + return; + } + try { + await dictationRecorder.stop(); + } catch (error) { + console.error("[CreateAgentModal] Failed to cancel dictation:", error); + } finally { + setIsDictating(false); + setDictationVolume(0); + } + }, [dictationRecorder, isDictating]); + + const handleDictationConfirm = useCallback(async () => { + if (!isDictating || isDictationProcessing) { + return; + } + try { + const audioData = await dictationRecorder.stop(); + setIsDictating(false); + setDictationVolume(0); + if (!audioData) { + return; + } + const requestId = generateMessageId(); + dictationRequestIdRef.current = requestId; + setIsDictationProcessing(true); + await sendAgentAudio( + DICTATION_AGENT_ID, + audioData, + requestId, + { mode: "transcribe_only" } + ); + } catch (error) { + console.error("[CreateAgentModal] Failed to complete dictation:", error); + dictationRequestIdRef.current = null; + setIsDictationProcessing(false); + } + }, [dictationRecorder, isDictating, isDictationProcessing, sendAgentAudio]); + const navigateToAgentIfNeeded = useCallback(() => { const agentId = pendingNavigationAgentIdRef.current; if (!agentId) { @@ -643,16 +781,16 @@ export function CreateAgentModal({ navigateToAgentIfNeeded(); }, [navigateToAgentIfNeeded, resetFormState]); - const requestResumeCandidates = useCallback( + const requestImportCandidates = useCallback( (provider?: AgentProvider) => { - setIsResumeLoading(true); - setResumeError(null); + setIsImportLoading(true); + setImportError(null); const msg: WSInboundMessage = { type: "session", message: { type: "list_persisted_agents_request", ...(provider ? { provider } : {}), - limit: RESUME_PAGE_SIZE, + limit: IMPORT_PAGE_SIZE, }, }; try { @@ -662,8 +800,8 @@ export function CreateAgentModal({ "[CreateAgentModal] Failed to request persisted agents:", error ); - setIsResumeLoading(false); - setResumeError("Unable to load saved agents. Please try again."); + setIsImportLoading(false); + setImportError("Unable to load agents to import. Please try again."); } }, [ws] @@ -736,6 +874,23 @@ export function CreateAgentModal({ handleCloseAnimationComplete, ]); + useEffect(() => { + if (isVisible || !dictationRecorder.isRecording?.()) { + return; + } + void dictationRecorder.stop().catch(() => {}); + setIsDictating(false); + setDictationVolume(0); + }, [dictationRecorder, isVisible]); + + useEffect(() => { + return () => { + if (dictationRecorder.isRecording?.()) { + void dictationRecorder.stop().catch(() => {}); + } + }; + }, [dictationRecorder]); + const footerAnimatedStyle = useAnimatedStyle(() => { "worklet"; const absoluteHeight = Math.abs(keyboardHeight.value); @@ -779,10 +934,10 @@ export function CreateAgentModal({ }; } - if (!/^[a-z0-9-]+$/.test(name)) { + if (!/^[a-z0-9-/]+$/.test(name)) { return { valid: false, - error: "Must contain only lowercase letters, numbers, and hyphens", + error: "Must contain only lowercase letters, numbers, hyphens, and forward slashes", }; } @@ -847,14 +1002,23 @@ export function CreateAgentModal({ ); useEffect(() => { + if (!isCreateFlow || !isVisible) { + return; + } shouldSyncBaseBranchRef.current = true; - }, [workingDir]); + }, [isCreateFlow, isVisible, workingDir]); useEffect(() => { + if (!isCreateFlow || !isVisible) { + return; + } requestRepoInfo(workingDir); - }, [requestRepoInfo, workingDir]); + }, [isCreateFlow, isVisible, requestRepoInfo, workingDir]); useEffect(() => { + if (!isCreateFlow || !isVisible) { + return; + } const unsubscribe = ws.on("git_repo_info_response", (message) => { if (message.type !== "git_repo_info_response") { return; @@ -899,7 +1063,7 @@ export function CreateAgentModal({ return () => { unsubscribe(); }; - }, [ws]); + }, [isCreateFlow, isVisible, ws]); const handleCreate = useCallback(async () => { const trimmedPath = workingDir.trim(); @@ -1000,8 +1164,8 @@ export function CreateAgentModal({ createAgent, ]); - const handleResumeCandidatePress = useCallback( - (candidate: ResumeCandidate) => { + const handleImportCandidatePress = useCallback( + (candidate: ImportCandidate) => { if (isLoading) { return; } @@ -1017,16 +1181,16 @@ export function CreateAgentModal({ [isLoading, resumeAgent] ); - const renderResumeItem = useCallback>( + const renderImportItem = useCallback>( ({ item }) => ( handleResumeCandidatePress(item)} + onPress={() => handleImportCandidatePress(item)} disabled={isLoading} style={styles.resumeItem} > - {getResumePreview(item)} + {getImportPreview(item)} {formatRelativeTime(item.lastActivityAt)} @@ -1041,14 +1205,17 @@ export function CreateAgentModal({ {getProviderLabel(item.provider)} - Tap to resume + Tap to import ), - [getProviderLabel, handleResumeCandidatePress, isLoading] + [getProviderLabel, handleImportCandidatePress, isLoading] ); useEffect(() => { + if (!isImportFlow || !isVisible) { + return; + } const unsubscribe = ws.on("list_persisted_agents_response", (message) => { if (message.type !== "list_persisted_agents_response") { return; @@ -1061,18 +1228,21 @@ export function CreateAgentModal({ lastActivityAt: new Date(item.lastActivityAt), persistence: item.persistence, timeline: item.timeline ?? [], - })) as ResumeCandidate[]; + })) as ImportCandidate[]; - setResumeCandidates(mapped); - setIsResumeLoading(false); + setImportCandidates(mapped); + setIsImportLoading(false); }); return () => { unsubscribe(); }; - }, [ws]); + }, [isImportFlow, isVisible, ws]); useEffect(() => { + if (!shouldListenForStatus) { + return; + } const unsubscribe = ws.on("status", (message) => { if (message.type !== "status") { return; @@ -1122,24 +1292,81 @@ export function CreateAgentModal({ return () => { unsubscribe(); }; - }, [ws, handleClose]); + }, [handleClose, shouldListenForStatus, ws]); useEffect(() => { - if (!isVisible || activeTab !== "resume") { + if (!shouldListenForDictation) { + return; + } + const unsubscribe = ws.on("transcription_result", (message) => { + if (message.type !== "transcription_result") { + return; + } + const pendingId = dictationRequestIdRef.current; + if (!pendingId || message.payload.requestId !== pendingId) { + return; + } + dictationRequestIdRef.current = null; + setIsDictationProcessing(false); + if (shouldShowAudioDebug) { + setDictationDebugInfo({ + requestId: pendingId, + transcript: message.payload.text?.trim(), + debugRecordingPath: message.payload.debugRecordingPath ?? undefined, + format: message.payload.format, + byteLength: message.payload.byteLength, + duration: message.payload.duration, + avgLogprob: message.payload.avgLogprob, + isLowConfidence: message.payload.isLowConfidence, + }); + } + const transcriptText = message.payload.text?.trim(); + if (!transcriptText) { + return; + } + setInitialPrompt((prev) => { + if (!prev) { + return transcriptText; + } + const needsSpace = /\s$/.test(prev); + return `${prev}${needsSpace ? "" : " "}${transcriptText}`; + }); + focusPromptInput(); + }); + + return () => { + unsubscribe(); + }; + }, [focusPromptInput, shouldListenForDictation, ws, shouldShowAudioDebug]); + + useEffect(() => { + if (!isVisible || !isImportFlow) { return; } const provider = - resumeProviderFilter === "all" ? undefined : resumeProviderFilter; - requestResumeCandidates(provider); - }, [activeTab, isVisible, requestResumeCandidates, resumeProviderFilter]); + importProviderFilter === "all" ? undefined : importProviderFilter; + requestImportCandidates(provider); + }, [importProviderFilter, isImportFlow, isVisible, requestImportCandidates]); - const refreshResumeList = useCallback(() => { + const refreshImportList = useCallback(() => { const provider = - resumeProviderFilter === "all" ? undefined : resumeProviderFilter; - requestResumeCandidates(provider); - }, [requestResumeCandidates, resumeProviderFilter]); + importProviderFilter === "all" ? undefined : importProviderFilter; + requestImportCandidates(provider); + }, [requestImportCandidates, importProviderFilter]); const shouldRender = isVisible || isMounted; + const modalTitle = isImportFlow ? "Import Agent" : "Create New Agent"; + const dictationAccessory = isCreateFlow ? ( + + ) : null; const gitBlockingError = useMemo(() => { const trimmedBase = baseBranch.trim(); @@ -1158,7 +1385,7 @@ export function CreateAgentModal({ const validation = validateWorktreeName(slug); if (!slug || !validation.valid) { return `Invalid branch name: ${ - validation.error ?? "Must use lowercase letters, numbers, or hyphens" + validation.error ?? "Must use lowercase letters, numbers, hyphens, or forward slashes" }`; } } @@ -1273,44 +1500,9 @@ export function CreateAgentModal({ paddingLeft={horizontalPaddingLeft} paddingRight={horizontalPaddingRight} onClose={handleClose} - title={ - activeTab === "resume" ? "Resume Agent" : "Create New Agent" - } + title={modalTitle} /> - - {tabOptions.map((tab) => { - const isActive = activeTab === tab.id; - return ( - setActiveTab(tab.id)} - style={[ - styles.tabButton, - isActive && styles.tabButtonActive, - ]} - > - - {tab.label} - - - ); - })} - - - {activeTab === "new" ? ( + {isCreateFlow ? ( <> = PROMPT_MAX_HEIGHT} + accessory={dictationAccessory} /> + {shouldShowAudioDebug && dictationDebugInfo ? ( + setDictationDebugInfo(null)} + title="Prompt Dictation Debug" + /> + ) : null} + {providerFilterOptions.map((option) => { - const isActive = resumeProviderFilter === option.id; + const isActive = importProviderFilter === option.id; return ( setResumeProviderFilter(option.id)} + onPress={() => setImportProviderFilter(option.id)} style={[ styles.providerFilterButton, isActive && styles.providerFilterButtonActive, @@ -1529,48 +1730,48 @@ export function CreateAgentModal({ style={styles.resumeSearchInput} placeholder="Search by title or path" placeholderTextColor={defaultTheme.colors.mutedForeground} - value={resumeSearchQuery} - onChangeText={setResumeSearchQuery} + value={importSearchQuery} + onChangeText={setImportSearchQuery} /> Refresh - {resumeError ? ( - {resumeError} + {importError ? ( + {importError} ) : null} - {isResumeLoading ? ( + {isImportLoading ? ( - Loading saved agents... + Loading agents to import... - ) : filteredResumeCandidates.length === 0 ? ( + ) : filteredImportCandidates.length === 0 ? ( - No agents found + No agents to import We'll load the latest Claude and Codex sessions from your - local history. + local history so you can import them. Try Again ) : ( `${item.provider}:${item.sessionId}` } @@ -1590,6 +1791,14 @@ export function CreateAgentModal({ ); } +export function CreateAgentModal(props: ModalWrapperProps) { + return ; +} + +export function ImportAgentModal(props: ModalWrapperProps) { + return ; +} + interface ModalHeaderProps { paddingTop: number; paddingLeft: number; @@ -2078,6 +2287,7 @@ interface PromptSectionProps { onDesktopSubmit?: (event: WebTextInputKeyPressEvent) => void; autoFocus?: boolean; scrollEnabled: boolean; + accessory?: ReactNode; } function PromptSection({ @@ -2090,10 +2300,14 @@ function PromptSection({ onDesktopSubmit, autoFocus, scrollEnabled, + accessory, }: PromptSectionProps): ReactElement { return ( - Initial Prompt + + Initial Prompt + {accessory} + void; + onCancel: () => void; + onConfirm: () => void; +} + +function PromptDictationControls({ + isRecording, + isProcessing, + disabled, + volume, + onStart, + onCancel, + onConfirm, +}: PromptDictationControlsProps): ReactElement { + const { theme } = useUnistyles(); + + if (!isRecording && !isProcessing) { + return ( + + + + ); + } + + return ( + + + + + + + + + + {isProcessing ? ( + + ) : ( + + )} + + + + ); +} + interface GitOptionsSectionProps { baseBranch: string; onBaseBranchChange: (value: string) => void; @@ -2401,35 +2695,6 @@ const styles = StyleSheet.create(((theme: any) => ({ alignItems: "center", justifyContent: "center", }, - tabSelector: { - flexDirection: "row", - gap: theme.spacing[2], - paddingTop: theme.spacing[4], - paddingBottom: theme.spacing[4], - }, - tabButton: { - flex: 1, - borderRadius: theme.borderRadius.lg, - borderWidth: theme.borderWidth[1], - borderColor: theme.colors.border, - paddingVertical: theme.spacing[3], - alignItems: "center", - justifyContent: "center", - backgroundColor: theme.colors.background, - }, - tabButtonActive: { - backgroundColor: theme.colors.muted, - borderColor: theme.colors.palette.blue[500], - }, - tabButtonText: { - color: theme.colors.mutedForeground, - fontSize: theme.fontSize.sm, - fontWeight: theme.fontWeight.normal, - }, - tabButtonTextActive: { - color: theme.colors.foreground, - fontWeight: theme.fontWeight.semibold, - }, scroll: { flex: 1, }, @@ -2441,6 +2706,12 @@ const styles = StyleSheet.create(((theme: any) => ({ formSection: { gap: theme.spacing[3], }, + labelRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + gap: theme.spacing[3], + }, label: { color: theme.colors.foreground, fontSize: theme.fontSize.sm, @@ -2587,6 +2858,50 @@ const styles = StyleSheet.create(((theme: any) => ({ color: theme.colors.mutedForeground, fontSize: theme.fontSize.sm, }, + dictationButton: { + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + borderRadius: theme.borderRadius.full, + padding: theme.spacing[2], + alignItems: "center", + justifyContent: "center", + }, + dictationButtonDisabled: { + opacity: theme.opacity[50], + }, + dictationActiveContainer: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + dictationMeterWrapper: { + width: 48, + alignItems: "center", + justifyContent: "center", + }, + dictationActionGroup: { + flexDirection: "row", + gap: theme.spacing[2], + }, + dictationActionButton: { + width: 32, + height: 32, + borderRadius: theme.borderRadius.full, + alignItems: "center", + justifyContent: "center", + borderWidth: theme.borderWidth[1], + }, + dictationActionButtonCancel: { + borderColor: theme.colors.border, + backgroundColor: theme.colors.background, + }, + dictationActionButtonConfirm: { + borderColor: theme.colors.foreground, + backgroundColor: theme.colors.foreground, + }, + dictationActionButtonDisabled: { + opacity: theme.opacity[40], + }, selectorRow: { flexDirection: "row", gap: theme.spacing[4], @@ -2727,7 +3042,7 @@ const styles = StyleSheet.create(((theme: any) => ({ fontSize: theme.fontSize.sm, fontWeight: theme.fontWeight.semibold, }, - resumeErrorText: { + importErrorText: { color: theme.colors.palette.red[500], fontSize: theme.fontSize.sm, }, diff --git a/packages/app/src/components/headers/home-header.tsx b/packages/app/src/components/headers/home-header.tsx index b1cfe6b87..ecc9ca15f 100644 --- a/packages/app/src/components/headers/home-header.tsx +++ b/packages/app/src/components/headers/home-header.tsx @@ -1,40 +1,81 @@ -import { Pressable } from "react-native"; +import { useCallback, useState } from "react"; +import { Modal, Pressable, Text, View } from "react-native"; import { router } from "expo-router"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; import { StyleSheet, useUnistyles } from "react-native-unistyles"; -import { Settings, MessageSquare, Plus } from "lucide-react-native"; +import { Settings, MessageSquare, MoreVertical, Plus } from "lucide-react-native"; import { ScreenHeader } from "./screen-header"; interface HomeHeaderProps { onCreateAgent: () => void; + onImportAgent: () => void; } -export function HomeHeader({ onCreateAgent }: HomeHeaderProps) { +export function HomeHeader({ onCreateAgent, onImportAgent }: HomeHeaderProps) { const { theme } = useUnistyles(); + const insets = useSafeAreaInsets(); + const [isMenuVisible, setIsMenuVisible] = useState(false); + + const openMenu = useCallback(() => setIsMenuVisible(true), []); + const closeMenu = useCallback(() => setIsMenuVisible(false), []); + const handleImportPress = useCallback(() => { + closeMenu(); + onImportAgent(); + }, [closeMenu, onImportAgent]); return ( - router.push("/settings")} - style={styles.iconButton} - > - - - } - right={ - <> + <> + router.push("/orchestrator")} + onPress={() => router.push("/settings")} style={styles.iconButton} > - + - - - - - } - /> + } + right={ + <> + router.push("/orchestrator")} + style={styles.iconButton} + > + + + + + + + + + + } + /> + + + + + + + Import Agent + + + + + ); } @@ -43,4 +84,32 @@ const styles = StyleSheet.create((theme) => ({ padding: theme.spacing[3], borderRadius: theme.borderRadius.lg, }, + menuOverlay: { + flex: 1, + }, + menuBackdrop: { + ...StyleSheet.absoluteFillObject, + }, + menuContainer: { + position: "absolute", + minWidth: 180, + borderRadius: theme.borderRadius.xl, + backgroundColor: theme.colors.background, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.border, + shadowColor: "#000", + shadowOpacity: 0.15, + shadowRadius: 8, + shadowOffset: { width: 0, height: 4 }, + elevation: 4, + }, + menuItem: { + paddingVertical: theme.spacing[3], + paddingHorizontal: theme.spacing[4], + }, + menuItemText: { + color: theme.colors.foreground, + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + }, })); diff --git a/packages/app/src/config/audio-debug.ts b/packages/app/src/config/audio-debug.ts new file mode 100644 index 000000000..a8c1b8003 --- /dev/null +++ b/packages/app/src/config/audio-debug.ts @@ -0,0 +1,2 @@ +export const AUDIO_DEBUG_ENABLED = + typeof process !== "undefined" && process.env?.EXPO_PUBLIC_ENABLE_AUDIO_DEBUG === "1"; diff --git a/packages/app/src/contexts/session-context.tsx b/packages/app/src/contexts/session-context.tsx index f86c44b6e..8687b7b43 100644 --- a/packages/app/src/contexts/session-context.tsx +++ b/packages/app/src/contexts/session-context.tsx @@ -38,6 +38,11 @@ const derivePendingPermissionKey = (agentId: string, request: AgentPermissionReq return `${agentId}:${fallbackId}`; }; +type DraftInput = { + text: string; + images: Array<{ uri: string; mimeType: string }>; +}; + export type MessageEntry = | { type: "user"; @@ -237,8 +242,8 @@ interface SessionContextValue { initializingAgents: Map; // Queued messages and draft input per agent - draftInputs: Map }>; - setDraftInputs: (value: Map }> | ((prev: Map }>) => Map }>)) => void; + getDraftInput: (agentId: string) => DraftInput | undefined; + saveDraftInput: (agentId: string, draft: DraftInput) => void; queuedMessages: Map }>>; setQueuedMessages: (value: Map }>> | ((prev: Map }>>) => Map }>>)) => void; @@ -271,7 +276,12 @@ interface SessionContextValue { refreshAgent: (params: { agentId: string; requestId?: string }) => void; cancelAgentRun: (agentId: string) => void; sendAgentMessage: (agentId: string, message: string, imageUris?: string[]) => Promise; - sendAgentAudio: (agentId: string, audioBlob: Blob, requestId?: string) => Promise; + sendAgentAudio: ( + agentId: string, + audioBlob: Blob, + requestId?: string, + options?: { mode?: "transcribe_only" | "auto_run" } + ) => Promise; deleteAgent: (agentId: string) => void; createAgent: (options: { config: AgentSessionConfig; @@ -327,7 +337,17 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { const [gitDiffs, setGitDiffs] = useState>(new Map()); const [fileExplorer, setFileExplorer] = useState>(new Map()); const [providerModels, setProviderModels] = useState>(new Map()); - const [draftInputs, setDraftInputs] = useState(new Map()); + const draftInputsRef = useRef>(new Map()); + const getDraftInput = useCallback((agentId) => { + return draftInputsRef.current.get(agentId); + }, []); + + const saveDraftInput = useCallback((agentId, draft) => { + draftInputsRef.current.set(agentId, { + text: draft.text, + images: draft.images, + }); + }, []); const [queuedMessages, setQueuedMessages] = useState(new Map()); const activeAudioGroupsRef = useRef>(new Set()); const previousAgentStatusRef = useRef>(new Map()); @@ -1000,6 +1020,8 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { return next; }); + draftInputsRef.current.delete(agentId); + setPendingPermissions((prev) => { let changed = false; const next = new Map(prev); @@ -1253,7 +1275,12 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { }, [ws]); - const sendAgentAudio = useCallback(async (agentId: string, audioBlob: Blob, requestId?: string) => { + const sendAgentAudio = useCallback(async ( + agentId: string, + audioBlob: Blob, + requestId?: string, + options?: { mode?: "transcribe_only" | "auto_run" } + ) => { try { // Convert blob to base64 const arrayBuffer = await audioBlob.arrayBuffer(); @@ -1264,8 +1291,21 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { } const base64Audio = btoa(binary); - // Determine format from MIME type - const format = audioBlob.type.split('/')[1] || 'm4a'; + const deriveFormat = (mimeType: string | undefined): string => { + if (!mimeType || mimeType.length === 0) { + return "webm"; + } + const slashIndex = mimeType.indexOf("/"); + let formatPart = slashIndex >= 0 ? mimeType.slice(slashIndex + 1) : mimeType; + const semicolonIndex = formatPart.indexOf(";"); + if (semicolonIndex >= 0) { + formatPart = formatPart.slice(0, semicolonIndex); + } + return formatPart.trim().length > 0 ? formatPart.trim() : "webm"; + }; + + // Determine format from MIME type (strip codec metadata) + const format = deriveFormat(audioBlob.type); // Send audio message const msg: WSInboundMessage = { @@ -1277,6 +1317,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { format, isLast: true, requestId, + ...(options?.mode ? { mode: options.mode } : {}), }, }; ws.send(msg); @@ -1469,8 +1510,8 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) { fileExplorer, providerModels, requestProviderModels, - draftInputs, - setDraftInputs, + getDraftInput, + saveDraftInput, queuedMessages, setQueuedMessages, requestDirectoryListing, diff --git a/packages/server/README.md b/packages/server/README.md index 1695b664a..a1973b5c9 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -72,6 +72,9 @@ See [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) for complete details. ```bash OPENAI_API_KEY=sk-... # GPT-4 and TTS DEEPGRAM_API_KEY=... # Streaming STT +STT_MODEL=whisper-1 # Optional: override to gpt-4o-transcribe, etc. +STT_CONFIDENCE_THRESHOLD=-3.0 # Optional: reject low-confidence clips +STT_DEBUG_AUDIO_DIR=.stt-debug # Optional: persist raw dictation audio for debugging PORT=3000 # Server port NODE_ENV=development # Environment ``` diff --git a/packages/server/src/server/agent/agent-manager.ts b/packages/server/src/server/agent/agent-manager.ts index 2db7ee7e1..f632f72fe 100644 --- a/packages/server/src/server/agent/agent-manager.ts +++ b/packages/server/src/server/agent/agent-manager.ts @@ -19,6 +19,7 @@ import type { ListPersistedAgentsOptions, PersistedAgentDescriptor, } from "./agent-sdk-types.js"; +import { resolveAgentModel } from "./model-resolver.js"; export type AgentLifecycleStatus = | "initializing" @@ -31,6 +32,7 @@ export type AgentSnapshot = { id: string; provider: AgentProvider; cwd: string; + model: string | null; createdAt: Date; updatedAt: Date; lastUserMessageAt: Date | null; @@ -203,9 +205,14 @@ export class AgentManager { config: AgentSessionConfig, agentId?: string ): Promise { - const client = this.requireClient(config.provider); - const session = await client.createSession(config); - return this.registerSession(session, config, agentId ?? this.idFactory()); + const normalizedConfig = await this.normalizeConfig(config); + const client = this.requireClient(normalizedConfig.provider); + const session = await client.createSession(normalizedConfig); + return this.registerSession( + session, + normalizedConfig, + agentId ?? this.idFactory() + ); } async resumeAgent( @@ -213,17 +220,22 @@ export class AgentManager { overrides?: Partial, agentId?: string ): Promise { - const client = this.requireClient(handle.provider); - const session = await client.resumeSession(handle, overrides); const metadata = (handle.metadata ?? {}) as Partial; const mergedConfig = { ...metadata, ...overrides, provider: handle.provider, } as AgentSessionConfig; + const normalizedConfig = await this.normalizeConfig(mergedConfig); + const resumeOverrides = + normalizedConfig.model !== mergedConfig.model + ? { ...overrides, model: normalizedConfig.model } + : overrides; + const client = this.requireClient(handle.provider); + const session = await client.resumeSession(handle, resumeOverrides); return this.registerSession( session, - mergedConfig, + normalizedConfig, agentId ?? this.idFactory() ); } @@ -608,6 +620,7 @@ export class AgentManager { id: agent.id, provider: agent.provider, cwd: agent.cwd, + model: agent.config.model ?? null, createdAt: agent.createdAt, updatedAt: agent.updatedAt, lastUserMessageAt: agent.lastUserMessageAt, @@ -623,6 +636,26 @@ export class AgentManager { }; } + private async normalizeConfig( + config: AgentSessionConfig + ): Promise { + const normalized: AgentSessionConfig = { ...config }; + const resolvedModel = await resolveAgentModel({ + provider: normalized.provider, + requestedModel: normalized.model, + cwd: normalized.cwd, + }); + + if (resolvedModel) { + normalized.model = resolvedModel; + } else if (typeof normalized.model === "string") { + const trimmed = normalized.model.trim(); + normalized.model = trimmed.length > 0 ? trimmed : undefined; + } + + return normalized; + } + private requireClient(provider: AgentProvider): AgentClient { const client = this.clients.get(provider); if (!client) { diff --git a/packages/server/src/server/agent/agent-registry.test.ts b/packages/server/src/server/agent/agent-registry.test.ts index b1375ac36..4785ff7b7 100644 --- a/packages/server/src/server/agent/agent-registry.test.ts +++ b/packages/server/src/server/agent/agent-registry.test.ts @@ -12,6 +12,7 @@ function createSnapshot(overrides?: Partial): AgentSnapshot { id: "agent-test", provider: "claude", cwd: "/tmp/project", + model: null, createdAt: now, updatedAt: now, lastUserMessageAt: null, diff --git a/packages/server/src/server/agent/audio-utils.ts b/packages/server/src/server/agent/audio-utils.ts new file mode 100644 index 000000000..2bda234c3 --- /dev/null +++ b/packages/server/src/server/agent/audio-utils.ts @@ -0,0 +1,20 @@ +const AUDIO_EXTENSIONS: Array<{ match: (format: string) => boolean; ext: string }> = [ + { match: (format) => format.includes("webm"), ext: "webm" }, + { match: (format) => format.includes("ogg"), ext: "ogg" }, + { match: (format) => format.includes("mp3"), ext: "mp3" }, + { match: (format) => format.includes("wav"), ext: "wav" }, + { match: (format) => format.includes("m4a") || format.includes("aac"), ext: "m4a" }, + { match: (format) => format.includes("mp4"), ext: "mp4" }, + { match: (format) => format.includes("flac"), ext: "flac" }, +]; + +export function inferAudioExtension(format: string | undefined): string { + const normalized = (format || "webm").toLowerCase(); + const candidate = AUDIO_EXTENSIONS.find((entry) => entry.match(normalized)); + return candidate?.ext ?? "webm"; +} + +export function sanitizeForFilename(segment: string | undefined, fallback: string): string { + const value = segment && segment.length > 0 ? segment : fallback; + return value.replace(/[^a-zA-Z0-9-_]/g, "_").slice(0, 64); +} diff --git a/packages/server/src/server/agent/model-catalog.ts b/packages/server/src/server/agent/model-catalog.ts index 1772ea938..e6c95ec5a 100644 --- a/packages/server/src/server/agent/model-catalog.ts +++ b/packages/server/src/server/agent/model-catalog.ts @@ -1,7 +1,7 @@ import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import path from "node:path"; import readline from "node:readline"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, URL } from "node:url"; import { query, @@ -105,7 +105,9 @@ function emptySdkMessageStream(): AsyncIterable { } function resolveCodexBinary(): string { - const repoRoot = path.resolve(fileURLToPath(new URL("../../../../..", import.meta.url))); + const repoRoot = path.resolve( + fileURLToPath(new URL("../../../../..", import.meta.url)) + ); const packageRoot = path.join(repoRoot, "node_modules", "@openai", "codex-sdk"); const vendorDir = path.join(packageRoot, "vendor"); @@ -144,7 +146,7 @@ type CodexModelInfo = { type PendingRequest = { resolve: (value: unknown) => void; reject: (error: Error) => void; - timer: NodeJS.Timeout; + timer: ReturnType; }; class CodexAppServerClient { diff --git a/packages/server/src/server/agent/model-resolver.test.ts b/packages/server/src/server/agent/model-resolver.test.ts new file mode 100644 index 000000000..374d0e15a --- /dev/null +++ b/packages/server/src/server/agent/model-resolver.test.ts @@ -0,0 +1,61 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { resolveAgentModel } from "./model-resolver.js"; + +vi.mock("./model-catalog.js", () => ({ + fetchProviderModelCatalog: vi.fn(), +})); + +import { fetchProviderModelCatalog } from "./model-catalog.js"; + +const mockedFetch = vi.mocked(fetchProviderModelCatalog); + +describe("resolveAgentModel", () => { + beforeEach(() => { + mockedFetch.mockReset(); + }); + + it("returns the trimmed requested model when provided", async () => { + const result = await resolveAgentModel({ + provider: "codex", + requestedModel: " gpt-5.1 ", + cwd: "/tmp", + }); + + expect(result).toBe("gpt-5.1"); + expect(mockedFetch).not.toHaveBeenCalled(); + }); + + it("uses the default model from the provider catalog when no model specified", async () => { + mockedFetch.mockResolvedValue([ + { id: "claude-3.5-haiku", isDefault: false } as any, + { id: "claude-3.5-sonnet", isDefault: true } as any, + ]); + + const result = await resolveAgentModel({ provider: "claude", cwd: "~/repo" }); + + expect(result).toBe("claude-3.5-sonnet"); + expect(mockedFetch).toHaveBeenCalledWith("claude", { + cwd: expect.stringMatching(/repo$/), + }); + }); + + it("falls back to the first model when none are flagged as default", async () => { + mockedFetch.mockResolvedValue([ + { id: "model-a", isDefault: false } as any, + { id: "model-b", isDefault: false } as any, + ]); + + const result = await resolveAgentModel({ provider: "codex" }); + + expect(result).toBe("model-a"); + }); + + it("returns undefined when the catalog lookup fails", async () => { + mockedFetch.mockRejectedValue(new Error("boom")); + + const result = await resolveAgentModel({ provider: "codex" }); + + expect(result).toBeUndefined(); + }); +}); diff --git a/packages/server/src/server/agent/model-resolver.ts b/packages/server/src/server/agent/model-resolver.ts new file mode 100644 index 000000000..c6579d58d --- /dev/null +++ b/packages/server/src/server/agent/model-resolver.ts @@ -0,0 +1,32 @@ +import { fetchProviderModelCatalog } from "./model-catalog.js"; +import type { AgentProvider } from "./agent-sdk-types.js"; +import { expandTilde } from "../terminal-mcp/tmux.js"; + +type ResolveAgentModelOptions = { + provider: AgentProvider; + requestedModel?: string | null; + cwd?: string; +}; + +export async function resolveAgentModel( + options: ResolveAgentModelOptions +): Promise { + const trimmed = options.requestedModel?.trim(); + if (trimmed) { + return trimmed; + } + + try { + const models = await fetchProviderModelCatalog(options.provider, { + cwd: options.cwd ? expandTilde(options.cwd) : undefined, + }); + const preferred = models.find((model) => model.isDefault) ?? models[0]; + return preferred?.id; + } catch (error) { + console.warn( + `[AgentModelResolver] Failed to resolve default model for ${options.provider}:`, + error + ); + return undefined; + } +} diff --git a/packages/server/src/server/agent/stt-debug.ts b/packages/server/src/server/agent/stt-debug.ts new file mode 100644 index 000000000..c278dbd5c --- /dev/null +++ b/packages/server/src/server/agent/stt-debug.ts @@ -0,0 +1,50 @@ +import { mkdir, writeFile } from "fs/promises"; +import { join, resolve } from "path"; +import { inferAudioExtension, sanitizeForFilename } from "./audio-utils.js"; + +const debugDir = process.env.STT_DEBUG_AUDIO_DIR + ? resolve(process.env.STT_DEBUG_AUDIO_DIR) + : null; +let announced = false; + +export interface DebugAudioMetadata { + sessionId: string; + agentId?: string; + requestId?: string; + label?: string; + format: string; +} + +export async function maybePersistDebugAudio( + audio: Buffer, + metadata: DebugAudioMetadata +): Promise { + if (!debugDir) { + return null; + } + + if (!announced) { + console.log(`[STT][Debug] Raw audio capture enabled at ${debugDir}`); + announced = true; + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const folder = join(debugDir, sanitizeForFilename(metadata.sessionId, "session")); + await mkdir(folder, { recursive: true }); + + const parts = [timestamp]; + if (metadata.agentId) { + parts.push(sanitizeForFilename(metadata.agentId, "agent")); + } + if (metadata.label) { + parts.push(sanitizeForFilename(metadata.label, "source")); + } + if (metadata.requestId) { + parts.push(sanitizeForFilename(metadata.requestId, "request")); + } + + const ext = inferAudioExtension(metadata.format); + const filePath = join(folder, `${parts.join("_")}.${ext}`); + await writeFile(filePath, audio); + return filePath; +} diff --git a/packages/server/src/server/agent/stt-manager.ts b/packages/server/src/server/agent/stt-manager.ts index afeb06216..ea03b77b4 100644 --- a/packages/server/src/server/agent/stt-manager.ts +++ b/packages/server/src/server/agent/stt-manager.ts @@ -1,4 +1,17 @@ import { transcribeAudio, type TranscriptionResult } from "./stt-openai.js"; +import { maybePersistDebugAudio } from "./stt-debug.js"; + +interface TranscriptionMetadata { + agentId?: string; + requestId?: string; + label?: string; +} + +export interface SessionTranscriptionResult extends TranscriptionResult { + debugRecordingPath?: string; + byteLength: number; + format: string; +} /** * Per-session STT manager @@ -16,12 +29,30 @@ export class STTManager { */ public async transcribe( audio: Buffer, - format: string - ): Promise { + format: string, + metadata?: TranscriptionMetadata + ): Promise { + const context = metadata?.label ? ` (${metadata.label})` : ""; console.log( - `[STT-Manager ${this.sessionId}] Transcribing ${audio.length} bytes of ${format} audio` + `[STT-Manager ${this.sessionId}] Transcribing ${audio.length} bytes of ${format} audio${context}` ); + let debugRecordingPath: string | null = null; + try { + debugRecordingPath = await maybePersistDebugAudio(audio, { + sessionId: this.sessionId, + agentId: metadata?.agentId, + requestId: metadata?.requestId, + label: metadata?.label, + format, + }); + } catch (error) { + console.warn( + `[STT-Manager ${this.sessionId}] Failed to persist debug audio:`, + error + ); + } + const result = await transcribeAudio(audio, format); // Filter out low-confidence transcriptions (non-speech sounds) @@ -34,6 +65,9 @@ export class STTManager { return { ...result, text: "", + byteLength: audio.length, + format, + debugRecordingPath: debugRecordingPath ?? undefined, }; } @@ -43,7 +77,12 @@ export class STTManager { }` ); - return result; + return { + ...result, + debugRecordingPath: debugRecordingPath ?? undefined, + byteLength: audio.length, + format, + }; } /** diff --git a/packages/server/src/server/agent/stt-openai.ts b/packages/server/src/server/agent/stt-openai.ts index 183857898..bda338445 100644 --- a/packages/server/src/server/agent/stt-openai.ts +++ b/packages/server/src/server/agent/stt-openai.ts @@ -3,10 +3,11 @@ import { writeFile, unlink } from "fs/promises"; import { join } from "path"; import { tmpdir } from "os"; import { v4 as uuidv4 } from "uuid"; +import { inferAudioExtension } from "./audio-utils.js"; export interface STTConfig { apiKey: string; - model?: "whisper-1"; + model?: "whisper-1" | "gpt-4o-transcribe" | "gpt-4o-mini-transcribe" | (string & {}); confidenceThreshold?: number; // Default: -3.0 } @@ -49,7 +50,7 @@ export async function transcribeAudio( try { // Map format to file extension - const ext = getFileExtension(format); + const ext = inferAudioExtension(format); // Write audio buffer to temporary file // OpenAI API requires file upload, not raw buffer @@ -61,11 +62,15 @@ export async function transcribeAudio( ); // Call OpenAI Whisper API + const modelToUse = config.model ?? "whisper-1"; + const supportsLogprobs = + modelToUse === "gpt-4o-transcribe" || modelToUse === "gpt-4o-mini-transcribe"; + const response = await openaiClient.audio.transcriptions.create({ file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)), language: "en", - model: config.model ?? "gpt-4o-transcribe", - include: ["logprobs"], + model: modelToUse, + ...(supportsLogprobs ? { include: ["logprobs"] as ["logprobs"] } : {}), response_format: "json", // Get language and duration info }); @@ -77,7 +82,9 @@ export async function transcribeAudio( // Analyze logprobs if available let avgLogprob: number | undefined; let isLowConfidence = false; - const logprobs = response.logprobs as LogprobToken[] | undefined; + const logprobs = supportsLogprobs + ? (response.logprobs as LogprobToken[] | undefined) + : undefined; if (logprobs && logprobs.length > 0) { // Calculate average logprob @@ -117,6 +124,7 @@ export async function transcribeAudio( logprobs: logprobs, avgLogprob: avgLogprob, isLowConfidence: isLowConfidence, + language: (response as { language?: string }).language, }; } catch (error: any) { console.error("[STT] Transcription error:", error); @@ -132,23 +140,6 @@ export async function transcribeAudio( } } } - -function getFileExtension(format: string): string { - // Map mime types or format strings to file extensions - const formatLower = format.toLowerCase(); - - if (formatLower.includes("webm")) return "webm"; - if (formatLower.includes("ogg")) return "ogg"; - if (formatLower.includes("mp3")) return "mp3"; - if (formatLower.includes("wav")) return "wav"; - if (formatLower.includes("m4a")) return "m4a"; - if (formatLower.includes("mp4")) return "mp4"; - if (formatLower.includes("flac")) return "flac"; - - // Default to webm (common browser format) - return "webm"; -} - export function isSTTInitialized(): boolean { return openaiClient !== null && config !== null; } diff --git a/packages/server/src/server/index.ts b/packages/server/src/server/index.ts index 45948dccd..3537ddb3e 100644 --- a/packages/server/src/server/index.ts +++ b/packages/server/src/server/index.ts @@ -3,7 +3,7 @@ import express from "express"; import basicAuth from "express-basic-auth"; import { createServer as createHTTPServer } from "http"; import { VoiceAssistantWebSocketServer } from "./websocket-server.js"; -import { initializeSTT } from "./agent/stt-openai.js"; +import { initializeSTT, type STTConfig } from "./agent/stt-openai.js"; import { initializeTTS } from "./agent/tts-openai.js"; import { listConversations, deleteConversation } from "./persistence.js"; import { AgentManager } from "./agent/agent-manager.js"; @@ -101,9 +101,12 @@ async function main() { ? parseFloat(process.env.STT_CONFIDENCE_THRESHOLD) : undefined; // Will default to -3.0 in stt-openai.ts + const sttModel = process.env.STT_MODEL as STTConfig["model"]; + initializeSTT({ apiKey, - confidenceThreshold: sttConfidenceThreshold + confidenceThreshold: sttConfidenceThreshold, + ...(sttModel ? { model: sttModel } : {}), }); // Initialize TTS diff --git a/packages/server/src/server/messages.ts b/packages/server/src/server/messages.ts index ae5fcc55b..2f478c390 100644 --- a/packages/server/src/server/messages.ts +++ b/packages/server/src/server/messages.ts @@ -228,6 +228,7 @@ export const AgentSnapshotPayloadSchema = z.object({ id: z.string(), provider: AgentProviderSchema, cwd: z.string(), + model: z.string().nullable(), createdAt: z.string(), updatedAt: z.string(), lastUserMessageAt: z.string().nullable(), @@ -254,6 +255,7 @@ export function serializeAgentSnapshot( const { createdAt, updatedAt, lastUserMessageAt, ...rest } = snapshot; return { ...rest, + model: snapshot.model, createdAt: createdAt.toISOString(), updatedAt: updatedAt.toISOString(), lastUserMessageAt: lastUserMessageAt ? lastUserMessageAt.toISOString() : null, @@ -334,6 +336,9 @@ export const SendAgentAudioSchema = z.object({ format: z.string(), isLast: z.boolean(), requestId: z.string().optional(), // Client-provided ID for tracking transcription + mode: z + .enum(["transcribe_only", "auto_run"]) + .optional(), }); const GitSetupOptionsSchema = z.object({ @@ -531,6 +536,11 @@ export const TranscriptionResultMessageSchema = z.object({ language: z.string().optional(), duration: z.number().optional(), requestId: z.string().optional(), // Echoed back from request for tracking + avgLogprob: z.number().optional(), + isLowConfidence: z.boolean().optional(), + byteLength: z.number().optional(), + format: z.string().optional(), + debugRecordingPath: z.string().optional(), }), }); diff --git a/packages/server/src/server/persistence-hooks.test.ts b/packages/server/src/server/persistence-hooks.test.ts index 3f8733be7..c8bf47d39 100644 --- a/packages/server/src/server/persistence-hooks.test.ts +++ b/packages/server/src/server/persistence-hooks.test.ts @@ -13,6 +13,7 @@ function createSnapshot(overrides?: Partial): AgentSnapshot { id: "agent-1", provider: "claude", cwd: "/tmp/project", + model: null, createdAt: now, updatedAt: now, lastUserMessageAt: null, diff --git a/packages/server/src/server/session.ts b/packages/server/src/server/session.ts index 76b0bb68c..4b94ac58a 100644 --- a/packages/server/src/server/session.ts +++ b/packages/server/src/server/session.ts @@ -545,6 +545,7 @@ export class Session { id: record.id, provider, cwd: record.cwd, + model: record.config?.model ?? null, createdAt: createdAt.toISOString(), updatedAt: updatedAt.toISOString(), lastUserMessageAt: lastUserMessageAt ? lastUserMessageAt.toISOString() : null, @@ -971,7 +972,8 @@ export class Session { private async handleSendAgentAudio( msg: Extract ): Promise { - const { agentId, audio, format, isLast, requestId } = msg; + const { agentId, audio, format, isLast, requestId, mode } = msg; + const shouldAutoRun = mode === "auto_run"; // Decode base64 const audioBuffer = Buffer.from(audio, "base64"); @@ -992,7 +994,11 @@ export class Session { try { // Transcribe the audio - const result = await this.sttManager.transcribe(audioBuffer, format); + const result = await this.sttManager.transcribe(audioBuffer, format, { + agentId, + requestId, + label: shouldAutoRun ? "dictation:auto_run" : "dictation", + }); const transcriptText = result.text.trim(); if (!transcriptText) { @@ -1014,9 +1020,21 @@ export class Session { language: result.language, duration: result.duration, requestId, + avgLogprob: result.avgLogprob, + isLowConfidence: result.isLowConfidence, + byteLength: result.byteLength, + format: result.format, + debugRecordingPath: result.debugRecordingPath, }, }); + if (!shouldAutoRun) { + console.log( + `[Session ${this.clientId}] Completed transcription for agent ${agentId} (requestId: ${requestId ?? "n/a"})` + ); + return; + } + try { await this.interruptAgentIfRunning(agentId); } catch (error) { @@ -2225,7 +2243,9 @@ export class Session { }); try { - const result = await this.sttManager.transcribe(audio, format); + const result = await this.sttManager.transcribe(audio, format, { + label: this.isRealtimeMode ? "realtime" : "buffered", + }); const transcriptText = result.text.trim(); if (!transcriptText) { @@ -2255,6 +2275,11 @@ export class Session { text: result.text, language: result.language, duration: result.duration, + avgLogprob: result.avgLogprob, + isLowConfidence: result.isLowConfidence, + byteLength: result.byteLength, + format: result.format, + debugRecordingPath: result.debugRecordingPath, }, }); diff --git a/packages/server/src/utils/worktree.ts b/packages/server/src/utils/worktree.ts index 0a4b9e3e3..25d597b1e 100644 --- a/packages/server/src/utils/worktree.ts +++ b/packages/server/src/utils/worktree.ts @@ -119,13 +119,13 @@ export function validateBranchSlug(slug: string): { return { valid: false, error: "Branch name too long (max 100 characters)" }; } - // Check for valid characters: lowercase letters, numbers, hyphens - const validPattern = /^[a-z0-9-]+$/; + // Check for valid characters: lowercase letters, numbers, hyphens, forward slashes + const validPattern = /^[a-z0-9-/]+$/; if (!validPattern.test(slug)) { return { valid: false, error: - "Branch name must contain only lowercase letters, numbers, and hyphens", + "Branch name must contain only lowercase letters, numbers, hyphens, and forward slashes", }; } diff --git a/packages/server/tsconfig.server.json b/packages/server/tsconfig.server.json index 6eaacd042..8aac9e25f 100644 --- a/packages/server/tsconfig.server.json +++ b/packages/server/tsconfig.server.json @@ -24,5 +24,12 @@ "sourceMap": true }, "include": ["src/server/**/*", "src/services/**/*"], - "exclude": ["node_modules", "dist"] + "exclude": [ + "node_modules", + "dist", + "src/**/*.test.ts", + "src/**/*.test.tsx", + "src/**/*.spec.ts", + "src/**/*.spec.tsx" + ] }