chore: typecheck and build readiness

This commit is contained in:
Mohamed Boudra
2025-11-24 22:31:41 +01:00
parent a95184cc5a
commit 30d1b63b2b
25 changed files with 1502 additions and 440 deletions

View File

@@ -48,3 +48,7 @@ Join our community of developers creating universal apps.
- [Expo on GitHub](https://github.com/expo/expo): View our open source platform and contribute.
- [Discord community](https://chat.expo.dev): Chat with Expo users and ask questions.
## Dictation debugging
Set `EXPO_PUBLIC_ENABLE_AUDIO_DEBUG=1` before running `npx expo start` to render the in-app audio debug card. Pair it with the server-side `STT_DEBUG_AUDIO_DIR` flag so every dictation includes a copyable path to the saved raw audio file.

View File

@@ -13,10 +13,11 @@ import { useSafeAreaInsets } from "react-native-safe-area-context";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
import ReanimatedAnimated, { useAnimatedStyle, useSharedValue } from "react-native-reanimated";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { MoreVertical, GitBranch, Folder, RotateCcw } from "lucide-react-native";
import { MoreVertical, GitBranch, Folder, RotateCcw, PlusCircle } from "lucide-react-native";
import { BackHeader } from "@/components/headers/back-header";
import { AgentStreamView } from "@/components/agent-stream-view";
import { AgentInputArea } from "@/components/agent-input-area";
import { CreateAgentModal, type CreateAgentInitialValues } from "@/components/create-agent-modal";
import { useSession } from "@/contexts/session-context";
import type { Agent } from "@/contexts/session-context";
import { useFooterControls } from "@/contexts/footer-controls-context";
@@ -92,7 +93,16 @@ export default function AgentScreen() {
const [branchStatus, setBranchStatus] = useState<BranchStatus>("idle");
const [branchLabel, setBranchLabel] = useState<string | null>(null);
const [branchError, setBranchError] = useState<string | null>(null);
const [showCreateAgentModal, setShowCreateAgentModal] = useState(false);
const [createAgentInitialValues, setCreateAgentInitialValues] =
useState<CreateAgentInitialValues | undefined>();
const repoInfoRequestIdRef = useRef<string | null>(null);
const hasPendingRepoRequest = repoInfoRequestIdRef.current !== null;
const branchDisplayValue =
branchStatus === "error"
? branchError ?? "Unavailable"
: branchLabel ?? "Unknown";
const shouldListenForBranchInfo = menuVisible || hasPendingRepoRequest;
// Keyboard animation
const { height: keyboardHeight } = useReanimatedKeyboardAnimation();
@@ -161,6 +171,9 @@ export default function AgentScreen() {
}, [agent?.cwd, resetBranchState, sendGitRepoInfoRequest]);
useEffect(() => {
if (!shouldListenForBranchInfo) {
return;
}
const unsubscribe = ws.on("git_repo_info_response", (message) => {
if (message.type !== "git_repo_info_response") {
return;
@@ -195,7 +208,7 @@ export default function AgentScreen() {
return () => {
unsubscribe();
};
}, [agent?.cwd, ws]);
}, [agent?.cwd, shouldListenForBranchInfo, ws]);
useEffect(() => {
if (!id) {
@@ -352,164 +365,196 @@ export default function AgentScreen() {
refreshAgent({ agentId: id });
}, [handleCloseMenu, id, refreshAgent]);
const branchDisplayValue =
branchStatus === "error"
? branchError ?? "Unavailable"
: branchLabel ?? "Unknown";
const handleCreateNewAgent = useCallback(() => {
if (!agent) {
return;
}
handleCloseMenu();
setCreateAgentInitialValues({
workingDir: agent.cwd,
provider: agent.provider,
modeId: agent.currentModeId,
model: agentModel ?? undefined,
});
setShowCreateAgentModal(true);
}, [agent, agentModel, handleCloseMenu]);
const handleCloseCreateAgentModal = useCallback(() => {
setShowCreateAgentModal(false);
}, []);
const createAgentModal = (
<CreateAgentModal
isVisible={showCreateAgentModal}
onClose={handleCloseCreateAgentModal}
initialValues={createAgentInitialValues}
/>
);
if (!agent) {
return (
<View style={styles.container}>
<BackHeader onBack={handleBackToHome} />
<View style={styles.errorContainer}>
<Text style={styles.errorText}>Agent not found</Text>
<>
<View style={styles.container}>
<BackHeader onBack={handleBackToHome} />
<View style={styles.errorContainer}>
<Text style={styles.errorText}>Agent not found</Text>
</View>
</View>
</View>
{createAgentModal}
</>
);
}
return (
<View style={styles.container}>
{/* Header */}
<BackHeader
title={agent.title || "Agent"}
onBack={handleBackToHome}
rightContent={
<View ref={menuButtonRef} collapsable={false}>
<Pressable onPress={handleOpenMenu} style={styles.menuButton}>
<MoreVertical size={20} color={theme.colors.foreground} />
</Pressable>
</View>
}
/>
{/* Content Area with Keyboard Animation */}
<View style={styles.contentContainer}>
<ReanimatedAnimated.View style={[styles.content, animatedKeyboardStyle]}>
{isInitializing ? (
<View style={styles.loadingContainer}>
<ActivityIndicator size="large" color={theme.colors.primary} />
<Text style={styles.loadingText}>Loading agent...</Text>
<>
<View style={styles.container}>
{/* Header */}
<BackHeader
title={agent.title || "Agent"}
onBack={handleBackToHome}
rightContent={
<View ref={menuButtonRef} collapsable={false}>
<Pressable onPress={handleOpenMenu} style={styles.menuButton}>
<MoreVertical size={20} color={theme.colors.foreground} />
</Pressable>
</View>
) : (
<AgentStreamView
agentId={id!}
agent={agent}
streamItems={streamItems}
pendingPermissions={agentPermissions}
onPermissionResponse={(agentId, requestId, response) =>
respondToPermission(agentId, requestId, response)
}
/>
)}
</ReanimatedAnimated.View>
</View>
}
/>
{/* Dropdown Menu */}
<Modal
visible={menuVisible}
animationType="fade"
transparent={true}
onRequestClose={handleCloseMenu}
>
<View style={styles.menuOverlay}>
<Pressable style={styles.menuBackdrop} onPress={handleCloseMenu} />
<View
style={[
styles.dropdownMenu,
{
position: "absolute",
top: menuPosition.top,
left: menuPosition.left,
width: DROPDOWN_WIDTH,
},
]}
onLayout={handleMenuLayout}
>
<View style={styles.menuMetaContainer}>
<View style={styles.menuMetaRow}>
<Text style={styles.menuMetaLabel}>Directory</Text>
<Text
style={styles.menuMetaValue}
numberOfLines={2}
ellipsizeMode="middle"
>
{agent.cwd}
</Text>
{/* Content Area with Keyboard Animation */}
<View style={styles.contentContainer}>
<ReanimatedAnimated.View style={[styles.content, animatedKeyboardStyle]}>
{isInitializing ? (
<View style={styles.loadingContainer}>
<ActivityIndicator size="large" color={theme.colors.primary} />
<Text style={styles.loadingText}>Loading agent...</Text>
</View>
) : (
<AgentStreamView
agentId={id!}
agent={agent}
streamItems={streamItems}
pendingPermissions={agentPermissions}
onPermissionResponse={(agentId, requestId, response) =>
respondToPermission(agentId, requestId, response)
}
/>
)}
</ReanimatedAnimated.View>
</View>
<View style={styles.menuMetaRow}>
<Text style={styles.menuMetaLabel}>Model</Text>
<Text
style={styles.menuMetaValue}
numberOfLines={1}
ellipsizeMode="middle"
>
{modelDisplayValue}
</Text>
</View>
{/* Dropdown Menu */}
<Modal
visible={menuVisible}
animationType="fade"
transparent={true}
onRequestClose={handleCloseMenu}
>
<View style={styles.menuOverlay}>
<Pressable style={styles.menuBackdrop} onPress={handleCloseMenu} />
<View
style={[
styles.dropdownMenu,
{
position: "absolute",
top: menuPosition.top,
left: menuPosition.left,
width: DROPDOWN_WIDTH,
},
]}
onLayout={handleMenuLayout}
>
<View style={styles.menuMetaContainer}>
<View style={styles.menuMetaRow}>
<Text style={styles.menuMetaLabel}>Directory</Text>
<Text
style={styles.menuMetaValue}
numberOfLines={2}
ellipsizeMode="middle"
>
{agent.cwd}
</Text>
</View>
<View style={styles.menuMetaRow}>
<Text style={styles.menuMetaLabel}>Branch</Text>
<View style={styles.menuMetaValueRow}>
{branchStatus === "loading" ? (
<>
<ActivityIndicator
size="small"
color={theme.colors.mutedForeground}
/>
<Text style={styles.menuMetaPendingText}>Fetching</Text>
</>
) : (
<Text
style={[
styles.menuMetaValue,
branchStatus === "error" ? styles.menuMetaValueError : null,
]}
numberOfLines={1}
ellipsizeMode="middle"
>
{branchDisplayValue}
</Text>
)}
<View style={styles.menuMetaRow}>
<Text style={styles.menuMetaLabel}>Model</Text>
<Text
style={styles.menuMetaValue}
numberOfLines={1}
ellipsizeMode="middle"
>
{modelDisplayValue}
</Text>
</View>
<View style={styles.menuMetaRow}>
<Text style={styles.menuMetaLabel}>Branch</Text>
<View style={styles.menuMetaValueRow}>
{branchStatus === "loading" ? (
<>
<ActivityIndicator
size="small"
color={theme.colors.mutedForeground}
/>
<Text style={styles.menuMetaPendingText}>Fetching</Text>
</>
) : (
<Text
style={[
styles.menuMetaValue,
branchStatus === "error" ? styles.menuMetaValueError : null,
]}
numberOfLines={1}
ellipsizeMode="middle"
>
{branchDisplayValue}
</Text>
)}
</View>
</View>
</View>
<View style={styles.menuDivider} />
<Pressable onPress={handleViewChanges} style={styles.menuItem}>
<GitBranch size={20} color={theme.colors.foreground} />
<Text style={styles.menuItemText}>View Changes</Text>
</Pressable>
<Pressable onPress={handleBrowseFiles} style={styles.menuItem}>
<Folder size={20} color={theme.colors.foreground} />
<Text style={styles.menuItemText}>Browse Files</Text>
</Pressable>
<Pressable onPress={handleCreateNewAgent} style={styles.menuItem}>
<PlusCircle size={20} color={theme.colors.foreground} />
<Text style={styles.menuItemText}>New Agent</Text>
</Pressable>
<Pressable
onPress={handleRefreshAgent}
style={[
styles.menuItem,
isInitializing ? styles.menuItemDisabled : null,
]}
disabled={isInitializing}
>
<RotateCcw size={20} color={theme.colors.foreground} />
<Text style={styles.menuItemText}>
{isInitializing ? "Refreshing..." : "Refresh"}
</Text>
{isInitializing && (
<ActivityIndicator
size="small"
color={theme.colors.primary}
style={styles.menuItemSpinner}
/>
)}
</Pressable>
</View>
<View style={styles.menuDivider} />
<Pressable onPress={handleViewChanges} style={styles.menuItem}>
<GitBranch size={20} color={theme.colors.foreground} />
<Text style={styles.menuItemText}>View Changes</Text>
</Pressable>
<Pressable onPress={handleBrowseFiles} style={styles.menuItem}>
<Folder size={20} color={theme.colors.foreground} />
<Text style={styles.menuItemText}>Browse Files</Text>
</Pressable>
<Pressable
onPress={handleRefreshAgent}
style={[
styles.menuItem,
isInitializing ? styles.menuItemDisabled : null,
]}
disabled={isInitializing}
>
<RotateCcw size={20} color={theme.colors.foreground} />
<Text style={styles.menuItemText}>
{isInitializing ? "Refreshing..." : "Refresh"}
</Text>
{isInitializing && (
<ActivityIndicator
size="small"
color={theme.colors.primary}
style={styles.menuItemSpinner}
/>
)}
</Pressable>
</View>
</View>
</Modal>
</View>
</Modal>
</View>
{createAgentModal}
</>
);
}

View File

@@ -1,5 +1,5 @@
import { View } from "react-native";
import { useState } from "react";
import { useState, useCallback } from "react";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
import ReanimatedAnimated, { useAnimatedStyle } from "react-native-reanimated";
@@ -7,7 +7,7 @@ import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { HomeHeader } from "@/components/headers/home-header";
import { EmptyState } from "@/components/empty-state";
import { AgentList } from "@/components/agent-list";
import { CreateAgentModal } from "@/components/create-agent-modal";
import { CreateAgentModal, ImportAgentModal } from "@/components/create-agent-modal";
import { useSession } from "@/contexts/session-context";
export default function HomeScreen() {
@@ -15,6 +15,9 @@ export default function HomeScreen() {
const insets = useSafeAreaInsets();
const { agents } = useSession();
const [showCreateModal, setShowCreateModal] = useState(false);
const [showImportModal, setShowImportModal] = useState(false);
const [createModalMounted, setCreateModalMounted] = useState(false);
const [importModalMounted, setImportModalMounted] = useState(false);
// Keyboard animation
const { height: keyboardHeight } = useReanimatedKeyboardAnimation();
@@ -29,14 +32,31 @@ export default function HomeScreen() {
const hasAgents = agents.size > 0;
function handleCreateAgent() {
const handleCreateAgent = useCallback(() => {
setCreateModalMounted(true);
setShowCreateModal(true);
}
}, []);
const handleImportAgent = useCallback(() => {
setImportModalMounted(true);
setShowImportModal(true);
}, []);
const handleCloseCreateModal = useCallback(() => {
setShowCreateModal(false);
}, []);
const handleCloseImportModal = useCallback(() => {
setShowImportModal(false);
}, []);
return (
<View style={styles.container}>
{/* Header */}
<HomeHeader onCreateAgent={handleCreateAgent} />
<HomeHeader
onCreateAgent={handleCreateAgent}
onImportAgent={handleImportAgent}
/>
{/* Content Area with Keyboard Animation */}
<ReanimatedAnimated.View style={[styles.content, animatedKeyboardStyle]}>
@@ -48,10 +68,12 @@ export default function HomeScreen() {
</ReanimatedAnimated.View>
{/* Create Agent Modal */}
<CreateAgentModal
isVisible={showCreateModal}
onClose={() => setShowCreateModal(false)}
/>
{createModalMounted ? (
<CreateAgentModal isVisible={showCreateModal} onClose={handleCloseCreateModal} />
) : null}
{importModalMounted ? (
<ImportAgentModal isVisible={showImportModal} onClose={handleCloseImportModal} />
) : null}
</View>
);
}

View File

@@ -10,7 +10,7 @@ import {
Text,
ActivityIndicator,
} from "react-native";
import { useState, useEffect, useRef, useLayoutEffect } from "react";
import { useState, useEffect, useRef, useLayoutEffect, useCallback } from "react";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { Mic, ArrowUp, AudioLines, Square, Paperclip, X, Pencil } from "lucide-react-native";
import Animated, {
@@ -29,6 +29,14 @@ import { generateMessageId } from "@/types/stream";
import { AgentStatusBar } from "./agent-status-bar";
import { RealtimeControls } from "./realtime-controls";
import { useImageAttachmentPicker } from "@/hooks/use-image-attachment-picker";
import { AUDIO_DEBUG_ENABLED } from "@/config/audio-debug";
import { AudioDebugNotice, type AudioDebugInfo } from "./audio-debug-notice";
type QueuedMessage = {
id: string;
text: string;
images?: Array<{ uri: string; mimeType: string }>;
};
interface AgentInputAreaProps {
agentId: string;
@@ -68,8 +76,8 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
sendAgentAudio,
agents,
cancelAgentRun,
draftInputs,
setDraftInputs,
getDraftInput,
saveDraftInput,
queuedMessages: queuedMessagesByAgent,
setQueuedMessages: setQueuedMessagesByAgent,
} = useSession();
@@ -84,6 +92,7 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
const [recordingDuration, setRecordingDuration] = useState(0);
const [transcribingRequestId, setTranscribingRequestId] = useState<string | null>(null);
const [isCancellingAgent, setIsCancellingAgent] = useState(false);
const [audioDebugInfo, setAudioDebugInfo] = useState<AudioDebugInfo | null>(null);
const recordingIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const textInputRef = useRef<TextInput | (TextInput & { getNativeRef?: () => unknown }) | null>(null);
@@ -91,7 +100,15 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
const baselineInputHeightRef = useRef<number | null>(null);
const overlayTransition = useSharedValue(0);
const { pickImages } = useImageAttachmentPicker();
const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED;
const pendingTranscriptionRef = useRef<{ requestId: string } | null>(null);
const agentIdRef = useRef(agentId);
const sendAgentMessageRef = useRef(sendAgentMessage);
const agentStatusRef = useRef<string | undefined>(undefined);
const updateQueueRef = useRef<
((updater: (current: QueuedMessage[]) => QueuedMessage[]) => void) | null
>(null);
const audioRecorder = useAudioRecorder({
onAudioLevel: (level) => {
setRecordingVolume(level);
@@ -108,6 +125,14 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
inputHeightRef.current = inputHeight;
}, [inputHeight]);
useEffect(() => {
agentIdRef.current = agentId;
}, [agentId]);
useEffect(() => {
sendAgentMessageRef.current = sendAgentMessage;
}, [sendAgentMessage]);
async function handleSendMessage() {
if (!userInput.trim() || !ws.isConnected) return;
@@ -153,7 +178,11 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
if (isRealtimeMode) {
return;
}
if (shouldShowAudioDebug) {
setAudioDebugInfo(null);
}
// Start recording
try {
await audioRecorder.start();
@@ -208,16 +237,20 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
// Generate request ID for tracking transcription
const requestId = generateMessageId();
setTranscribingRequestId(requestId);
pendingTranscriptionRef.current = {
requestId,
};
console.log("[AgentInput] Audio recorded:", audioData.size, "bytes", "requestId:", requestId);
try {
// Send audio to agent for transcription and processing
await sendAgentAudio(agentId, audioData, requestId);
await sendAgentAudio(agentId, audioData, requestId, { mode: "transcribe_only" });
console.log("[AgentInput] Audio sent to agent");
} catch (error) {
console.error("[AgentInput] Failed to send audio:", error);
// Clear transcribing state on error
setTranscribingRequestId(null);
pendingTranscriptionRef.current = null;
overlayTransition.value = withTiming(0, { duration: 250 });
}
} else {
@@ -228,29 +261,81 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
console.error("[AgentInput] Failed to stop recording:", error);
setIsRecording(false);
setTranscribingRequestId(null);
pendingTranscriptionRef.current = null;
overlayTransition.value = withTiming(0, { duration: 250 });
}
}
// Listen for transcription completion
useEffect(() => {
if (!transcribingRequestId) return;
const unsubscribe = ws.on("transcription_result", (message) => {
if (message.type !== "transcription_result") return;
// Check if this transcription result matches our request
if (message.payload.requestId === transcribingRequestId) {
console.log("[AgentInput] Transcription completed for requestId:", transcribingRequestId);
setTranscribingRequestId(null);
overlayTransition.value = withTiming(0, { duration: 250 });
if (message.type !== "transcription_result") {
return;
}
const pending = pendingTranscriptionRef.current;
if (!pending || !message.payload.requestId) {
return;
}
if (message.payload.requestId !== pending.requestId) {
return;
}
console.log("[AgentInput] Transcription completed for requestId:", pending.requestId);
pendingTranscriptionRef.current = null;
setTranscribingRequestId(null);
overlayTransition.value = withTiming(0, { duration: 250 });
if (shouldShowAudioDebug) {
setAudioDebugInfo({
requestId: pending.requestId,
transcript: message.payload.text?.trim(),
debugRecordingPath: message.payload.debugRecordingPath ?? undefined,
format: message.payload.format,
byteLength: message.payload.byteLength,
duration: message.payload.duration,
avgLogprob: message.payload.avgLogprob,
isLowConfidence: message.payload.isLowConfidence,
});
}
const transcriptText = message.payload.text?.trim();
if (!transcriptText) {
return;
}
const shouldQueue = agentStatusRef.current === "running";
if (shouldQueue) {
updateQueueRef.current?.((current) => [
...current,
{
id: generateMessageId(),
text: transcriptText,
},
]);
return;
}
void (async () => {
try {
await sendAgentMessageRef.current?.(agentIdRef.current, transcriptText);
} catch (error) {
console.error("[AgentInput] Failed to send transcribed message:", error);
updateQueueRef.current?.((current) => [
...current,
{
id: generateMessageId(),
text: transcriptText,
},
]);
}
})();
});
return () => {
unsubscribe();
};
}, [transcribingRequestId, ws, overlayTransition]);
}, [ws, overlayTransition, shouldShowAudioDebug]);
// Cleanup timer on unmount
useEffect(() => {
@@ -412,14 +497,30 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
const agent = agents.get(agentId);
const isAgentRunning = agent?.status === "running";
agentStatusRef.current = agent?.status;
const hasText = userInput.trim().length > 0;
const hasImages = selectedImages.length > 0;
const hasSendableContent = hasText || hasImages;
const shouldShowSendButton = !isAgentRunning && hasSendableContent;
const shouldShowVoiceControls = !isAgentRunning && !hasSendableContent;
const shouldShowVoiceControls = !hasSendableContent;
const queuedMessages = queuedMessagesByAgent.get(agentId) ?? [];
const shouldHandleDesktopSubmit = IS_WEB;
const updateQueue = useCallback(
(updater: (current: QueuedMessage[]) => QueuedMessage[]) => {
setQueuedMessagesByAgent((prev) => {
const next = new Map(prev);
next.set(agentId, updater(prev.get(agentId) ?? []));
return next;
});
},
[agentId, setQueuedMessagesByAgent],
);
useEffect(() => {
updateQueueRef.current = updateQueue;
}, [updateQueue]);
useLayoutEffect(() => {
if (!IS_WEB) {
return;
@@ -509,7 +610,7 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
// Hydrate draft only when switching agents
useEffect(() => {
const draft = draftInputs.get(agentId);
const draft = getDraftInput(agentId);
if (!draft) {
setUserInput("");
setSelectedImages([]);
@@ -518,28 +619,23 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
setUserInput(draft.text);
setSelectedImages(draft.images);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [agentId]);
}, [agentId, getDraftInput]);
// Persist drafts into context with change detection to avoid render loops
// Persist drafts into the shared session store with change detection to avoid redundant work
useEffect(() => {
setDraftInputs((prev) => {
const existing = prev.get(agentId);
const isSameText = existing?.text === userInput;
const existingImages = existing?.images ?? [];
const isSameImages =
existingImages.length === selectedImages.length &&
existingImages.every((img, idx) => img.uri === selectedImages[idx]?.uri && img.mimeType === selectedImages[idx]?.mimeType);
const existing = getDraftInput(agentId);
const isSameText = existing?.text === userInput;
const existingImages = existing?.images ?? [];
const isSameImages =
existingImages.length === selectedImages.length &&
existingImages.every((img, idx) => img.uri === selectedImages[idx]?.uri && img.mimeType === selectedImages[idx]?.mimeType);
if (isSameText && isSameImages) {
return prev;
}
if (isSameText && isSameImages) {
return;
}
const next = new Map(prev);
next.set(agentId, { text: userInput, images: selectedImages });
return next;
});
}, [agentId, userInput, selectedImages, setDraftInputs]);
saveDraftInput(agentId, { text: userInput, images: selectedImages });
}, [agentId, userInput, selectedImages, getDraftInput, saveDraftInput]);
const overlayAnimatedStyle = useAnimatedStyle(() => {
return {
@@ -582,14 +678,6 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
cancelAgentRun(agentId);
}
function updateQueue(updater: (current: typeof queuedMessages) => typeof queuedMessages) {
setQueuedMessagesByAgent((prev) => {
const next = new Map(prev);
next.set(agentId, updater(prev.get(agentId) ?? []));
return next;
});
}
function handleQueueCurrentInput() {
if (!hasSendableContent) return;
@@ -643,6 +731,24 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
</Pressable>
);
const voiceButton = (
<Pressable
onPress={handleVoicePress}
disabled={!ws.isConnected || isRealtimeMode}
style={[
styles.voiceButton as any,
(!ws.isConnected || isRealtimeMode ? styles.buttonDisabled : undefined) as any,
(isRecording ? styles.voiceButtonRecording : undefined) as any,
]}
>
{isRecording ? (
<Square size={14} color="white" fill="white" />
) : (
<Mic size={20} color={theme.colors.foreground} />
)}
</Pressable>
);
return (
<View style={styles.container}>
{/* Border separator */}
@@ -664,6 +770,12 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
<View style={styles.inputAreaContent}>
{/* Regular input controls */}
<Animated.View style={[styles.inputContainer, inputAnimatedStyle]}>
{shouldShowAudioDebug && audioDebugInfo ? (
<AudioDebugNotice
info={audioDebugInfo}
onDismiss={() => setAudioDebugInfo(null)}
/>
) : null}
{/* Queue list */}
{queuedMessages.length > 0 && (
<View style={styles.queueContainer}>
@@ -741,6 +853,7 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
<View style={styles.rightButtonGroup}>
{isAgentRunning ? (
<>
{shouldShowVoiceControls && voiceButton}
{hasSendableContent && (
<Pressable
onPress={handleQueueCurrentInput}
@@ -784,26 +897,12 @@ export function AgentInputArea({ agentId }: AgentInputAreaProps) {
>
<ArrowUp size={20} color="white" />
</Pressable>
) : shouldShowVoiceControls ? (
<>
<Pressable
onPress={handleVoicePress}
disabled={!ws.isConnected || isRealtimeMode}
style={[
styles.voiceButton as any,
(!ws.isConnected || isRealtimeMode ? styles.buttonDisabled : undefined) as any,
(isRecording ? styles.voiceButtonRecording : undefined) as any,
]}
>
{isRecording ? (
<Square size={14} color="white" fill="white" />
) : (
<Mic size={20} color={theme.colors.foreground} />
)}
</Pressable>
{realtimeButton}
</>
) : null}
) : shouldShowVoiceControls ? (
<>
{voiceButton}
{realtimeButton}
</>
) : null}
</View>
</View>
</Animated.View>

View File

@@ -0,0 +1,187 @@
import { useState, useMemo } from "react";
import { View, Text, Pressable } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import * as Clipboard from "expo-clipboard";
import { Check, Copy, X } from "lucide-react-native";
export interface AudioDebugInfo {
requestId?: string | null;
transcript?: string;
debugRecordingPath?: string;
format?: string;
byteLength?: number;
duration?: number;
avgLogprob?: number;
isLowConfidence?: boolean;
}
interface AudioDebugNoticeProps {
info: AudioDebugInfo | null;
onDismiss?: () => void;
title?: string;
}
function formatBytes(bytes?: number): string | null {
if (!bytes || bytes <= 0) {
return null;
}
if (bytes < 1024) {
return `${bytes} B`;
}
const units = ["KB", "MB", "GB"] as const;
let value = bytes;
let unitIndex = -1;
while (value >= 1024 && unitIndex < units.length - 1) {
value /= 1024;
unitIndex += 1;
}
return `${value.toFixed(value >= 100 ? 0 : 1)} ${units[unitIndex]}`;
}
function formatDuration(duration?: number): string | null {
if (!duration || duration <= 0) {
return null;
}
const seconds = duration / 1000;
if (seconds < 1) {
return `${(seconds * 1000).toFixed(0)} ms`;
}
return `${seconds.toFixed(seconds >= 10 ? 0 : 1)} s`;
}
export function AudioDebugNotice({ info, onDismiss, title = "Dictation Debug" }: AudioDebugNoticeProps) {
const { theme } = useUnistyles();
const [copied, setCopied] = useState(false);
const stats = useMemo(() => {
if (!info) {
return null;
}
const parts: string[] = [];
if (info.format) {
parts.push(info.format);
}
const size = formatBytes(info.byteLength);
if (size) {
parts.push(size);
}
const duration = formatDuration(info.duration);
if (duration) {
parts.push(duration);
}
if (info.avgLogprob !== undefined) {
const label = `${info.avgLogprob.toFixed(2)} avg logprob`;
parts.push(info.isLowConfidence ? `${label} (low confidence)` : label);
}
return parts.join(" · ");
}, [info]);
if (!info) {
return null;
}
const handleCopyPath = async () => {
if (!info.debugRecordingPath) {
return;
}
try {
await Clipboard.setStringAsync(info.debugRecordingPath);
setCopied(true);
setTimeout(() => setCopied(false), 1500);
} catch (error) {
console.warn("[AudioDebug] Failed to copy path", error);
}
};
const pathMissing = !info.debugRecordingPath;
return (
<View
style={[
styles.container,
{
backgroundColor: theme.colors.secondary,
borderColor: theme.colors.border,
},
]}
>
<View style={styles.headerRow}>
<Text style={[styles.title, { color: theme.colors.mutedForeground }]}>{title}</Text>
{onDismiss ? (
<Pressable accessibilityLabel="Dismiss audio debug" onPress={onDismiss} hitSlop={8}>
<X size={14} color={theme.colors.mutedForeground} />
</Pressable>
) : null}
</View>
{info.debugRecordingPath ? (
<Pressable style={styles.pathRow} onPress={handleCopyPath} accessibilityLabel="Copy raw audio path">
<Text
numberOfLines={2}
style={[styles.pathText, { color: theme.colors.foreground }]}
>
{info.debugRecordingPath}
</Text>
<View style={[styles.copyPill, { backgroundColor: theme.colors.primary }]}>
{copied ? (
<Check size={12} color={theme.colors.primaryForeground} />
) : (
<Copy size={12} color={theme.colors.primaryForeground} />
)}
</View>
</Pressable>
) : (
<Text style={[styles.hint, { color: theme.colors.mutedForeground }]}>
Raw audio path unavailable. Set STT_DEBUG_AUDIO_DIR on the server to persist recordings.
</Text>
)}
{stats ? (
<Text style={[styles.stats, { color: theme.colors.mutedForeground }]}>
{stats}
</Text>
) : null}
</View>
);
}
const styles = StyleSheet.create({
container: {
borderRadius: 10,
borderWidth: StyleSheet.hairlineWidth,
padding: 12,
gap: 8,
},
headerRow: {
flexDirection: "row",
alignItems: "center",
justifyContent: "space-between",
},
title: {
fontSize: 12,
fontWeight: "600",
textTransform: "uppercase",
},
pathRow: {
flexDirection: "row",
alignItems: "center",
gap: 8,
},
pathText: {
flex: 1,
fontSize: 13,
},
copyPill: {
width: 28,
height: 28,
borderRadius: 14,
alignItems: "center",
justifyContent: "center",
},
stats: {
fontSize: 12,
},
hint: {
fontSize: 12,
},
});

View File

@@ -35,13 +35,17 @@ import Animated, {
Easing,
runOnJS,
} from "react-native-reanimated";
import { StyleSheet } from "react-native-unistyles";
import { X, ChevronDown } from "lucide-react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { Mic, Check, X, ChevronDown } from "lucide-react-native";
import { theme as defaultTheme } from "@/styles/theme";
import { useRecentPaths } from "@/hooks/use-recent-paths";
import { useSession } from "@/contexts/session-context";
import { useRouter } from "expo-router";
import { generateMessageId } from "@/types/stream";
import { useAudioRecorder } from "@/hooks/use-audio-recorder";
import { VolumeMeter } from "@/components/volume-meter";
import { AUDIO_DEBUG_ENABLED } from "@/config/audio-debug";
import { AudioDebugNotice, type AudioDebugInfo } from "./audio-debug-notice";
import {
AGENT_PROVIDER_DEFINITIONS,
type AgentProviderDefinition,
@@ -56,9 +60,24 @@ import type {
} from "@server/server/agent/agent-sdk-types";
import type { WSInboundMessage } from "@server/server/messages";
interface CreateAgentModalProps {
export type CreateAgentInitialValues = {
workingDir?: string;
provider?: AgentProvider;
modeId?: string | null;
model?: string | null;
};
interface AgentFlowModalProps {
isVisible: boolean;
onClose: () => void;
flow: "create" | "import";
initialValues?: CreateAgentInitialValues;
}
interface ModalWrapperProps {
isVisible: boolean;
onClose: () => void;
initialValues?: CreateAgentInitialValues;
}
const providerDefinitions = AGENT_PROVIDER_DEFINITIONS;
@@ -71,11 +90,12 @@ const DEFAULT_PROVIDER: AgentProvider = fallbackDefinition?.id ?? "claude";
const DEFAULT_MODE_FOR_DEFAULT_PROVIDER =
fallbackDefinition?.defaultModeId ?? "";
const BACKDROP_OPACITY = 0.55;
const RESUME_PAGE_SIZE = 20;
const IMPORT_PAGE_SIZE = 20;
const PROMPT_MIN_HEIGHT = 64;
const PROMPT_MAX_HEIGHT = 200;
const FORM_PREFERENCES_STORAGE_KEY = "@paseo:create-agent-preferences";
const IS_WEB = Platform.OS === "web";
const DICTATION_AGENT_ID = "__dictation__";
type WebTextInputKeyPressEvent = NativeSyntheticEvent<
TextInputKeyPressEventData & {
@@ -109,7 +129,7 @@ const isTextAreaLike = (node: unknown): node is TextAreaHandle => {
return true;
};
type ResumeCandidate = {
type ImportCandidate = {
provider: AgentProvider;
sessionId: string;
cwd: string;
@@ -119,7 +139,6 @@ type ResumeCandidate = {
timeline: AgentTimelineItem[];
};
type ResumeTab = "new" | "resume";
type ProviderFilter = "all" | AgentProvider;
type DropdownKey = "assistant" | "permissions" | "model" | "workingDir" | "baseBranch";
@@ -152,7 +171,7 @@ function formatRelativeTime(date: Date): string {
return `${days}d ago`;
}
function getResumePreview(candidate: ResumeCandidate): string {
function getImportPreview(candidate: ImportCandidate): string {
for (const item of candidate.timeline) {
if (item.type === "user_message") {
const text = item.text.trim();
@@ -164,10 +183,12 @@ function getResumePreview(candidate: ResumeCandidate): string {
return candidate.title || candidate.cwd;
}
export function CreateAgentModal({
function AgentFlowModal({
isVisible,
onClose,
}: CreateAgentModalProps) {
flow,
initialValues,
}: AgentFlowModalProps) {
const insets = useSafeAreaInsets();
const { height: screenHeight, width: screenWidth } = useWindowDimensions();
const slideOffset = useSharedValue(screenHeight);
@@ -176,16 +197,20 @@ export function CreateAgentModal({
const isCompactLayout = screenWidth < 720;
const shouldHandlePromptDesktopSubmit = IS_WEB;
const shouldAutoFocusPrompt = IS_WEB;
const isImportFlow = flow === "import";
const isCreateFlow = !isImportFlow;
const { recentPaths, addRecentPath } = useRecentPaths();
const {
ws,
createAgent,
resumeAgent,
sendAgentAudio,
agents,
providerModels,
requestProviderModels,
} = useSession();
const isWsConnected = ws.isConnected;
const router = useRouter();
const [isMounted, setIsMounted] = useState(isVisible);
@@ -207,15 +232,18 @@ export function CreateAgentModal({
const [worktreeSlugEdited, setWorktreeSlugEdited] = useState(false);
const [errorMessage, setErrorMessage] = useState("");
const [isLoading, setIsLoading] = useState(false);
const [activeTab, setActiveTab] = useState<ResumeTab>("new");
const [resumeProviderFilter, setResumeProviderFilter] =
const [importProviderFilter, setImportProviderFilter] =
useState<ProviderFilter>("all");
const [resumeSearchQuery, setResumeSearchQuery] = useState("");
const [resumeCandidates, setResumeCandidates] = useState<ResumeCandidate[]>(
const [importSearchQuery, setImportSearchQuery] = useState("");
const [importCandidates, setImportCandidates] = useState<ImportCandidate[]>(
[]
);
const [isResumeLoading, setIsResumeLoading] = useState(false);
const [resumeError, setResumeError] = useState<string | null>(null);
const [isImportLoading, setIsImportLoading] = useState(false);
const [importError, setImportError] = useState<string | null>(null);
const [isDictating, setIsDictating] = useState(false);
const [isDictationProcessing, setIsDictationProcessing] = useState(false);
const [dictationVolume, setDictationVolume] = useState(0);
const [dictationDebugInfo, setDictationDebugInfo] = useState<AudioDebugInfo | null>(null);
const [openDropdown, setOpenDropdown] = useState<DropdownKey | null>(null);
const [repoInfo, setRepoInfo] = useState<RepoInfoState | null>(null);
const [repoInfoStatus, setRepoInfoStatus] = useState<
@@ -237,6 +265,19 @@ export function CreateAgentModal({
model: false,
workingDir: false,
});
const prevVisibilityRef = useRef(isVisible);
const dictationRequestIdRef = useRef<string | null>(null);
const dictationRecorder = useAudioRecorder({
onAudioLevel: (level) => {
setDictationVolume(level);
},
});
const shouldShowAudioDebug = AUDIO_DEBUG_ENABLED;
const hasPendingCreateOrResume = pendingRequestIdRef.current !== null;
const hasPendingDictation = dictationRequestIdRef.current !== null;
const shouldListenForStatus = isVisible || hasPendingCreateOrResume;
const shouldListenForDictation =
isCreateFlow && (isVisible || hasPendingDictation);
const pendingNavigationAgentIdRef = useRef<string | null>(null);
const openDropdownSheet = useCallback((key: DropdownKey) => {
@@ -276,6 +317,41 @@ export function CreateAgentModal({
[setWorkingDirFromUser]
);
const applyInitialValues = useCallback(() => {
if (!isCreateFlow || !initialValues) {
return;
}
if (Object.prototype.hasOwnProperty.call(initialValues, "workingDir")) {
const providedWorkingDir = initialValues.workingDir ?? "";
userEditedPreferencesRef.current.workingDir = true;
setWorkingDir(providedWorkingDir);
}
if (initialValues.provider && providerDefinitionMap.has(initialValues.provider)) {
userEditedPreferencesRef.current.provider = true;
setSelectedProvider(initialValues.provider);
}
if (typeof initialValues.modeId === "string" && initialValues.modeId.length > 0) {
userEditedPreferencesRef.current.mode = true;
setSelectedMode(initialValues.modeId);
}
if (typeof initialValues.model === "string" && initialValues.model.length > 0) {
userEditedPreferencesRef.current.model = true;
setSelectedModel(initialValues.model);
}
}, [initialValues, isCreateFlow]);
useEffect(() => {
const wasVisible = prevVisibilityRef.current;
if (isVisible && !wasVisible) {
applyInitialValues();
}
prevVisibilityRef.current = isVisible;
}, [applyInitialValues, isVisible]);
const refreshProviderModels = useCallback(() => {
const trimmed = workingDir.trim();
requestProviderModels(selectedProvider, {
@@ -373,13 +449,6 @@ export function CreateAgentModal({
void persist();
}, [selectedMode, selectedProvider, workingDir]);
const tabOptions = useMemo(
() => [
{ id: "new" as ResumeTab, label: "New Agent" },
{ id: "resume" as ResumeTab, label: "Resume Agent" },
],
[]
);
const providerFilterOptions = useMemo(
() => [
{ id: "all" as ProviderFilter, label: "All" },
@@ -403,6 +472,9 @@ export function CreateAgentModal({
const modelError = modelState?.error ?? null;
useEffect(() => {
if (!isVisible) {
return;
}
const trimmed = workingDir.trim();
const currentState = providerModels.get(selectedProvider);
if (currentState?.models?.length || currentState?.isLoading) {
@@ -411,7 +483,7 @@ export function CreateAgentModal({
requestProviderModels(selectedProvider, {
cwd: trimmed.length > 0 ? trimmed : undefined,
});
}, [providerModels, requestProviderModels, selectedProvider, workingDir]);
}, [isVisible, providerModels, requestProviderModels, selectedProvider, workingDir]);
const setPromptHeight = useCallback((nextHeight: number) => {
const bounded = Math.min(
PROMPT_MAX_HEIGHT,
@@ -538,10 +610,10 @@ export function CreateAgentModal({
});
return ids;
}, [agents]);
const filteredResumeCandidates = useMemo(() => {
const providerFilter = resumeProviderFilter;
const query = resumeSearchQuery.trim().toLowerCase();
return resumeCandidates
const filteredImportCandidates = useMemo(() => {
const providerFilter = importProviderFilter;
const query = importSearchQuery.trim().toLowerCase();
return importCandidates
.filter((candidate) => !activeSessionIds.has(candidate.sessionId))
.filter(
(candidate) =>
@@ -558,9 +630,9 @@ export function CreateAgentModal({
.sort((a, b) => b.lastActivityAt.getTime() - a.lastActivityAt.getTime());
}, [
activeSessionIds,
resumeCandidates,
resumeProviderFilter,
resumeSearchQuery,
importCandidates,
importProviderFilter,
importSearchQuery,
]);
useEffect(() => {
@@ -595,11 +667,11 @@ export function CreateAgentModal({
if (!shouldAutoFocusPrompt) {
return;
}
if (!isVisible || activeTab !== "new") {
if (!isVisible || !isCreateFlow) {
return;
}
focusPromptInput();
}, [activeTab, focusPromptInput, isVisible, shouldAutoFocusPrompt]);
}, [focusPromptInput, isCreateFlow, isVisible, shouldAutoFocusPrompt]);
const resetFormState = useCallback(() => {
setInitialPrompt("");
@@ -622,8 +694,74 @@ export function CreateAgentModal({
pendingRequestIdRef.current = null;
repoInfoRequestIdRef.current = null;
shouldSyncBaseBranchRef.current = true;
dictationRequestIdRef.current = null;
setIsDictating(false);
setIsDictationProcessing(false);
setDictationVolume(0);
setDictationDebugInfo(null);
if (dictationRecorder.isRecording?.()) {
void dictationRecorder.stop().catch(() => {});
}
}, [setPromptHeight]);
const handleDictationStart = useCallback(async () => {
if (!isCreateFlow || isLoading || !isWsConnected || isDictating || isDictationProcessing) {
return;
}
try {
if (shouldShowAudioDebug) {
setDictationDebugInfo(null);
}
await dictationRecorder.start();
setIsDictating(true);
setDictationVolume(0);
setErrorMessage("");
} catch (error) {
console.error("[CreateAgentModal] Failed to start dictation:", error);
}
}, [dictationRecorder, isCreateFlow, isDictating, isDictationProcessing, isLoading, isWsConnected, shouldShowAudioDebug]);
const handleDictationCancel = useCallback(async () => {
if (!isDictating) {
return;
}
try {
await dictationRecorder.stop();
} catch (error) {
console.error("[CreateAgentModal] Failed to cancel dictation:", error);
} finally {
setIsDictating(false);
setDictationVolume(0);
}
}, [dictationRecorder, isDictating]);
const handleDictationConfirm = useCallback(async () => {
if (!isDictating || isDictationProcessing) {
return;
}
try {
const audioData = await dictationRecorder.stop();
setIsDictating(false);
setDictationVolume(0);
if (!audioData) {
return;
}
const requestId = generateMessageId();
dictationRequestIdRef.current = requestId;
setIsDictationProcessing(true);
await sendAgentAudio(
DICTATION_AGENT_ID,
audioData,
requestId,
{ mode: "transcribe_only" }
);
} catch (error) {
console.error("[CreateAgentModal] Failed to complete dictation:", error);
dictationRequestIdRef.current = null;
setIsDictationProcessing(false);
}
}, [dictationRecorder, isDictating, isDictationProcessing, sendAgentAudio]);
const navigateToAgentIfNeeded = useCallback(() => {
const agentId = pendingNavigationAgentIdRef.current;
if (!agentId) {
@@ -643,16 +781,16 @@ export function CreateAgentModal({
navigateToAgentIfNeeded();
}, [navigateToAgentIfNeeded, resetFormState]);
const requestResumeCandidates = useCallback(
const requestImportCandidates = useCallback(
(provider?: AgentProvider) => {
setIsResumeLoading(true);
setResumeError(null);
setIsImportLoading(true);
setImportError(null);
const msg: WSInboundMessage = {
type: "session",
message: {
type: "list_persisted_agents_request",
...(provider ? { provider } : {}),
limit: RESUME_PAGE_SIZE,
limit: IMPORT_PAGE_SIZE,
},
};
try {
@@ -662,8 +800,8 @@ export function CreateAgentModal({
"[CreateAgentModal] Failed to request persisted agents:",
error
);
setIsResumeLoading(false);
setResumeError("Unable to load saved agents. Please try again.");
setIsImportLoading(false);
setImportError("Unable to load agents to import. Please try again.");
}
},
[ws]
@@ -736,6 +874,23 @@ export function CreateAgentModal({
handleCloseAnimationComplete,
]);
useEffect(() => {
if (isVisible || !dictationRecorder.isRecording?.()) {
return;
}
void dictationRecorder.stop().catch(() => {});
setIsDictating(false);
setDictationVolume(0);
}, [dictationRecorder, isVisible]);
useEffect(() => {
return () => {
if (dictationRecorder.isRecording?.()) {
void dictationRecorder.stop().catch(() => {});
}
};
}, [dictationRecorder]);
const footerAnimatedStyle = useAnimatedStyle(() => {
"worklet";
const absoluteHeight = Math.abs(keyboardHeight.value);
@@ -779,10 +934,10 @@ export function CreateAgentModal({
};
}
if (!/^[a-z0-9-]+$/.test(name)) {
if (!/^[a-z0-9-/]+$/.test(name)) {
return {
valid: false,
error: "Must contain only lowercase letters, numbers, and hyphens",
error: "Must contain only lowercase letters, numbers, hyphens, and forward slashes",
};
}
@@ -847,14 +1002,23 @@ export function CreateAgentModal({
);
useEffect(() => {
if (!isCreateFlow || !isVisible) {
return;
}
shouldSyncBaseBranchRef.current = true;
}, [workingDir]);
}, [isCreateFlow, isVisible, workingDir]);
useEffect(() => {
if (!isCreateFlow || !isVisible) {
return;
}
requestRepoInfo(workingDir);
}, [requestRepoInfo, workingDir]);
}, [isCreateFlow, isVisible, requestRepoInfo, workingDir]);
useEffect(() => {
if (!isCreateFlow || !isVisible) {
return;
}
const unsubscribe = ws.on("git_repo_info_response", (message) => {
if (message.type !== "git_repo_info_response") {
return;
@@ -899,7 +1063,7 @@ export function CreateAgentModal({
return () => {
unsubscribe();
};
}, [ws]);
}, [isCreateFlow, isVisible, ws]);
const handleCreate = useCallback(async () => {
const trimmedPath = workingDir.trim();
@@ -1000,8 +1164,8 @@ export function CreateAgentModal({
createAgent,
]);
const handleResumeCandidatePress = useCallback(
(candidate: ResumeCandidate) => {
const handleImportCandidatePress = useCallback(
(candidate: ImportCandidate) => {
if (isLoading) {
return;
}
@@ -1017,16 +1181,16 @@ export function CreateAgentModal({
[isLoading, resumeAgent]
);
const renderResumeItem = useCallback<ListRenderItem<ResumeCandidate>>(
const renderImportItem = useCallback<ListRenderItem<ImportCandidate>>(
({ item }) => (
<Pressable
onPress={() => handleResumeCandidatePress(item)}
onPress={() => handleImportCandidatePress(item)}
disabled={isLoading}
style={styles.resumeItem}
>
<View style={styles.resumeItemHeader}>
<Text style={styles.resumeItemTitle} numberOfLines={1}>
{getResumePreview(item)}
{getImportPreview(item)}
</Text>
<Text style={styles.resumeItemTimestamp}>
{formatRelativeTime(item.lastActivityAt)}
@@ -1041,14 +1205,17 @@ export function CreateAgentModal({
{getProviderLabel(item.provider)}
</Text>
</View>
<Text style={styles.resumeItemHint}>Tap to resume</Text>
<Text style={styles.resumeItemHint}>Tap to import</Text>
</View>
</Pressable>
),
[getProviderLabel, handleResumeCandidatePress, isLoading]
[getProviderLabel, handleImportCandidatePress, isLoading]
);
useEffect(() => {
if (!isImportFlow || !isVisible) {
return;
}
const unsubscribe = ws.on("list_persisted_agents_response", (message) => {
if (message.type !== "list_persisted_agents_response") {
return;
@@ -1061,18 +1228,21 @@ export function CreateAgentModal({
lastActivityAt: new Date(item.lastActivityAt),
persistence: item.persistence,
timeline: item.timeline ?? [],
})) as ResumeCandidate[];
})) as ImportCandidate[];
setResumeCandidates(mapped);
setIsResumeLoading(false);
setImportCandidates(mapped);
setIsImportLoading(false);
});
return () => {
unsubscribe();
};
}, [ws]);
}, [isImportFlow, isVisible, ws]);
useEffect(() => {
if (!shouldListenForStatus) {
return;
}
const unsubscribe = ws.on("status", (message) => {
if (message.type !== "status") {
return;
@@ -1122,24 +1292,81 @@ export function CreateAgentModal({
return () => {
unsubscribe();
};
}, [ws, handleClose]);
}, [handleClose, shouldListenForStatus, ws]);
useEffect(() => {
if (!isVisible || activeTab !== "resume") {
if (!shouldListenForDictation) {
return;
}
const unsubscribe = ws.on("transcription_result", (message) => {
if (message.type !== "transcription_result") {
return;
}
const pendingId = dictationRequestIdRef.current;
if (!pendingId || message.payload.requestId !== pendingId) {
return;
}
dictationRequestIdRef.current = null;
setIsDictationProcessing(false);
if (shouldShowAudioDebug) {
setDictationDebugInfo({
requestId: pendingId,
transcript: message.payload.text?.trim(),
debugRecordingPath: message.payload.debugRecordingPath ?? undefined,
format: message.payload.format,
byteLength: message.payload.byteLength,
duration: message.payload.duration,
avgLogprob: message.payload.avgLogprob,
isLowConfidence: message.payload.isLowConfidence,
});
}
const transcriptText = message.payload.text?.trim();
if (!transcriptText) {
return;
}
setInitialPrompt((prev) => {
if (!prev) {
return transcriptText;
}
const needsSpace = /\s$/.test(prev);
return `${prev}${needsSpace ? "" : " "}${transcriptText}`;
});
focusPromptInput();
});
return () => {
unsubscribe();
};
}, [focusPromptInput, shouldListenForDictation, ws, shouldShowAudioDebug]);
useEffect(() => {
if (!isVisible || !isImportFlow) {
return;
}
const provider =
resumeProviderFilter === "all" ? undefined : resumeProviderFilter;
requestResumeCandidates(provider);
}, [activeTab, isVisible, requestResumeCandidates, resumeProviderFilter]);
importProviderFilter === "all" ? undefined : importProviderFilter;
requestImportCandidates(provider);
}, [importProviderFilter, isImportFlow, isVisible, requestImportCandidates]);
const refreshResumeList = useCallback(() => {
const refreshImportList = useCallback(() => {
const provider =
resumeProviderFilter === "all" ? undefined : resumeProviderFilter;
requestResumeCandidates(provider);
}, [requestResumeCandidates, resumeProviderFilter]);
importProviderFilter === "all" ? undefined : importProviderFilter;
requestImportCandidates(provider);
}, [requestImportCandidates, importProviderFilter]);
const shouldRender = isVisible || isMounted;
const modalTitle = isImportFlow ? "Import Agent" : "Create New Agent";
const dictationAccessory = isCreateFlow ? (
<PromptDictationControls
isRecording={isDictating}
isProcessing={isDictationProcessing}
disabled={isLoading || !isWsConnected}
volume={dictationVolume}
onStart={handleDictationStart}
onCancel={handleDictationCancel}
onConfirm={handleDictationConfirm}
/>
) : null;
const gitBlockingError = useMemo(() => {
const trimmedBase = baseBranch.trim();
@@ -1158,7 +1385,7 @@ export function CreateAgentModal({
const validation = validateWorktreeName(slug);
if (!slug || !validation.valid) {
return `Invalid branch name: ${
validation.error ?? "Must use lowercase letters, numbers, or hyphens"
validation.error ?? "Must use lowercase letters, numbers, hyphens, or forward slashes"
}`;
}
}
@@ -1273,44 +1500,9 @@ export function CreateAgentModal({
paddingLeft={horizontalPaddingLeft}
paddingRight={horizontalPaddingRight}
onClose={handleClose}
title={
activeTab === "resume" ? "Resume Agent" : "Create New Agent"
}
title={modalTitle}
/>
<View
style={[
styles.tabSelector,
{
paddingLeft: horizontalPaddingLeft,
paddingRight: horizontalPaddingRight,
},
]}
>
{tabOptions.map((tab) => {
const isActive = activeTab === tab.id;
return (
<Pressable
key={tab.id}
onPress={() => setActiveTab(tab.id)}
style={[
styles.tabButton,
isActive && styles.tabButtonActive,
]}
>
<Text
style={[
styles.tabButtonText,
isActive && styles.tabButtonTextActive,
]}
>
{tab.label}
</Text>
</Pressable>
);
})}
</View>
{activeTab === "new" ? (
{isCreateFlow ? (
<>
<ScrollView
style={styles.scroll}
@@ -1338,8 +1530,17 @@ export function CreateAgentModal({
onDesktopSubmit={handlePromptDesktopSubmitKeyPress}
autoFocus={shouldAutoFocusPrompt}
scrollEnabled={promptInputHeight >= PROMPT_MAX_HEIGHT}
accessory={dictationAccessory}
/>
{shouldShowAudioDebug && dictationDebugInfo ? (
<AudioDebugNotice
info={dictationDebugInfo}
onDismiss={() => setDictationDebugInfo(null)}
title="Prompt Dictation Debug"
/>
) : null}
<View
style={[
styles.selectorRow,
@@ -1502,11 +1703,11 @@ export function CreateAgentModal({
<View style={styles.resumeFilters}>
<View style={styles.providerFilterRow}>
{providerFilterOptions.map((option) => {
const isActive = resumeProviderFilter === option.id;
const isActive = importProviderFilter === option.id;
return (
<Pressable
key={option.id}
onPress={() => setResumeProviderFilter(option.id)}
onPress={() => setImportProviderFilter(option.id)}
style={[
styles.providerFilterButton,
isActive && styles.providerFilterButtonActive,
@@ -1529,48 +1730,48 @@ export function CreateAgentModal({
style={styles.resumeSearchInput}
placeholder="Search by title or path"
placeholderTextColor={defaultTheme.colors.mutedForeground}
value={resumeSearchQuery}
onChangeText={setResumeSearchQuery}
value={importSearchQuery}
onChangeText={setImportSearchQuery}
/>
<Pressable
style={styles.refreshButton}
onPress={refreshResumeList}
disabled={isResumeLoading}
onPress={refreshImportList}
disabled={isImportLoading}
>
<Text style={styles.refreshButtonText}>Refresh</Text>
</Pressable>
</View>
</View>
{resumeError ? (
<Text style={styles.resumeErrorText}>{resumeError}</Text>
{importError ? (
<Text style={styles.importErrorText}>{importError}</Text>
) : null}
{isResumeLoading ? (
{isImportLoading ? (
<View style={styles.resumeLoading}>
<ActivityIndicator
color={defaultTheme.colors.mutedForeground}
/>
<Text style={styles.resumeLoadingText}>
Loading saved agents...
Loading agents to import...
</Text>
</View>
) : filteredResumeCandidates.length === 0 ? (
) : filteredImportCandidates.length === 0 ? (
<View style={styles.resumeEmptyState}>
<Text style={styles.resumeEmptyTitle}>No agents found</Text>
<Text style={styles.resumeEmptyTitle}>No agents to import</Text>
<Text style={styles.resumeEmptySubtitle}>
We'll load the latest Claude and Codex sessions from your
local history.
local history so you can import them.
</Text>
<Pressable
style={styles.refreshButtonAlt}
onPress={refreshResumeList}
onPress={refreshImportList}
>
<Text style={styles.refreshButtonAltText}>Try Again</Text>
</Pressable>
</View>
) : (
<FlatList
data={filteredResumeCandidates}
renderItem={renderResumeItem}
data={filteredImportCandidates}
renderItem={renderImportItem}
keyExtractor={(item) =>
`${item.provider}:${item.sessionId}`
}
@@ -1590,6 +1791,14 @@ export function CreateAgentModal({
);
}
export function CreateAgentModal(props: ModalWrapperProps) {
return <AgentFlowModal {...props} flow="create" />;
}
export function ImportAgentModal(props: ModalWrapperProps) {
return <AgentFlowModal {...props} flow="import" />;
}
interface ModalHeaderProps {
paddingTop: number;
paddingLeft: number;
@@ -2078,6 +2287,7 @@ interface PromptSectionProps {
onDesktopSubmit?: (event: WebTextInputKeyPressEvent) => void;
autoFocus?: boolean;
scrollEnabled: boolean;
accessory?: ReactNode;
}
function PromptSection({
@@ -2090,10 +2300,14 @@ function PromptSection({
onDesktopSubmit,
autoFocus,
scrollEnabled,
accessory,
}: PromptSectionProps): ReactElement {
return (
<View style={styles.formSection}>
<Text style={styles.label}>Initial Prompt</Text>
<View style={styles.labelRow}>
<Text style={styles.label}>Initial Prompt</Text>
{accessory}
</View>
<TextInput
ref={inputRef}
style={[
@@ -2124,6 +2338,86 @@ function PromptSection({
);
}
interface PromptDictationControlsProps {
isRecording: boolean;
isProcessing: boolean;
disabled: boolean;
volume: number;
onStart: () => void;
onCancel: () => void;
onConfirm: () => void;
}
function PromptDictationControls({
isRecording,
isProcessing,
disabled,
volume,
onStart,
onCancel,
onConfirm,
}: PromptDictationControlsProps): ReactElement {
const { theme } = useUnistyles();
if (!isRecording && !isProcessing) {
return (
<Pressable
onPress={onStart}
disabled={disabled}
accessibilityRole="button"
accessibilityLabel="Start voice dictation"
style={[styles.dictationButton, disabled && styles.dictationButtonDisabled]}
>
<Mic size={16} color={theme.colors.foreground} />
</Pressable>
);
}
return (
<View style={styles.dictationActiveContainer}>
<View style={styles.dictationMeterWrapper}>
<VolumeMeter
volume={volume}
isMuted={false}
isDetecting
isSpeaking={false}
orientation="horizontal"
/>
</View>
<View style={styles.dictationActionGroup}>
<Pressable
onPress={onCancel}
disabled={isProcessing}
accessibilityLabel="Cancel dictation"
style={[
styles.dictationActionButton,
styles.dictationActionButtonCancel,
isProcessing ? styles.dictationActionButtonDisabled : undefined,
]}
>
<X size={14} color={theme.colors.foreground} />
</Pressable>
<Pressable
onPress={onConfirm}
disabled={isProcessing}
accessibilityLabel="Insert transcription"
style={[
styles.dictationActionButton,
styles.dictationActionButtonConfirm,
isProcessing ? styles.dictationActionButtonDisabled : undefined,
]}
>
{isProcessing ? (
<ActivityIndicator size="small" color={theme.colors.background} />
) : (
<Check size={14} color={theme.colors.background} />
)}
</Pressable>
</View>
</View>
);
}
interface GitOptionsSectionProps {
baseBranch: string;
onBaseBranchChange: (value: string) => void;
@@ -2401,35 +2695,6 @@ const styles = StyleSheet.create(((theme: any) => ({
alignItems: "center",
justifyContent: "center",
},
tabSelector: {
flexDirection: "row",
gap: theme.spacing[2],
paddingTop: theme.spacing[4],
paddingBottom: theme.spacing[4],
},
tabButton: {
flex: 1,
borderRadius: theme.borderRadius.lg,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
paddingVertical: theme.spacing[3],
alignItems: "center",
justifyContent: "center",
backgroundColor: theme.colors.background,
},
tabButtonActive: {
backgroundColor: theme.colors.muted,
borderColor: theme.colors.palette.blue[500],
},
tabButtonText: {
color: theme.colors.mutedForeground,
fontSize: theme.fontSize.sm,
fontWeight: theme.fontWeight.normal,
},
tabButtonTextActive: {
color: theme.colors.foreground,
fontWeight: theme.fontWeight.semibold,
},
scroll: {
flex: 1,
},
@@ -2441,6 +2706,12 @@ const styles = StyleSheet.create(((theme: any) => ({
formSection: {
gap: theme.spacing[3],
},
labelRow: {
flexDirection: "row",
alignItems: "center",
justifyContent: "space-between",
gap: theme.spacing[3],
},
label: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
@@ -2587,6 +2858,50 @@ const styles = StyleSheet.create(((theme: any) => ({
color: theme.colors.mutedForeground,
fontSize: theme.fontSize.sm,
},
dictationButton: {
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
borderRadius: theme.borderRadius.full,
padding: theme.spacing[2],
alignItems: "center",
justifyContent: "center",
},
dictationButtonDisabled: {
opacity: theme.opacity[50],
},
dictationActiveContainer: {
flexDirection: "row",
alignItems: "center",
gap: theme.spacing[2],
},
dictationMeterWrapper: {
width: 48,
alignItems: "center",
justifyContent: "center",
},
dictationActionGroup: {
flexDirection: "row",
gap: theme.spacing[2],
},
dictationActionButton: {
width: 32,
height: 32,
borderRadius: theme.borderRadius.full,
alignItems: "center",
justifyContent: "center",
borderWidth: theme.borderWidth[1],
},
dictationActionButtonCancel: {
borderColor: theme.colors.border,
backgroundColor: theme.colors.background,
},
dictationActionButtonConfirm: {
borderColor: theme.colors.foreground,
backgroundColor: theme.colors.foreground,
},
dictationActionButtonDisabled: {
opacity: theme.opacity[40],
},
selectorRow: {
flexDirection: "row",
gap: theme.spacing[4],
@@ -2727,7 +3042,7 @@ const styles = StyleSheet.create(((theme: any) => ({
fontSize: theme.fontSize.sm,
fontWeight: theme.fontWeight.semibold,
},
resumeErrorText: {
importErrorText: {
color: theme.colors.palette.red[500],
fontSize: theme.fontSize.sm,
},

View File

@@ -1,40 +1,81 @@
import { Pressable } from "react-native";
import { useCallback, useState } from "react";
import { Modal, Pressable, Text, View } from "react-native";
import { router } from "expo-router";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { Settings, MessageSquare, Plus } from "lucide-react-native";
import { Settings, MessageSquare, MoreVertical, Plus } from "lucide-react-native";
import { ScreenHeader } from "./screen-header";
interface HomeHeaderProps {
onCreateAgent: () => void;
onImportAgent: () => void;
}
export function HomeHeader({ onCreateAgent }: HomeHeaderProps) {
export function HomeHeader({ onCreateAgent, onImportAgent }: HomeHeaderProps) {
const { theme } = useUnistyles();
const insets = useSafeAreaInsets();
const [isMenuVisible, setIsMenuVisible] = useState(false);
const openMenu = useCallback(() => setIsMenuVisible(true), []);
const closeMenu = useCallback(() => setIsMenuVisible(false), []);
const handleImportPress = useCallback(() => {
closeMenu();
onImportAgent();
}, [closeMenu, onImportAgent]);
return (
<ScreenHeader
left={
<Pressable
onPress={() => router.push("/settings")}
style={styles.iconButton}
>
<Settings size={20} color={theme.colors.foreground} />
</Pressable>
}
right={
<>
<>
<ScreenHeader
left={
<Pressable
onPress={() => router.push("/orchestrator")}
onPress={() => router.push("/settings")}
style={styles.iconButton}
>
<MessageSquare size={20} color={theme.colors.foreground} />
<Settings size={20} color={theme.colors.foreground} />
</Pressable>
<Pressable onPress={onCreateAgent} style={styles.iconButton}>
<Plus size={20} color={theme.colors.foreground} />
</Pressable>
</>
}
/>
}
right={
<>
<Pressable
onPress={() => router.push("/orchestrator")}
style={styles.iconButton}
>
<MessageSquare size={20} color={theme.colors.foreground} />
</Pressable>
<Pressable onPress={onCreateAgent} style={styles.iconButton}>
<Plus size={20} color={theme.colors.foreground} />
</Pressable>
<Pressable onPress={openMenu} style={styles.iconButton}>
<MoreVertical size={20} color={theme.colors.foreground} />
</Pressable>
</>
}
/>
<Modal
visible={isMenuVisible}
transparent
animationType="fade"
onRequestClose={closeMenu}
>
<View style={styles.menuOverlay}>
<Pressable style={styles.menuBackdrop} onPress={closeMenu} />
<View
style={[
styles.menuContainer,
{
top: insets.top + theme.spacing[4],
right: theme.spacing[3],
},
]}
>
<Pressable style={styles.menuItem} onPress={handleImportPress}>
<Text style={styles.menuItemText}>Import Agent</Text>
</Pressable>
</View>
</View>
</Modal>
</>
);
}
@@ -43,4 +84,32 @@ const styles = StyleSheet.create((theme) => ({
padding: theme.spacing[3],
borderRadius: theme.borderRadius.lg,
},
menuOverlay: {
flex: 1,
},
menuBackdrop: {
...StyleSheet.absoluteFillObject,
},
menuContainer: {
position: "absolute",
minWidth: 180,
borderRadius: theme.borderRadius.xl,
backgroundColor: theme.colors.background,
borderWidth: theme.borderWidth[1],
borderColor: theme.colors.border,
shadowColor: "#000",
shadowOpacity: 0.15,
shadowRadius: 8,
shadowOffset: { width: 0, height: 4 },
elevation: 4,
},
menuItem: {
paddingVertical: theme.spacing[3],
paddingHorizontal: theme.spacing[4],
},
menuItemText: {
color: theme.colors.foreground,
fontSize: theme.fontSize.sm,
fontWeight: theme.fontWeight.semibold,
},
}));

View File

@@ -0,0 +1,2 @@
export const AUDIO_DEBUG_ENABLED =
typeof process !== "undefined" && process.env?.EXPO_PUBLIC_ENABLE_AUDIO_DEBUG === "1";

View File

@@ -38,6 +38,11 @@ const derivePendingPermissionKey = (agentId: string, request: AgentPermissionReq
return `${agentId}:${fallbackId}`;
};
type DraftInput = {
text: string;
images: Array<{ uri: string; mimeType: string }>;
};
export type MessageEntry =
| {
type: "user";
@@ -237,8 +242,8 @@ interface SessionContextValue {
initializingAgents: Map<string, boolean>;
// Queued messages and draft input per agent
draftInputs: Map<string, { text: string; images: Array<{ uri: string; mimeType: string }> }>;
setDraftInputs: (value: Map<string, { text: string; images: Array<{ uri: string; mimeType: string }> }> | ((prev: Map<string, { text: string; images: Array<{ uri: string; mimeType: string }> }>) => Map<string, { text: string; images: Array<{ uri: string; mimeType: string }> }>)) => void;
getDraftInput: (agentId: string) => DraftInput | undefined;
saveDraftInput: (agentId: string, draft: DraftInput) => void;
queuedMessages: Map<string, Array<{ id: string; text: string; images?: Array<{ uri: string; mimeType: string }> }>>;
setQueuedMessages: (value: Map<string, Array<{ id: string; text: string; images?: Array<{ uri: string; mimeType: string }> }>> | ((prev: Map<string, Array<{ id: string; text: string; images?: Array<{ uri: string; mimeType: string }> }>>) => Map<string, Array<{ id: string; text: string; images?: Array<{ uri: string; mimeType: string }> }>>)) => void;
@@ -271,7 +276,12 @@ interface SessionContextValue {
refreshAgent: (params: { agentId: string; requestId?: string }) => void;
cancelAgentRun: (agentId: string) => void;
sendAgentMessage: (agentId: string, message: string, imageUris?: string[]) => Promise<void>;
sendAgentAudio: (agentId: string, audioBlob: Blob, requestId?: string) => Promise<void>;
sendAgentAudio: (
agentId: string,
audioBlob: Blob,
requestId?: string,
options?: { mode?: "transcribe_only" | "auto_run" }
) => Promise<void>;
deleteAgent: (agentId: string) => void;
createAgent: (options: {
config: AgentSessionConfig;
@@ -327,7 +337,17 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
const [gitDiffs, setGitDiffs] = useState<Map<string, string>>(new Map());
const [fileExplorer, setFileExplorer] = useState<Map<string, AgentFileExplorerState>>(new Map());
const [providerModels, setProviderModels] = useState<Map<AgentProvider, ProviderModelState>>(new Map());
const [draftInputs, setDraftInputs] = useState<SessionContextValue["draftInputs"]>(new Map());
const draftInputsRef = useRef<Map<string, DraftInput>>(new Map());
const getDraftInput = useCallback<SessionContextValue["getDraftInput"]>((agentId) => {
return draftInputsRef.current.get(agentId);
}, []);
const saveDraftInput = useCallback<SessionContextValue["saveDraftInput"]>((agentId, draft) => {
draftInputsRef.current.set(agentId, {
text: draft.text,
images: draft.images,
});
}, []);
const [queuedMessages, setQueuedMessages] = useState<SessionContextValue["queuedMessages"]>(new Map());
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
const previousAgentStatusRef = useRef<Map<string, AgentLifecycleStatus>>(new Map());
@@ -1000,6 +1020,8 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
return next;
});
draftInputsRef.current.delete(agentId);
setPendingPermissions((prev) => {
let changed = false;
const next = new Map(prev);
@@ -1253,7 +1275,12 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
}, [ws]);
const sendAgentAudio = useCallback(async (agentId: string, audioBlob: Blob, requestId?: string) => {
const sendAgentAudio = useCallback(async (
agentId: string,
audioBlob: Blob,
requestId?: string,
options?: { mode?: "transcribe_only" | "auto_run" }
) => {
try {
// Convert blob to base64
const arrayBuffer = await audioBlob.arrayBuffer();
@@ -1264,8 +1291,21 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
}
const base64Audio = btoa(binary);
// Determine format from MIME type
const format = audioBlob.type.split('/')[1] || 'm4a';
const deriveFormat = (mimeType: string | undefined): string => {
if (!mimeType || mimeType.length === 0) {
return "webm";
}
const slashIndex = mimeType.indexOf("/");
let formatPart = slashIndex >= 0 ? mimeType.slice(slashIndex + 1) : mimeType;
const semicolonIndex = formatPart.indexOf(";");
if (semicolonIndex >= 0) {
formatPart = formatPart.slice(0, semicolonIndex);
}
return formatPart.trim().length > 0 ? formatPart.trim() : "webm";
};
// Determine format from MIME type (strip codec metadata)
const format = deriveFormat(audioBlob.type);
// Send audio message
const msg: WSInboundMessage = {
@@ -1277,6 +1317,7 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
format,
isLast: true,
requestId,
...(options?.mode ? { mode: options.mode } : {}),
},
};
ws.send(msg);
@@ -1469,8 +1510,8 @@ export function SessionProvider({ children, serverUrl }: SessionProviderProps) {
fileExplorer,
providerModels,
requestProviderModels,
draftInputs,
setDraftInputs,
getDraftInput,
saveDraftInput,
queuedMessages,
setQueuedMessages,
requestDirectoryListing,

View File

@@ -72,6 +72,9 @@ See [IMPLEMENTATION_PLAN.md](./IMPLEMENTATION_PLAN.md) for complete details.
```bash
OPENAI_API_KEY=sk-... # GPT-4 and TTS
DEEPGRAM_API_KEY=... # Streaming STT
STT_MODEL=whisper-1 # Optional: override to gpt-4o-transcribe, etc.
STT_CONFIDENCE_THRESHOLD=-3.0 # Optional: reject low-confidence clips
STT_DEBUG_AUDIO_DIR=.stt-debug # Optional: persist raw dictation audio for debugging
PORT=3000 # Server port
NODE_ENV=development # Environment
```

View File

@@ -19,6 +19,7 @@ import type {
ListPersistedAgentsOptions,
PersistedAgentDescriptor,
} from "./agent-sdk-types.js";
import { resolveAgentModel } from "./model-resolver.js";
export type AgentLifecycleStatus =
| "initializing"
@@ -31,6 +32,7 @@ export type AgentSnapshot = {
id: string;
provider: AgentProvider;
cwd: string;
model: string | null;
createdAt: Date;
updatedAt: Date;
lastUserMessageAt: Date | null;
@@ -203,9 +205,14 @@ export class AgentManager {
config: AgentSessionConfig,
agentId?: string
): Promise<AgentSnapshot> {
const client = this.requireClient(config.provider);
const session = await client.createSession(config);
return this.registerSession(session, config, agentId ?? this.idFactory());
const normalizedConfig = await this.normalizeConfig(config);
const client = this.requireClient(normalizedConfig.provider);
const session = await client.createSession(normalizedConfig);
return this.registerSession(
session,
normalizedConfig,
agentId ?? this.idFactory()
);
}
async resumeAgent(
@@ -213,17 +220,22 @@ export class AgentManager {
overrides?: Partial<AgentSessionConfig>,
agentId?: string
): Promise<AgentSnapshot> {
const client = this.requireClient(handle.provider);
const session = await client.resumeSession(handle, overrides);
const metadata = (handle.metadata ?? {}) as Partial<AgentSessionConfig>;
const mergedConfig = {
...metadata,
...overrides,
provider: handle.provider,
} as AgentSessionConfig;
const normalizedConfig = await this.normalizeConfig(mergedConfig);
const resumeOverrides =
normalizedConfig.model !== mergedConfig.model
? { ...overrides, model: normalizedConfig.model }
: overrides;
const client = this.requireClient(handle.provider);
const session = await client.resumeSession(handle, resumeOverrides);
return this.registerSession(
session,
mergedConfig,
normalizedConfig,
agentId ?? this.idFactory()
);
}
@@ -608,6 +620,7 @@ export class AgentManager {
id: agent.id,
provider: agent.provider,
cwd: agent.cwd,
model: agent.config.model ?? null,
createdAt: agent.createdAt,
updatedAt: agent.updatedAt,
lastUserMessageAt: agent.lastUserMessageAt,
@@ -623,6 +636,26 @@ export class AgentManager {
};
}
private async normalizeConfig(
config: AgentSessionConfig
): Promise<AgentSessionConfig> {
const normalized: AgentSessionConfig = { ...config };
const resolvedModel = await resolveAgentModel({
provider: normalized.provider,
requestedModel: normalized.model,
cwd: normalized.cwd,
});
if (resolvedModel) {
normalized.model = resolvedModel;
} else if (typeof normalized.model === "string") {
const trimmed = normalized.model.trim();
normalized.model = trimmed.length > 0 ? trimmed : undefined;
}
return normalized;
}
private requireClient(provider: AgentProvider): AgentClient {
const client = this.clients.get(provider);
if (!client) {

View File

@@ -12,6 +12,7 @@ function createSnapshot(overrides?: Partial<AgentSnapshot>): AgentSnapshot {
id: "agent-test",
provider: "claude",
cwd: "/tmp/project",
model: null,
createdAt: now,
updatedAt: now,
lastUserMessageAt: null,

View File

@@ -0,0 +1,20 @@
const AUDIO_EXTENSIONS: Array<{ match: (format: string) => boolean; ext: string }> = [
{ match: (format) => format.includes("webm"), ext: "webm" },
{ match: (format) => format.includes("ogg"), ext: "ogg" },
{ match: (format) => format.includes("mp3"), ext: "mp3" },
{ match: (format) => format.includes("wav"), ext: "wav" },
{ match: (format) => format.includes("m4a") || format.includes("aac"), ext: "m4a" },
{ match: (format) => format.includes("mp4"), ext: "mp4" },
{ match: (format) => format.includes("flac"), ext: "flac" },
];
export function inferAudioExtension(format: string | undefined): string {
const normalized = (format || "webm").toLowerCase();
const candidate = AUDIO_EXTENSIONS.find((entry) => entry.match(normalized));
return candidate?.ext ?? "webm";
}
export function sanitizeForFilename(segment: string | undefined, fallback: string): string {
const value = segment && segment.length > 0 ? segment : fallback;
return value.replace(/[^a-zA-Z0-9-_]/g, "_").slice(0, 64);
}

View File

@@ -1,7 +1,7 @@
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
import path from "node:path";
import readline from "node:readline";
import { fileURLToPath } from "node:url";
import { fileURLToPath, URL } from "node:url";
import {
query,
@@ -105,7 +105,9 @@ function emptySdkMessageStream(): AsyncIterable<SDKUserMessage> {
}
function resolveCodexBinary(): string {
const repoRoot = path.resolve(fileURLToPath(new URL("../../../../..", import.meta.url)));
const repoRoot = path.resolve(
fileURLToPath(new URL("../../../../..", import.meta.url))
);
const packageRoot = path.join(repoRoot, "node_modules", "@openai", "codex-sdk");
const vendorDir = path.join(packageRoot, "vendor");
@@ -144,7 +146,7 @@ type CodexModelInfo = {
type PendingRequest = {
resolve: (value: unknown) => void;
reject: (error: Error) => void;
timer: NodeJS.Timeout;
timer: ReturnType<typeof setTimeout>;
};
class CodexAppServerClient {

View File

@@ -0,0 +1,61 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { resolveAgentModel } from "./model-resolver.js";
vi.mock("./model-catalog.js", () => ({
fetchProviderModelCatalog: vi.fn(),
}));
import { fetchProviderModelCatalog } from "./model-catalog.js";
const mockedFetch = vi.mocked(fetchProviderModelCatalog);
describe("resolveAgentModel", () => {
beforeEach(() => {
mockedFetch.mockReset();
});
it("returns the trimmed requested model when provided", async () => {
const result = await resolveAgentModel({
provider: "codex",
requestedModel: " gpt-5.1 ",
cwd: "/tmp",
});
expect(result).toBe("gpt-5.1");
expect(mockedFetch).not.toHaveBeenCalled();
});
it("uses the default model from the provider catalog when no model specified", async () => {
mockedFetch.mockResolvedValue([
{ id: "claude-3.5-haiku", isDefault: false } as any,
{ id: "claude-3.5-sonnet", isDefault: true } as any,
]);
const result = await resolveAgentModel({ provider: "claude", cwd: "~/repo" });
expect(result).toBe("claude-3.5-sonnet");
expect(mockedFetch).toHaveBeenCalledWith("claude", {
cwd: expect.stringMatching(/repo$/),
});
});
it("falls back to the first model when none are flagged as default", async () => {
mockedFetch.mockResolvedValue([
{ id: "model-a", isDefault: false } as any,
{ id: "model-b", isDefault: false } as any,
]);
const result = await resolveAgentModel({ provider: "codex" });
expect(result).toBe("model-a");
});
it("returns undefined when the catalog lookup fails", async () => {
mockedFetch.mockRejectedValue(new Error("boom"));
const result = await resolveAgentModel({ provider: "codex" });
expect(result).toBeUndefined();
});
});

View File

@@ -0,0 +1,32 @@
import { fetchProviderModelCatalog } from "./model-catalog.js";
import type { AgentProvider } from "./agent-sdk-types.js";
import { expandTilde } from "../terminal-mcp/tmux.js";
type ResolveAgentModelOptions = {
provider: AgentProvider;
requestedModel?: string | null;
cwd?: string;
};
export async function resolveAgentModel(
options: ResolveAgentModelOptions
): Promise<string | undefined> {
const trimmed = options.requestedModel?.trim();
if (trimmed) {
return trimmed;
}
try {
const models = await fetchProviderModelCatalog(options.provider, {
cwd: options.cwd ? expandTilde(options.cwd) : undefined,
});
const preferred = models.find((model) => model.isDefault) ?? models[0];
return preferred?.id;
} catch (error) {
console.warn(
`[AgentModelResolver] Failed to resolve default model for ${options.provider}:`,
error
);
return undefined;
}
}

View File

@@ -0,0 +1,50 @@
import { mkdir, writeFile } from "fs/promises";
import { join, resolve } from "path";
import { inferAudioExtension, sanitizeForFilename } from "./audio-utils.js";
const debugDir = process.env.STT_DEBUG_AUDIO_DIR
? resolve(process.env.STT_DEBUG_AUDIO_DIR)
: null;
let announced = false;
export interface DebugAudioMetadata {
sessionId: string;
agentId?: string;
requestId?: string;
label?: string;
format: string;
}
export async function maybePersistDebugAudio(
audio: Buffer,
metadata: DebugAudioMetadata
): Promise<string | null> {
if (!debugDir) {
return null;
}
if (!announced) {
console.log(`[STT][Debug] Raw audio capture enabled at ${debugDir}`);
announced = true;
}
const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
const folder = join(debugDir, sanitizeForFilename(metadata.sessionId, "session"));
await mkdir(folder, { recursive: true });
const parts = [timestamp];
if (metadata.agentId) {
parts.push(sanitizeForFilename(metadata.agentId, "agent"));
}
if (metadata.label) {
parts.push(sanitizeForFilename(metadata.label, "source"));
}
if (metadata.requestId) {
parts.push(sanitizeForFilename(metadata.requestId, "request"));
}
const ext = inferAudioExtension(metadata.format);
const filePath = join(folder, `${parts.join("_")}.${ext}`);
await writeFile(filePath, audio);
return filePath;
}

View File

@@ -1,4 +1,17 @@
import { transcribeAudio, type TranscriptionResult } from "./stt-openai.js";
import { maybePersistDebugAudio } from "./stt-debug.js";
interface TranscriptionMetadata {
agentId?: string;
requestId?: string;
label?: string;
}
export interface SessionTranscriptionResult extends TranscriptionResult {
debugRecordingPath?: string;
byteLength: number;
format: string;
}
/**
* Per-session STT manager
@@ -16,12 +29,30 @@ export class STTManager {
*/
public async transcribe(
audio: Buffer,
format: string
): Promise<TranscriptionResult> {
format: string,
metadata?: TranscriptionMetadata
): Promise<SessionTranscriptionResult> {
const context = metadata?.label ? ` (${metadata.label})` : "";
console.log(
`[STT-Manager ${this.sessionId}] Transcribing ${audio.length} bytes of ${format} audio`
`[STT-Manager ${this.sessionId}] Transcribing ${audio.length} bytes of ${format} audio${context}`
);
let debugRecordingPath: string | null = null;
try {
debugRecordingPath = await maybePersistDebugAudio(audio, {
sessionId: this.sessionId,
agentId: metadata?.agentId,
requestId: metadata?.requestId,
label: metadata?.label,
format,
});
} catch (error) {
console.warn(
`[STT-Manager ${this.sessionId}] Failed to persist debug audio:`,
error
);
}
const result = await transcribeAudio(audio, format);
// Filter out low-confidence transcriptions (non-speech sounds)
@@ -34,6 +65,9 @@ export class STTManager {
return {
...result,
text: "",
byteLength: audio.length,
format,
debugRecordingPath: debugRecordingPath ?? undefined,
};
}
@@ -43,7 +77,12 @@ export class STTManager {
}`
);
return result;
return {
...result,
debugRecordingPath: debugRecordingPath ?? undefined,
byteLength: audio.length,
format,
};
}
/**

View File

@@ -3,10 +3,11 @@ import { writeFile, unlink } from "fs/promises";
import { join } from "path";
import { tmpdir } from "os";
import { v4 as uuidv4 } from "uuid";
import { inferAudioExtension } from "./audio-utils.js";
export interface STTConfig {
apiKey: string;
model?: "whisper-1";
model?: "whisper-1" | "gpt-4o-transcribe" | "gpt-4o-mini-transcribe" | (string & {});
confidenceThreshold?: number; // Default: -3.0
}
@@ -49,7 +50,7 @@ export async function transcribeAudio(
try {
// Map format to file extension
const ext = getFileExtension(format);
const ext = inferAudioExtension(format);
// Write audio buffer to temporary file
// OpenAI API requires file upload, not raw buffer
@@ -61,11 +62,15 @@ export async function transcribeAudio(
);
// Call OpenAI Whisper API
const modelToUse = config.model ?? "whisper-1";
const supportsLogprobs =
modelToUse === "gpt-4o-transcribe" || modelToUse === "gpt-4o-mini-transcribe";
const response = await openaiClient.audio.transcriptions.create({
file: await import("fs").then((fs) => fs.createReadStream(tempFilePath!)),
language: "en",
model: config.model ?? "gpt-4o-transcribe",
include: ["logprobs"],
model: modelToUse,
...(supportsLogprobs ? { include: ["logprobs"] as ["logprobs"] } : {}),
response_format: "json", // Get language and duration info
});
@@ -77,7 +82,9 @@ export async function transcribeAudio(
// Analyze logprobs if available
let avgLogprob: number | undefined;
let isLowConfidence = false;
const logprobs = response.logprobs as LogprobToken[] | undefined;
const logprobs = supportsLogprobs
? (response.logprobs as LogprobToken[] | undefined)
: undefined;
if (logprobs && logprobs.length > 0) {
// Calculate average logprob
@@ -117,6 +124,7 @@ export async function transcribeAudio(
logprobs: logprobs,
avgLogprob: avgLogprob,
isLowConfidence: isLowConfidence,
language: (response as { language?: string }).language,
};
} catch (error: any) {
console.error("[STT] Transcription error:", error);
@@ -132,23 +140,6 @@ export async function transcribeAudio(
}
}
}
function getFileExtension(format: string): string {
// Map mime types or format strings to file extensions
const formatLower = format.toLowerCase();
if (formatLower.includes("webm")) return "webm";
if (formatLower.includes("ogg")) return "ogg";
if (formatLower.includes("mp3")) return "mp3";
if (formatLower.includes("wav")) return "wav";
if (formatLower.includes("m4a")) return "m4a";
if (formatLower.includes("mp4")) return "mp4";
if (formatLower.includes("flac")) return "flac";
// Default to webm (common browser format)
return "webm";
}
export function isSTTInitialized(): boolean {
return openaiClient !== null && config !== null;
}

View File

@@ -3,7 +3,7 @@ import express from "express";
import basicAuth from "express-basic-auth";
import { createServer as createHTTPServer } from "http";
import { VoiceAssistantWebSocketServer } from "./websocket-server.js";
import { initializeSTT } from "./agent/stt-openai.js";
import { initializeSTT, type STTConfig } from "./agent/stt-openai.js";
import { initializeTTS } from "./agent/tts-openai.js";
import { listConversations, deleteConversation } from "./persistence.js";
import { AgentManager } from "./agent/agent-manager.js";
@@ -101,9 +101,12 @@ async function main() {
? parseFloat(process.env.STT_CONFIDENCE_THRESHOLD)
: undefined; // Will default to -3.0 in stt-openai.ts
const sttModel = process.env.STT_MODEL as STTConfig["model"];
initializeSTT({
apiKey,
confidenceThreshold: sttConfidenceThreshold
confidenceThreshold: sttConfidenceThreshold,
...(sttModel ? { model: sttModel } : {}),
});
// Initialize TTS

View File

@@ -228,6 +228,7 @@ export const AgentSnapshotPayloadSchema = z.object({
id: z.string(),
provider: AgentProviderSchema,
cwd: z.string(),
model: z.string().nullable(),
createdAt: z.string(),
updatedAt: z.string(),
lastUserMessageAt: z.string().nullable(),
@@ -254,6 +255,7 @@ export function serializeAgentSnapshot(
const { createdAt, updatedAt, lastUserMessageAt, ...rest } = snapshot;
return {
...rest,
model: snapshot.model,
createdAt: createdAt.toISOString(),
updatedAt: updatedAt.toISOString(),
lastUserMessageAt: lastUserMessageAt ? lastUserMessageAt.toISOString() : null,
@@ -334,6 +336,9 @@ export const SendAgentAudioSchema = z.object({
format: z.string(),
isLast: z.boolean(),
requestId: z.string().optional(), // Client-provided ID for tracking transcription
mode: z
.enum(["transcribe_only", "auto_run"])
.optional(),
});
const GitSetupOptionsSchema = z.object({
@@ -531,6 +536,11 @@ export const TranscriptionResultMessageSchema = z.object({
language: z.string().optional(),
duration: z.number().optional(),
requestId: z.string().optional(), // Echoed back from request for tracking
avgLogprob: z.number().optional(),
isLowConfidence: z.boolean().optional(),
byteLength: z.number().optional(),
format: z.string().optional(),
debugRecordingPath: z.string().optional(),
}),
});

View File

@@ -13,6 +13,7 @@ function createSnapshot(overrides?: Partial<AgentSnapshot>): AgentSnapshot {
id: "agent-1",
provider: "claude",
cwd: "/tmp/project",
model: null,
createdAt: now,
updatedAt: now,
lastUserMessageAt: null,

View File

@@ -545,6 +545,7 @@ export class Session {
id: record.id,
provider,
cwd: record.cwd,
model: record.config?.model ?? null,
createdAt: createdAt.toISOString(),
updatedAt: updatedAt.toISOString(),
lastUserMessageAt: lastUserMessageAt ? lastUserMessageAt.toISOString() : null,
@@ -971,7 +972,8 @@ export class Session {
private async handleSendAgentAudio(
msg: Extract<SessionInboundMessage, { type: "send_agent_audio" }>
): Promise<void> {
const { agentId, audio, format, isLast, requestId } = msg;
const { agentId, audio, format, isLast, requestId, mode } = msg;
const shouldAutoRun = mode === "auto_run";
// Decode base64
const audioBuffer = Buffer.from(audio, "base64");
@@ -992,7 +994,11 @@ export class Session {
try {
// Transcribe the audio
const result = await this.sttManager.transcribe(audioBuffer, format);
const result = await this.sttManager.transcribe(audioBuffer, format, {
agentId,
requestId,
label: shouldAutoRun ? "dictation:auto_run" : "dictation",
});
const transcriptText = result.text.trim();
if (!transcriptText) {
@@ -1014,9 +1020,21 @@ export class Session {
language: result.language,
duration: result.duration,
requestId,
avgLogprob: result.avgLogprob,
isLowConfidence: result.isLowConfidence,
byteLength: result.byteLength,
format: result.format,
debugRecordingPath: result.debugRecordingPath,
},
});
if (!shouldAutoRun) {
console.log(
`[Session ${this.clientId}] Completed transcription for agent ${agentId} (requestId: ${requestId ?? "n/a"})`
);
return;
}
try {
await this.interruptAgentIfRunning(agentId);
} catch (error) {
@@ -2225,7 +2243,9 @@ export class Session {
});
try {
const result = await this.sttManager.transcribe(audio, format);
const result = await this.sttManager.transcribe(audio, format, {
label: this.isRealtimeMode ? "realtime" : "buffered",
});
const transcriptText = result.text.trim();
if (!transcriptText) {
@@ -2255,6 +2275,11 @@ export class Session {
text: result.text,
language: result.language,
duration: result.duration,
avgLogprob: result.avgLogprob,
isLowConfidence: result.isLowConfidence,
byteLength: result.byteLength,
format: result.format,
debugRecordingPath: result.debugRecordingPath,
},
});

View File

@@ -119,13 +119,13 @@ export function validateBranchSlug(slug: string): {
return { valid: false, error: "Branch name too long (max 100 characters)" };
}
// Check for valid characters: lowercase letters, numbers, hyphens
const validPattern = /^[a-z0-9-]+$/;
// Check for valid characters: lowercase letters, numbers, hyphens, forward slashes
const validPattern = /^[a-z0-9-/]+$/;
if (!validPattern.test(slug)) {
return {
valid: false,
error:
"Branch name must contain only lowercase letters, numbers, and hyphens",
"Branch name must contain only lowercase letters, numbers, hyphens, and forward slashes",
};
}

View File

@@ -24,5 +24,12 @@
"sourceMap": true
},
"include": ["src/server/**/*", "src/services/**/*"],
"exclude": ["node_modules", "dist"]
"exclude": [
"node_modules",
"dist",
"src/**/*.test.ts",
"src/**/*.test.tsx",
"src/**/*.spec.ts",
"src/**/*.spec.tsx"
]
}