mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Prompt is too long
This commit is contained in:
Mohamed Boudra
316
app/settings.tsx
Normal file
316
app/settings.tsx
Normal file
@@ -0,0 +1,316 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { View, Text, TextInput, Switch, Pressable, ScrollView, Alert, ActivityIndicator } from 'react-native';
|
||||
import { router } from 'expo-router';
|
||||
import { StatusBar } from 'expo-status-bar';
|
||||
import {
|
||||
AppSettings,
|
||||
getSettings,
|
||||
saveSettings,
|
||||
resetSettings,
|
||||
validateServerUrl,
|
||||
} from '../lib/settings-storage';
|
||||
|
||||
export default function SettingsScreen() {
|
||||
const [settings, setSettings] = useState<AppSettings | null>(null);
|
||||
const [serverUrl, setServerUrl] = useState('');
|
||||
const [useSpeaker, setUseSpeaker] = useState(false);
|
||||
const [keepScreenOn, setKeepScreenOn] = useState(true);
|
||||
const [theme, setTheme] = useState<'dark' | 'light' | 'auto'>('dark');
|
||||
const [urlError, setUrlError] = useState<string | null>(null);
|
||||
const [isSaving, setIsSaving] = useState(false);
|
||||
const [isTesting, setIsTesting] = useState(false);
|
||||
const [testResult, setTestResult] = useState<{ success: boolean; message: string } | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
loadSettings();
|
||||
}, []);
|
||||
|
||||
async function loadSettings() {
|
||||
const loaded = await getSettings();
|
||||
setSettings(loaded);
|
||||
setServerUrl(loaded.serverUrl);
|
||||
setUseSpeaker(loaded.useSpeaker);
|
||||
setKeepScreenOn(loaded.keepScreenOn);
|
||||
setTheme(loaded.theme);
|
||||
}
|
||||
|
||||
function validateUrl(url: string) {
|
||||
const validation = validateServerUrl(url);
|
||||
setUrlError(validation.valid ? null : validation.error || null);
|
||||
return validation.valid;
|
||||
}
|
||||
|
||||
async function handleSaveSettings() {
|
||||
if (!validateUrl(serverUrl)) {
|
||||
return;
|
||||
}
|
||||
|
||||
setIsSaving(true);
|
||||
try {
|
||||
const newSettings: AppSettings = {
|
||||
serverUrl,
|
||||
useSpeaker,
|
||||
keepScreenOn,
|
||||
theme,
|
||||
};
|
||||
await saveSettings(newSettings);
|
||||
setSettings(newSettings);
|
||||
Alert.alert('Success', 'Settings saved successfully');
|
||||
} catch (error) {
|
||||
Alert.alert('Error', 'Failed to save settings');
|
||||
console.error('Save settings error:', error);
|
||||
} finally {
|
||||
setIsSaving(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleResetSettings() {
|
||||
Alert.alert(
|
||||
'Reset Settings',
|
||||
'Are you sure you want to reset all settings to defaults?',
|
||||
[
|
||||
{ text: 'Cancel', style: 'cancel' },
|
||||
{
|
||||
text: 'Reset',
|
||||
style: 'destructive',
|
||||
onPress: async () => {
|
||||
const defaults = await resetSettings();
|
||||
setServerUrl(defaults.serverUrl);
|
||||
setUseSpeaker(defaults.useSpeaker);
|
||||
setKeepScreenOn(defaults.keepScreenOn);
|
||||
setTheme(defaults.theme);
|
||||
setSettings(defaults);
|
||||
Alert.alert('Success', 'Settings reset to defaults');
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
async function handleTestConnection() {
|
||||
if (!validateUrl(serverUrl)) {
|
||||
return;
|
||||
}
|
||||
|
||||
setIsTesting(true);
|
||||
setTestResult(null);
|
||||
|
||||
try {
|
||||
const ws = new WebSocket(serverUrl);
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
ws.close();
|
||||
setTestResult({
|
||||
success: false,
|
||||
message: 'Connection timeout - server did not respond',
|
||||
});
|
||||
setIsTesting(false);
|
||||
}, 5000);
|
||||
|
||||
ws.onopen = () => {
|
||||
clearTimeout(timeout);
|
||||
ws.close();
|
||||
setTestResult({
|
||||
success: true,
|
||||
message: 'Connection successful',
|
||||
});
|
||||
setIsTesting(false);
|
||||
};
|
||||
|
||||
ws.onerror = () => {
|
||||
clearTimeout(timeout);
|
||||
setTestResult({
|
||||
success: false,
|
||||
message: 'Connection failed - check URL and network',
|
||||
});
|
||||
setIsTesting(false);
|
||||
};
|
||||
} catch (error) {
|
||||
setTestResult({
|
||||
success: false,
|
||||
message: 'Failed to create connection',
|
||||
});
|
||||
setIsTesting(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (!settings) {
|
||||
return (
|
||||
<View className="flex-1 bg-black items-center justify-center">
|
||||
<ActivityIndicator size="large" color="#3b82f6" />
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<View className="flex-1 bg-black">
|
||||
<StatusBar style="light" />
|
||||
|
||||
{/* Header */}
|
||||
<View className="pt-14 pb-4 px-6 border-b border-zinc-800">
|
||||
<View className="flex-row items-center justify-between">
|
||||
<Text className="text-white text-2xl font-bold">Settings</Text>
|
||||
<Pressable
|
||||
onPress={() => router.back()}
|
||||
className="px-4 py-2 rounded-lg bg-zinc-800 active:bg-zinc-700"
|
||||
>
|
||||
<Text className="text-white font-medium">Done</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
<ScrollView className="flex-1 px-6 py-6">
|
||||
{/* Server Configuration */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-zinc-400 text-sm font-semibold uppercase mb-3">
|
||||
Server Configuration
|
||||
</Text>
|
||||
|
||||
<View className="bg-zinc-900 rounded-lg p-4 border border-zinc-800">
|
||||
<Text className="text-white mb-2 font-medium">WebSocket URL</Text>
|
||||
<TextInput
|
||||
className="bg-zinc-800 text-white px-4 py-3 rounded-lg border border-zinc-700"
|
||||
value={serverUrl}
|
||||
onChangeText={(text) => {
|
||||
setServerUrl(text);
|
||||
validateUrl(text);
|
||||
setTestResult(null);
|
||||
}}
|
||||
placeholder="wss://your-server.com/ws"
|
||||
placeholderTextColor="#71717a"
|
||||
autoCapitalize="none"
|
||||
autoCorrect={false}
|
||||
keyboardType="url"
|
||||
/>
|
||||
{urlError && (
|
||||
<Text className="text-red-500 text-sm mt-2">{urlError}</Text>
|
||||
)}
|
||||
|
||||
{testResult && (
|
||||
<View
|
||||
className={`mt-3 p-3 rounded-lg ${
|
||||
testResult.success ? 'bg-green-900/30 border border-green-700' : 'bg-red-900/30 border border-red-700'
|
||||
}`}
|
||||
>
|
||||
<Text className={testResult.success ? 'text-green-400' : 'text-red-400'}>
|
||||
{testResult.message}
|
||||
</Text>
|
||||
</View>
|
||||
)}
|
||||
|
||||
<Pressable
|
||||
onPress={handleTestConnection}
|
||||
disabled={isTesting || !!urlError}
|
||||
className={`mt-3 py-3 rounded-lg flex-row items-center justify-center ${
|
||||
isTesting || urlError ? 'bg-zinc-700' : 'bg-blue-600 active:bg-blue-700'
|
||||
}`}
|
||||
>
|
||||
{isTesting ? (
|
||||
<>
|
||||
<ActivityIndicator size="small" color="#fff" />
|
||||
<Text className="text-white font-semibold ml-2">Testing...</Text>
|
||||
</>
|
||||
) : (
|
||||
<Text className="text-white font-semibold">Test Connection</Text>
|
||||
)}
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Audio Settings */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-zinc-400 text-sm font-semibold uppercase mb-3">
|
||||
Audio Settings
|
||||
</Text>
|
||||
|
||||
<View className="bg-zinc-900 rounded-lg border border-zinc-800">
|
||||
<View className="flex-row items-center justify-between p-4 border-b border-zinc-800">
|
||||
<View className="flex-1">
|
||||
<Text className="text-white font-medium">Use Speaker</Text>
|
||||
<Text className="text-zinc-500 text-sm mt-1">
|
||||
Use phone speaker instead of earpiece
|
||||
</Text>
|
||||
</View>
|
||||
<Switch
|
||||
value={useSpeaker}
|
||||
onValueChange={setUseSpeaker}
|
||||
trackColor={{ false: '#3f3f46', true: '#3b82f6' }}
|
||||
thumbColor="#f4f4f5"
|
||||
/>
|
||||
</View>
|
||||
|
||||
<View className="flex-row items-center justify-between p-4">
|
||||
<View className="flex-1">
|
||||
<Text className="text-white font-medium">Keep Screen On</Text>
|
||||
<Text className="text-zinc-500 text-sm mt-1">
|
||||
Prevent screen from sleeping during use
|
||||
</Text>
|
||||
</View>
|
||||
<Switch
|
||||
value={keepScreenOn}
|
||||
onValueChange={setKeepScreenOn}
|
||||
trackColor={{ false: '#3f3f46', true: '#3b82f6' }}
|
||||
thumbColor="#f4f4f5"
|
||||
/>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* App Information */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-zinc-400 text-sm font-semibold uppercase mb-3">
|
||||
App Information
|
||||
</Text>
|
||||
|
||||
<View className="bg-zinc-900 rounded-lg p-4 border border-zinc-800">
|
||||
<View className="flex-row justify-between mb-3">
|
||||
<Text className="text-zinc-400">App Version</Text>
|
||||
<Text className="text-white">1.0.0</Text>
|
||||
</View>
|
||||
<View className="flex-row justify-between mb-3">
|
||||
<Text className="text-zinc-400">Audio Format</Text>
|
||||
<Text className="text-white">PCM 16kHz Mono</Text>
|
||||
</View>
|
||||
<View className="flex-row justify-between">
|
||||
<Text className="text-zinc-400">Current Server</Text>
|
||||
<Text className="text-white text-xs" numberOfLines={1} ellipsizeMode="middle">
|
||||
{settings.serverUrl.replace('wss://', '').replace('ws://', '')}
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Actions */}
|
||||
<View className="mb-8">
|
||||
<Pressable
|
||||
onPress={handleSaveSettings}
|
||||
disabled={isSaving || !!urlError}
|
||||
className={`py-4 rounded-lg mb-3 ${
|
||||
isSaving || urlError ? 'bg-zinc-700' : 'bg-blue-600 active:bg-blue-700'
|
||||
}`}
|
||||
>
|
||||
{isSaving ? (
|
||||
<ActivityIndicator size="small" color="#fff" />
|
||||
) : (
|
||||
<Text className="text-white font-bold text-center text-lg">
|
||||
Save Settings
|
||||
</Text>
|
||||
)}
|
||||
</Pressable>
|
||||
|
||||
<Pressable
|
||||
onPress={handleResetSettings}
|
||||
className="py-4 rounded-lg border border-zinc-700 bg-zinc-900 active:bg-zinc-800"
|
||||
>
|
||||
<Text className="text-zinc-300 font-semibold text-center">
|
||||
Reset to Defaults
|
||||
</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
|
||||
{/* Bottom spacing */}
|
||||
<View className="h-8" />
|
||||
</ScrollView>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
@@ -30,12 +30,23 @@ export const LoadConversationRequestMessageSchema = z.object({
|
||||
conversationId: z.string(),
|
||||
});
|
||||
|
||||
export const ListConversationsRequestMessageSchema = z.object({
|
||||
type: z.literal("list_conversations_request"),
|
||||
});
|
||||
|
||||
export const DeleteConversationRequestMessageSchema = z.object({
|
||||
type: z.literal("delete_conversation_request"),
|
||||
conversationId: z.string(),
|
||||
});
|
||||
|
||||
export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
|
||||
UserTextMessageSchema,
|
||||
AudioChunkMessageSchema,
|
||||
AbortRequestMessageSchema,
|
||||
AudioPlayedMessageSchema,
|
||||
LoadConversationRequestMessageSchema,
|
||||
ListConversationsRequestMessageSchema,
|
||||
DeleteConversationRequestMessageSchema,
|
||||
]);
|
||||
|
||||
export type SessionInboundMessage = z.infer<typeof SessionInboundMessageSchema>;
|
||||
@@ -189,6 +200,28 @@ export const SessionStateMessageSchema = z.object({
|
||||
}),
|
||||
});
|
||||
|
||||
export const ListConversationsResponseMessageSchema = z.object({
|
||||
type: z.literal("list_conversations_response"),
|
||||
payload: z.object({
|
||||
conversations: z.array(
|
||||
z.object({
|
||||
id: z.string(),
|
||||
lastUpdated: z.string(),
|
||||
messageCount: z.number(),
|
||||
})
|
||||
),
|
||||
}),
|
||||
});
|
||||
|
||||
export const DeleteConversationResponseMessageSchema = z.object({
|
||||
type: z.literal("delete_conversation_response"),
|
||||
payload: z.object({
|
||||
conversationId: z.string(),
|
||||
success: z.boolean(),
|
||||
error: z.string().optional(),
|
||||
}),
|
||||
});
|
||||
|
||||
export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
||||
ActivityLogMessageSchema,
|
||||
AssistantChunkMessageSchema,
|
||||
@@ -201,6 +234,8 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
|
||||
AgentUpdateMessageSchema,
|
||||
AgentStatusMessageSchema,
|
||||
SessionStateMessageSchema,
|
||||
ListConversationsResponseMessageSchema,
|
||||
DeleteConversationResponseMessageSchema,
|
||||
]);
|
||||
|
||||
export type SessionOutboundMessage = z.infer<
|
||||
@@ -219,6 +254,8 @@ export type AgentCreatedMessage = z.infer<typeof AgentCreatedMessageSchema>;
|
||||
export type AgentUpdateMessage = z.infer<typeof AgentUpdateMessageSchema>;
|
||||
export type AgentStatusMessage = z.infer<typeof AgentStatusMessageSchema>;
|
||||
export type SessionStateMessage = z.infer<typeof SessionStateMessageSchema>;
|
||||
export type ListConversationsResponseMessage = z.infer<typeof ListConversationsResponseMessageSchema>;
|
||||
export type DeleteConversationResponseMessage = z.infer<typeof DeleteConversationResponseMessageSchema>;
|
||||
|
||||
// Type exports for payload types
|
||||
export type ActivityLogPayload = z.infer<typeof ActivityLogPayloadSchema>;
|
||||
|
||||
@@ -15,7 +15,7 @@ import { getSystemPrompt } from "./agent/system-prompt.js";
|
||||
import { getAllTools } from "./agent/llm-openai.js";
|
||||
import { TTSManager } from "./agent/tts-manager.js";
|
||||
import { STTManager } from "./agent/stt-manager.js";
|
||||
import { saveConversation } from "./persistence.js";
|
||||
import { saveConversation, listConversations, deleteConversation } from "./persistence.js";
|
||||
import { experimental_createMCPClient } from "ai";
|
||||
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
||||
import { createTerminalMcpServer } from "./terminal-mcp/index.js";
|
||||
@@ -257,6 +257,14 @@ export class Session {
|
||||
case "load_conversation_request":
|
||||
await this.handleLoadConversation();
|
||||
break;
|
||||
|
||||
case "list_conversations_request":
|
||||
await this.handleListConversations();
|
||||
break;
|
||||
|
||||
case "delete_conversation_request":
|
||||
await this.handleDeleteConversation(msg.conversationId);
|
||||
break;
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error(
|
||||
@@ -292,6 +300,71 @@ export class Session {
|
||||
await this.sendSessionState();
|
||||
}
|
||||
|
||||
/**
|
||||
* List all conversations
|
||||
*/
|
||||
public async handleListConversations(): Promise<void> {
|
||||
try {
|
||||
const conversations = await listConversations();
|
||||
this.emit({
|
||||
type: "list_conversations_response",
|
||||
payload: {
|
||||
conversations: conversations.map(conv => ({
|
||||
id: conv.id,
|
||||
lastUpdated: conv.lastUpdated.toISOString(),
|
||||
messageCount: conv.messageCount,
|
||||
})),
|
||||
},
|
||||
});
|
||||
} catch (error: any) {
|
||||
console.error(
|
||||
`[Session ${this.clientId}] Failed to list conversations:`,
|
||||
error
|
||||
);
|
||||
this.emit({
|
||||
type: "activity_log",
|
||||
payload: {
|
||||
id: uuidv4(),
|
||||
timestamp: new Date(),
|
||||
type: "error",
|
||||
content: `Failed to list conversations: ${error.message}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a conversation
|
||||
*/
|
||||
public async handleDeleteConversation(conversationId: string): Promise<void> {
|
||||
try {
|
||||
await deleteConversation(conversationId);
|
||||
this.emit({
|
||||
type: "delete_conversation_response",
|
||||
payload: {
|
||||
conversationId,
|
||||
success: true,
|
||||
},
|
||||
});
|
||||
console.log(
|
||||
`[Session ${this.clientId}] Deleted conversation ${conversationId}`
|
||||
);
|
||||
} catch (error: any) {
|
||||
console.error(
|
||||
`[Session ${this.clientId}] Failed to delete conversation ${conversationId}:`,
|
||||
error
|
||||
);
|
||||
this.emit({
|
||||
type: "delete_conversation_response",
|
||||
payload: {
|
||||
conversationId,
|
||||
success: false,
|
||||
error: error.message,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send current session state (live agents and commands) to client
|
||||
*/
|
||||
|
||||
@@ -19,7 +19,7 @@ export function useWebSocket(url: string, conversationId?: string | null): UseWe
|
||||
const [currentConversationId, setCurrentConversationId] = useState<string | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const handlersRef = useRef<Map<SessionOutboundMessage['type'], Set<(message: SessionOutboundMessage) => void>>>(new Map());
|
||||
const reconnectTimeoutRef = useRef<number>();
|
||||
const reconnectTimeoutRef = useRef<number | undefined>(undefined);
|
||||
|
||||
const connect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
|
||||
120
packages/voice-mobile/EAS_SETUP.md
Normal file
120
packages/voice-mobile/EAS_SETUP.md
Normal file
@@ -0,0 +1,120 @@
|
||||
# EAS Update Setup Guide
|
||||
|
||||
## Current Status
|
||||
|
||||
✅ **Completed:**
|
||||
- Installed `eas-cli` as dev dependency
|
||||
- Created `eas.json` with update channels (production, preview, development)
|
||||
- Configured `app.json` with runtime version policy
|
||||
|
||||
⏳ **Manual Steps Required:**
|
||||
|
||||
### 1. Initialize EAS Project
|
||||
|
||||
Run the following command and follow the prompts:
|
||||
|
||||
```bash
|
||||
npx eas init
|
||||
```
|
||||
|
||||
This will:
|
||||
- Prompt you to log in to your Expo account (create one if needed at expo.dev)
|
||||
- Create a new project in your Expo account
|
||||
- Add a `projectId` to your `app.json` under `expo.extra.eas.projectId`
|
||||
|
||||
### 2. Update the Updates URL
|
||||
|
||||
After running `eas init`, you'll receive a project ID. Update the `updates.url` in `app.json`:
|
||||
|
||||
```json
|
||||
"updates": {
|
||||
"url": "https://u.expo.dev/YOUR_PROJECT_ID_HERE"
|
||||
}
|
||||
```
|
||||
|
||||
Replace `YOUR_PROJECT_ID` with the actual project ID from the previous step.
|
||||
|
||||
### 3. Build a New Development Build
|
||||
|
||||
Since you're using a custom development client (`expo-dev-client`), you need to create a new build with EAS Update configured:
|
||||
|
||||
```bash
|
||||
# For Android
|
||||
npx eas build --profile development --platform android
|
||||
|
||||
# For iOS (requires Apple Developer account)
|
||||
npx eas build --profile development --platform ios
|
||||
```
|
||||
|
||||
Install this new build on your device. Your existing development build won't receive updates.
|
||||
|
||||
## Publishing Updates
|
||||
|
||||
Once setup is complete, publish updates on the go:
|
||||
|
||||
### Production Update
|
||||
|
||||
```bash
|
||||
npx eas update --branch production --message "Fix critical bug"
|
||||
```
|
||||
|
||||
Users on the production channel will receive this update.
|
||||
|
||||
### Preview Update (for testing)
|
||||
|
||||
```bash
|
||||
npx eas update --branch preview --message "Test new feature"
|
||||
```
|
||||
|
||||
### Development Update
|
||||
|
||||
```bash
|
||||
npx eas update --branch development --message "WIP changes"
|
||||
```
|
||||
|
||||
## Update Channels Explained
|
||||
|
||||
- **production**: For released apps in app stores
|
||||
- **preview**: For internal testing before production
|
||||
- **development**: For active development testing
|
||||
|
||||
Configure which channel your app uses in `eas.json` or at build time.
|
||||
|
||||
## Runtime Version Policy
|
||||
|
||||
We're using `"policy": "appVersion"` which means:
|
||||
- Updates only work within the same app version (1.0.0)
|
||||
- If you change native code or dependencies, increment the version in `app.json`
|
||||
- Users need a new build for native changes, but JS/React changes update via EAS
|
||||
|
||||
## Quick Reference
|
||||
|
||||
```bash
|
||||
# Check update status
|
||||
npx eas update:list --branch production
|
||||
|
||||
# Roll back to previous update
|
||||
npx eas update:rollback --branch production
|
||||
|
||||
# View update details
|
||||
npx eas update:view [update-id]
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**Update not appearing?**
|
||||
- Ensure device is on the same channel as the update
|
||||
- Force close and reopen the app
|
||||
- Check the update was published: `npx eas update:list`
|
||||
|
||||
**"Runtime version mismatch" error?**
|
||||
- Your app version doesn't match the update
|
||||
- Rebuild the app with `eas build`
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Run `npx eas init` now
|
||||
2. Update the `updates.url` in app.json with your project ID
|
||||
3. Create a new development build with `npx eas build --profile development --platform android`
|
||||
4. Install the new build on your device
|
||||
5. Test publishing an update!
|
||||
427
packages/voice-mobile/PHASE_2_AUDIO_IMPLEMENTATION.md
Normal file
427
packages/voice-mobile/PHASE_2_AUDIO_IMPLEMENTATION.md
Normal file
@@ -0,0 +1,427 @@
|
||||
# Phase 2: Audio Capture & Playback Implementation
|
||||
|
||||
## Overview
|
||||
|
||||
This document describes the implementation of audio recording and playback functionality for the voice-mobile React Native app using expo-audio.
|
||||
|
||||
## Dependencies Installed
|
||||
|
||||
### expo-audio (~1.0.13)
|
||||
- Cross-platform audio recording and playback library
|
||||
- Provides hooks: `useAudioRecorder` and `useAudioPlayer`
|
||||
- Supports iOS, Android, and Web platforms
|
||||
- Auto-configured via config plugin in app.json
|
||||
|
||||
### expo-file-system (~19.0.17)
|
||||
- File system operations for reading/writing audio files
|
||||
- New API using `File` and `Paths` classes
|
||||
- Required for converting between audio URIs and Blobs
|
||||
|
||||
## Permissions Configuration
|
||||
|
||||
### app.json Updates
|
||||
|
||||
**iOS**:
|
||||
```json
|
||||
{
|
||||
"ios": {
|
||||
"infoPlist": {
|
||||
"NSMicrophoneUsageDescription": "This app needs access to the microphone for voice commands."
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Android**:
|
||||
```json
|
||||
{
|
||||
"android": {
|
||||
"permissions": [
|
||||
"RECORD_AUDIO"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Plugins**:
|
||||
```json
|
||||
{
|
||||
"plugins": [
|
||||
"expo-audio"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
packages/voice-mobile/
|
||||
├── hooks/
|
||||
│ ├── use-audio-recorder.ts # Audio recording hook (NEW)
|
||||
│ └── use-audio-player.ts # Audio playback hook (NEW)
|
||||
├── lib/
|
||||
│ ├── audio-capture.ts # Legacy/factory version (NOT USED)
|
||||
│ └── audio-playback.ts # Legacy/factory version (NOT USED)
|
||||
├── app/
|
||||
│ ├── index.tsx # Updated with link to audio test
|
||||
│ └── audio-test.tsx # Audio test screen (NEW)
|
||||
└── app.json # Updated with permissions
|
||||
```
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Audio Recording (`use-audio-recorder.ts`)
|
||||
|
||||
**Key Features**:
|
||||
- Matches web app audio constraints:
|
||||
- Sample rate: 16000 Hz (optimal for Whisper STT)
|
||||
- Channels: 1 (mono)
|
||||
- Format: M4A/AAC on native, WebM/Opus on web
|
||||
- Audio source: `voice_communication` (enables echo cancellation, noise suppression, auto gain control on Android)
|
||||
|
||||
**Platform-Specific Configuration**:
|
||||
|
||||
Android:
|
||||
```typescript
|
||||
android: {
|
||||
extension: '.m4a',
|
||||
outputFormat: 'mpeg4',
|
||||
audioEncoder: 'aac',
|
||||
sampleRate: 16000,
|
||||
audioSource: 'voice_communication', // Key for voice optimization
|
||||
}
|
||||
```
|
||||
|
||||
iOS:
|
||||
```typescript
|
||||
ios: {
|
||||
extension: '.m4a',
|
||||
audioQuality: 127, // High quality
|
||||
sampleRate: 16000,
|
||||
linearPCMBitDepth: 16,
|
||||
linearPCMIsBigEndian: false,
|
||||
linearPCMIsFloat: false,
|
||||
}
|
||||
```
|
||||
|
||||
Web:
|
||||
```typescript
|
||||
web: {
|
||||
mimeType: 'audio/webm;codecs=opus',
|
||||
bitsPerSecond: 128000,
|
||||
}
|
||||
```
|
||||
|
||||
**Usage**:
|
||||
```typescript
|
||||
const audioRecorder = useAudioRecorder({
|
||||
sampleRate: 16000,
|
||||
numberOfChannels: 1,
|
||||
});
|
||||
|
||||
// Start recording
|
||||
await audioRecorder.start();
|
||||
|
||||
// Stop and get Blob for WebSocket transmission
|
||||
const audioBlob = await audioRecorder.stop();
|
||||
```
|
||||
|
||||
**File Handling**:
|
||||
1. Recording creates temporary file (managed by expo-audio)
|
||||
2. On stop, reads file URI from `audioRecorder.uri`
|
||||
3. Converts to Blob using expo-file-system's `File` class
|
||||
4. Cleans up temporary file after conversion
|
||||
|
||||
### Audio Playback (`use-audio-player.ts`)
|
||||
|
||||
**Key Features**:
|
||||
- Queue system matching web version
|
||||
- Supports playing Blobs from server (base64-encoded MP3)
|
||||
- Automatic cleanup of temporary files
|
||||
- Play/pause/resume/stop controls
|
||||
|
||||
**Queue System**:
|
||||
```typescript
|
||||
interface QueuedAudio {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
```
|
||||
|
||||
**Usage**:
|
||||
```typescript
|
||||
const audioPlayer = useAudioPlayer();
|
||||
|
||||
// Play audio (queued automatically)
|
||||
const duration = await audioPlayer.play(audioBlob);
|
||||
|
||||
// Controls
|
||||
audioPlayer.pause();
|
||||
audioPlayer.resume();
|
||||
audioPlayer.stop();
|
||||
audioPlayer.clearQueue();
|
||||
```
|
||||
|
||||
**File Handling**:
|
||||
1. Converts Blob to ArrayBuffer
|
||||
2. Creates temporary file in cache directory
|
||||
3. Writes ArrayBuffer to file
|
||||
4. Plays from file URI
|
||||
5. Cleans up after playback
|
||||
|
||||
### Audio Test Screen (`audio-test.tsx`)
|
||||
|
||||
**Features**:
|
||||
- Permission request/check on mount
|
||||
- Visual permission status indicator
|
||||
- Recording controls (Start/Stop)
|
||||
- Recording status display
|
||||
- Last recording size/type display
|
||||
- Playback controls (Play Last Recording, Stop)
|
||||
- Audio configuration info display
|
||||
- Styled with NativeWind (Tailwind CSS)
|
||||
|
||||
**Navigation**:
|
||||
- Added link button on main screen (index.tsx)
|
||||
- Route: `/audio-test`
|
||||
|
||||
## Key Differences from Web Implementation
|
||||
|
||||
### 1. API Differences
|
||||
|
||||
**Web (MediaRecorder)**:
|
||||
- `new MediaRecorder(stream, options)`
|
||||
- Events: `ondataavailable`, `onstop`
|
||||
- Returns Blob directly
|
||||
|
||||
**React Native (expo-audio)**:
|
||||
- Hook-based: `useAudioRecorder(options)`
|
||||
- Returns Promise from `stop()`
|
||||
- URI property for file location
|
||||
- Requires manual Blob conversion
|
||||
|
||||
### 2. File System
|
||||
|
||||
**Web**:
|
||||
- Direct Blob creation from chunks
|
||||
- No file system access needed
|
||||
|
||||
**React Native**:
|
||||
- Must use file system (expo-file-system)
|
||||
- Temporary files in cache directory
|
||||
- Manual cleanup required
|
||||
|
||||
### 3. Permissions
|
||||
|
||||
**Web**:
|
||||
- Runtime prompt via `getUserMedia()`
|
||||
- Browser-managed
|
||||
|
||||
**React Native**:
|
||||
- Platform-specific: Info.plist (iOS), AndroidManifest.xml (Android)
|
||||
- Requires explicit `requestRecordingPermissionsAsync()`
|
||||
- Must be configured in app.json
|
||||
|
||||
### 4. Audio Processing
|
||||
|
||||
**Web**:
|
||||
- WebRTC constraints: `echoCancellation`, `noiseSuppression`, `autoGainControl`
|
||||
|
||||
**React Native**:
|
||||
- Android: `audioSource: 'voice_communication'` enables similar features
|
||||
- iOS: Built-in audio quality settings
|
||||
- Platform-specific implementations
|
||||
|
||||
## Platform-Specific Considerations
|
||||
|
||||
### iOS
|
||||
- M4A/AAC format (better compatibility)
|
||||
- Microphone permission required (Info.plist)
|
||||
- Audio session management handled by expo-audio
|
||||
- No manual echo cancellation needed (handled by system)
|
||||
|
||||
### Android
|
||||
- M4A/AAC format
|
||||
- RECORD_AUDIO permission required
|
||||
- `voice_communication` audio source crucial for voice quality
|
||||
- Enables: echo cancellation, noise suppression, auto gain control
|
||||
- Works well with VoIP and voice commands
|
||||
|
||||
### Web
|
||||
- WebM/Opus format (better compression)
|
||||
- MediaRecorder API used under the hood
|
||||
- Permissions via browser prompt
|
||||
- WebRTC audio processing
|
||||
|
||||
## Testing
|
||||
|
||||
### Manual Testing Steps
|
||||
|
||||
1. **Start Expo Development Server**:
|
||||
```bash
|
||||
cd packages/voice-mobile
|
||||
npm start
|
||||
```
|
||||
|
||||
2. **Test on Web**:
|
||||
- Press 'w' in terminal or visit http://localhost:8081
|
||||
- Click "Test Audio Capture & Playback" button
|
||||
- Grant microphone permission
|
||||
- Test recording and playback
|
||||
|
||||
3. **Test on iOS Simulator**:
|
||||
- Press 'i' in terminal
|
||||
- Note: iOS Simulator doesn't have microphone access
|
||||
- Test on physical device for full functionality
|
||||
|
||||
4. **Test on Android Emulator/Device**:
|
||||
- Press 'a' in terminal
|
||||
- Grant microphone permission
|
||||
- Test recording and playback
|
||||
|
||||
### Expected Behavior
|
||||
|
||||
**Recording**:
|
||||
- Status shows "Recording..." with red indicator
|
||||
- Stop button becomes enabled
|
||||
- After stop, shows recording size and type
|
||||
|
||||
**Playback**:
|
||||
- "Play Last Recording" enabled after recording
|
||||
- Plays back the recorded audio
|
||||
- Shows duration after playback completes
|
||||
|
||||
**Permissions**:
|
||||
- Green indicator when granted
|
||||
- Red indicator when denied
|
||||
- Automatic request on mount
|
||||
|
||||
## Integration with WebSocket
|
||||
|
||||
The audio hooks are designed to work seamlessly with the WebSocket client:
|
||||
|
||||
```typescript
|
||||
// In your component with WebSocket
|
||||
const audioRecorder = useAudioRecorder();
|
||||
const { send } = useWebSocket(WS_URL);
|
||||
|
||||
// Record and send to server
|
||||
async function handleRecord() {
|
||||
await audioRecorder.start();
|
||||
// ... wait for user to stop
|
||||
const audioBlob = await audioRecorder.stop();
|
||||
|
||||
// Convert to base64 for WebSocket
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
const base64 = reader.result.split(',')[1];
|
||||
send({
|
||||
type: 'audio_chunk',
|
||||
payload: {
|
||||
audio: base64,
|
||||
format: 'audio/m4a',
|
||||
isLast: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
reader.readAsDataURL(audioBlob);
|
||||
}
|
||||
|
||||
// Play audio from server
|
||||
const audioPlayer = useAudioPlayer();
|
||||
|
||||
// When receiving audio from server
|
||||
function handleAudioOutput(message) {
|
||||
const audioBlob = base64ToBlob(message.payload.audio, 'audio/mp3');
|
||||
audioPlayer.play(audioBlob);
|
||||
}
|
||||
```
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
1. **Streaming Audio**:
|
||||
- Implement chunk-based recording for real-time streaming
|
||||
- Send audio chunks as they're recorded (not just on stop)
|
||||
- Requires updating expo-audio usage
|
||||
|
||||
2. **Audio Visualization**:
|
||||
- Add waveform display during recording
|
||||
- Use expo-audio's metering feature
|
||||
- Display audio levels in real-time
|
||||
|
||||
3. **Error Handling**:
|
||||
- Add retry logic for failed recordings
|
||||
- Better error messages for users
|
||||
- Fallback for unsupported formats
|
||||
|
||||
4. **Performance**:
|
||||
- Optimize Blob conversion
|
||||
- Reduce memory usage for long recordings
|
||||
- Implement audio compression if needed
|
||||
|
||||
## Common Issues & Solutions
|
||||
|
||||
### Issue: "No recording URI returned"
|
||||
**Cause**: Recording stopped before it started or expo-audio initialization failed
|
||||
**Solution**: Ensure `audioRecorder.record()` completes before stopping
|
||||
|
||||
### Issue: Playback fails with "Failed to load audio"
|
||||
**Cause**: File was deleted before playback or invalid Blob
|
||||
**Solution**: Ensure Blob is valid and file exists before playing
|
||||
|
||||
### Issue: Permission denied
|
||||
**Cause**: User denied microphone permission or app.json not configured
|
||||
**Solution**:
|
||||
1. Check app.json has correct permission entries
|
||||
2. Rebuild app after changing permissions
|
||||
3. Check device settings
|
||||
|
||||
### Issue: Poor audio quality
|
||||
**Cause**: Incorrect audio source or low sample rate
|
||||
**Solution**: Ensure `audioSource: 'voice_communication'` on Android
|
||||
|
||||
## TypeScript Support
|
||||
|
||||
All audio hooks are fully typed:
|
||||
|
||||
```typescript
|
||||
// Hook return types
|
||||
interface AudioRecorder {
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<Blob>;
|
||||
isRecording(): boolean;
|
||||
getSupportedMimeType(): string | null;
|
||||
}
|
||||
|
||||
interface AudioPlayer {
|
||||
play(audioData: Blob): Promise<number>;
|
||||
stop(): void;
|
||||
pause(): void;
|
||||
resume(): void;
|
||||
isPlaying(): boolean;
|
||||
isPaused(): boolean;
|
||||
clearQueue(): void;
|
||||
}
|
||||
|
||||
// Configuration
|
||||
interface AudioCaptureConfig {
|
||||
sampleRate?: number;
|
||||
numberOfChannels?: number;
|
||||
bitRate?: number;
|
||||
}
|
||||
```
|
||||
|
||||
## Summary
|
||||
|
||||
Phase 2 successfully implements audio capture and playback for the voice-mobile app using expo-audio. The implementation:
|
||||
|
||||
✅ Matches web app audio constraints (16kHz, mono, optimized for speech)
|
||||
✅ Supports all platforms (iOS, Android, Web)
|
||||
✅ Provides Blob output compatible with WebSocket transmission
|
||||
✅ Includes test screen for verification
|
||||
✅ Fully typed with TypeScript
|
||||
✅ Follows React Native best practices
|
||||
✅ Properly manages permissions and temporary files
|
||||
|
||||
The implementation is ready for integration with Phase 1 (WebSocket) to create the complete voice assistant mobile experience.
|
||||
@@ -8,8 +8,17 @@
|
||||
"scheme": "voicemobile",
|
||||
"userInterfaceStyle": "automatic",
|
||||
"newArchEnabled": true,
|
||||
"runtimeVersion": {
|
||||
"policy": "appVersion"
|
||||
},
|
||||
"updates": {
|
||||
"url": "https://u.expo.dev/0e7f65ce-0367-46c8-a238-2b65963d235a"
|
||||
},
|
||||
"ios": {
|
||||
"supportsTablet": true
|
||||
"supportsTablet": true,
|
||||
"infoPlist": {
|
||||
"NSMicrophoneUsageDescription": "This app needs access to the microphone for voice commands."
|
||||
}
|
||||
},
|
||||
"android": {
|
||||
"adaptiveIcon": {
|
||||
@@ -19,7 +28,14 @@
|
||||
"monochromeImage": "./assets/images/android-icon-monochrome.png"
|
||||
},
|
||||
"edgeToEdgeEnabled": true,
|
||||
"predictiveBackGestureEnabled": false
|
||||
"predictiveBackGestureEnabled": false,
|
||||
"softwareKeyboardLayoutMode": "pan",
|
||||
"permissions": [
|
||||
"RECORD_AUDIO",
|
||||
"android.permission.RECORD_AUDIO",
|
||||
"android.permission.MODIFY_AUDIO_SETTINGS"
|
||||
],
|
||||
"package": "com.moboudra.voicemobile"
|
||||
},
|
||||
"web": {
|
||||
"output": "static",
|
||||
@@ -38,11 +54,19 @@
|
||||
"backgroundColor": "#000000"
|
||||
}
|
||||
}
|
||||
]
|
||||
],
|
||||
"expo-audio"
|
||||
],
|
||||
"experiments": {
|
||||
"typedRoutes": true,
|
||||
"reactCompiler": false
|
||||
}
|
||||
},
|
||||
"extra": {
|
||||
"router": {},
|
||||
"eas": {
|
||||
"projectId": "0e7f65ce-0367-46c8-a238-2b65963d235a"
|
||||
}
|
||||
},
|
||||
"owner": "moboudra"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
import { Tabs } from 'expo-router';
|
||||
import React from 'react';
|
||||
|
||||
import { HapticTab } from '@/components/haptic-tab';
|
||||
import { IconSymbol } from '@/components/ui/icon-symbol';
|
||||
import { Colors } from '@/constants/theme';
|
||||
import { useColorScheme } from '@/hooks/use-color-scheme';
|
||||
|
||||
export default function TabLayout() {
|
||||
const colorScheme = useColorScheme();
|
||||
|
||||
return (
|
||||
<Tabs
|
||||
screenOptions={{
|
||||
tabBarActiveTintColor: Colors[colorScheme ?? 'light'].tint,
|
||||
headerShown: false,
|
||||
tabBarButton: HapticTab,
|
||||
}}>
|
||||
<Tabs.Screen
|
||||
name="index"
|
||||
options={{
|
||||
title: 'Home',
|
||||
tabBarIcon: ({ color }) => <IconSymbol size={28} name="house.fill" color={color} />,
|
||||
}}
|
||||
/>
|
||||
<Tabs.Screen
|
||||
name="explore"
|
||||
options={{
|
||||
title: 'Explore',
|
||||
tabBarIcon: ({ color }) => <IconSymbol size={28} name="paperplane.fill" color={color} />,
|
||||
}}
|
||||
/>
|
||||
</Tabs>
|
||||
);
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
import { Image } from 'expo-image';
|
||||
import { Platform, StyleSheet } from 'react-native';
|
||||
|
||||
import { Collapsible } from '@/components/ui/collapsible';
|
||||
import { ExternalLink } from '@/components/external-link';
|
||||
import ParallaxScrollView from '@/components/parallax-scroll-view';
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { IconSymbol } from '@/components/ui/icon-symbol';
|
||||
import { Fonts } from '@/constants/theme';
|
||||
|
||||
export default function TabTwoScreen() {
|
||||
return (
|
||||
<ParallaxScrollView
|
||||
headerBackgroundColor={{ light: '#D0D0D0', dark: '#353636' }}
|
||||
headerImage={
|
||||
<IconSymbol
|
||||
size={310}
|
||||
color="#808080"
|
||||
name="chevron.left.forwardslash.chevron.right"
|
||||
style={styles.headerImage}
|
||||
/>
|
||||
}>
|
||||
<ThemedView style={styles.titleContainer}>
|
||||
<ThemedText
|
||||
type="title"
|
||||
style={{
|
||||
fontFamily: Fonts.rounded,
|
||||
}}>
|
||||
Explore
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
<ThemedText>This app includes example code to help you get started.</ThemedText>
|
||||
<Collapsible title="File-based routing">
|
||||
<ThemedText>
|
||||
This app has two screens:{' '}
|
||||
<ThemedText type="defaultSemiBold">app/(tabs)/index.tsx</ThemedText> and{' '}
|
||||
<ThemedText type="defaultSemiBold">app/(tabs)/explore.tsx</ThemedText>
|
||||
</ThemedText>
|
||||
<ThemedText>
|
||||
The layout file in <ThemedText type="defaultSemiBold">app/(tabs)/_layout.tsx</ThemedText>{' '}
|
||||
sets up the tab navigator.
|
||||
</ThemedText>
|
||||
<ExternalLink href="https://docs.expo.dev/router/introduction">
|
||||
<ThemedText type="link">Learn more</ThemedText>
|
||||
</ExternalLink>
|
||||
</Collapsible>
|
||||
<Collapsible title="Android, iOS, and web support">
|
||||
<ThemedText>
|
||||
You can open this project on Android, iOS, and the web. To open the web version, press{' '}
|
||||
<ThemedText type="defaultSemiBold">w</ThemedText> in the terminal running this project.
|
||||
</ThemedText>
|
||||
</Collapsible>
|
||||
<Collapsible title="Images">
|
||||
<ThemedText>
|
||||
For static images, you can use the <ThemedText type="defaultSemiBold">@2x</ThemedText> and{' '}
|
||||
<ThemedText type="defaultSemiBold">@3x</ThemedText> suffixes to provide files for
|
||||
different screen densities
|
||||
</ThemedText>
|
||||
<Image
|
||||
source={require('@/assets/images/react-logo.png')}
|
||||
style={{ width: 100, height: 100, alignSelf: 'center' }}
|
||||
/>
|
||||
<ExternalLink href="https://reactnative.dev/docs/images">
|
||||
<ThemedText type="link">Learn more</ThemedText>
|
||||
</ExternalLink>
|
||||
</Collapsible>
|
||||
<Collapsible title="Light and dark mode components">
|
||||
<ThemedText>
|
||||
This template has light and dark mode support. The{' '}
|
||||
<ThemedText type="defaultSemiBold">useColorScheme()</ThemedText> hook lets you inspect
|
||||
what the user's current color scheme is, and so you can adjust UI colors accordingly.
|
||||
</ThemedText>
|
||||
<ExternalLink href="https://docs.expo.dev/develop/user-interface/color-themes/">
|
||||
<ThemedText type="link">Learn more</ThemedText>
|
||||
</ExternalLink>
|
||||
</Collapsible>
|
||||
<Collapsible title="Animations">
|
||||
<ThemedText>
|
||||
This template includes an example of an animated component. The{' '}
|
||||
<ThemedText type="defaultSemiBold">components/HelloWave.tsx</ThemedText> component uses
|
||||
the powerful{' '}
|
||||
<ThemedText type="defaultSemiBold" style={{ fontFamily: Fonts.mono }}>
|
||||
react-native-reanimated
|
||||
</ThemedText>{' '}
|
||||
library to create a waving hand animation.
|
||||
</ThemedText>
|
||||
{Platform.select({
|
||||
ios: (
|
||||
<ThemedText>
|
||||
The <ThemedText type="defaultSemiBold">components/ParallaxScrollView.tsx</ThemedText>{' '}
|
||||
component provides a parallax effect for the header image.
|
||||
</ThemedText>
|
||||
),
|
||||
})}
|
||||
</Collapsible>
|
||||
</ParallaxScrollView>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
headerImage: {
|
||||
color: '#808080',
|
||||
bottom: -90,
|
||||
left: -35,
|
||||
position: 'absolute',
|
||||
},
|
||||
titleContainer: {
|
||||
flexDirection: 'row',
|
||||
gap: 8,
|
||||
},
|
||||
});
|
||||
@@ -1,98 +0,0 @@
|
||||
import { Image } from 'expo-image';
|
||||
import { Platform, StyleSheet } from 'react-native';
|
||||
|
||||
import { HelloWave } from '@/components/hello-wave';
|
||||
import ParallaxScrollView from '@/components/parallax-scroll-view';
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { Link } from 'expo-router';
|
||||
|
||||
export default function HomeScreen() {
|
||||
return (
|
||||
<ParallaxScrollView
|
||||
headerBackgroundColor={{ light: '#A1CEDC', dark: '#1D3D47' }}
|
||||
headerImage={
|
||||
<Image
|
||||
source={require('@/assets/images/partial-react-logo.png')}
|
||||
style={styles.reactLogo}
|
||||
/>
|
||||
}>
|
||||
<ThemedView style={styles.titleContainer}>
|
||||
<ThemedText type="title">Welcome!</ThemedText>
|
||||
<HelloWave />
|
||||
</ThemedView>
|
||||
<ThemedView style={styles.stepContainer}>
|
||||
<ThemedText type="subtitle">Step 1: Try it</ThemedText>
|
||||
<ThemedText>
|
||||
Edit <ThemedText type="defaultSemiBold">app/(tabs)/index.tsx</ThemedText> to see changes.
|
||||
Press{' '}
|
||||
<ThemedText type="defaultSemiBold">
|
||||
{Platform.select({
|
||||
ios: 'cmd + d',
|
||||
android: 'cmd + m',
|
||||
web: 'F12',
|
||||
})}
|
||||
</ThemedText>{' '}
|
||||
to open developer tools.
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
<ThemedView style={styles.stepContainer}>
|
||||
<Link href="/modal">
|
||||
<Link.Trigger>
|
||||
<ThemedText type="subtitle">Step 2: Explore</ThemedText>
|
||||
</Link.Trigger>
|
||||
<Link.Preview />
|
||||
<Link.Menu>
|
||||
<Link.MenuAction title="Action" icon="cube" onPress={() => alert('Action pressed')} />
|
||||
<Link.MenuAction
|
||||
title="Share"
|
||||
icon="square.and.arrow.up"
|
||||
onPress={() => alert('Share pressed')}
|
||||
/>
|
||||
<Link.Menu title="More" icon="ellipsis">
|
||||
<Link.MenuAction
|
||||
title="Delete"
|
||||
icon="trash"
|
||||
destructive
|
||||
onPress={() => alert('Delete pressed')}
|
||||
/>
|
||||
</Link.Menu>
|
||||
</Link.Menu>
|
||||
</Link>
|
||||
|
||||
<ThemedText>
|
||||
{`Tap the Explore tab to learn more about what's included in this starter app.`}
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
<ThemedView style={styles.stepContainer}>
|
||||
<ThemedText type="subtitle">Step 3: Get a fresh start</ThemedText>
|
||||
<ThemedText>
|
||||
{`When you're ready, run `}
|
||||
<ThemedText type="defaultSemiBold">npm run reset-project</ThemedText> to get a fresh{' '}
|
||||
<ThemedText type="defaultSemiBold">app</ThemedText> directory. This will move the current{' '}
|
||||
<ThemedText type="defaultSemiBold">app</ThemedText> to{' '}
|
||||
<ThemedText type="defaultSemiBold">app-example</ThemedText>.
|
||||
</ThemedText>
|
||||
</ThemedView>
|
||||
</ParallaxScrollView>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
titleContainer: {
|
||||
flexDirection: 'row',
|
||||
alignItems: 'center',
|
||||
gap: 8,
|
||||
},
|
||||
stepContainer: {
|
||||
gap: 8,
|
||||
marginBottom: 8,
|
||||
},
|
||||
reactLogo: {
|
||||
height: 178,
|
||||
width: 290,
|
||||
bottom: 0,
|
||||
left: 0,
|
||||
position: 'absolute',
|
||||
},
|
||||
});
|
||||
@@ -1,25 +1,17 @@
|
||||
import '../global.css';
|
||||
import { DarkTheme, DefaultTheme, ThemeProvider } from '@react-navigation/native';
|
||||
import { Stack } from 'expo-router';
|
||||
import { StatusBar } from 'expo-status-bar';
|
||||
import 'react-native-reanimated';
|
||||
|
||||
import { useColorScheme } from '@/hooks/use-color-scheme';
|
||||
|
||||
export const unstable_settings = {
|
||||
anchor: '(tabs)',
|
||||
};
|
||||
import { SafeAreaProvider } from 'react-native-safe-area-context';
|
||||
|
||||
export default function RootLayout() {
|
||||
const colorScheme = useColorScheme();
|
||||
|
||||
return (
|
||||
<ThemeProvider value={colorScheme === 'dark' ? DarkTheme : DefaultTheme}>
|
||||
<Stack>
|
||||
<Stack.Screen name="(tabs)" options={{ headerShown: false }} />
|
||||
<Stack.Screen name="modal" options={{ presentation: 'modal', title: 'Modal' }} />
|
||||
<SafeAreaProvider>
|
||||
<Stack screenOptions={{ headerShown: false }}>
|
||||
<Stack.Screen name="index" />
|
||||
<Stack.Screen name="settings" />
|
||||
<Stack.Screen name="audio-test" />
|
||||
</Stack>
|
||||
<StatusBar style="auto" />
|
||||
</ThemeProvider>
|
||||
<StatusBar style="light" />
|
||||
</SafeAreaProvider>
|
||||
);
|
||||
}
|
||||
|
||||
299
packages/voice-mobile/app/audio-test.tsx
Normal file
299
packages/voice-mobile/app/audio-test.tsx
Normal file
@@ -0,0 +1,299 @@
|
||||
import { View, Text, Pressable, ScrollView, Alert } from 'react-native';
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useAudioRecorder } from '../hooks/use-audio-recorder';
|
||||
import { useAudioPlayer } from '../hooks/use-audio-player';
|
||||
import { getRecordingPermissionsAsync, requestRecordingPermissionsAsync } from 'expo-audio';
|
||||
|
||||
export default function AudioTestScreen() {
|
||||
const [permissionGranted, setPermissionGranted] = useState(false);
|
||||
const [recordingStatus, setRecordingStatus] = useState<'idle' | 'recording' | 'processing'>('idle');
|
||||
const [lastRecordingSize, setLastRecordingSize] = useState<number | null>(null);
|
||||
const [playbackStatus, setPlaybackStatus] = useState<string>('No audio playing');
|
||||
const [lastRecordedBlob, setLastRecordedBlob] = useState<Blob | null>(null);
|
||||
|
||||
const audioRecorder = useAudioRecorder({
|
||||
sampleRate: 16000,
|
||||
numberOfChannels: 1,
|
||||
});
|
||||
|
||||
const audioPlayer = useAudioPlayer();
|
||||
|
||||
useEffect(() => {
|
||||
checkAndRequestPermissions();
|
||||
}, []);
|
||||
|
||||
async function checkAndRequestPermissions() {
|
||||
try {
|
||||
const { status } = await getRecordingPermissionsAsync();
|
||||
|
||||
if (status === 'granted') {
|
||||
setPermissionGranted(true);
|
||||
} else {
|
||||
const { granted } = await requestRecordingPermissionsAsync();
|
||||
setPermissionGranted(granted);
|
||||
|
||||
if (!granted) {
|
||||
Alert.alert(
|
||||
'Permission Required',
|
||||
'Microphone permission is required to test audio recording.'
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[AudioTest] Permission error:', error);
|
||||
Alert.alert('Error', 'Failed to check/request microphone permissions');
|
||||
}
|
||||
}
|
||||
|
||||
async function handleStartRecording() {
|
||||
if (!permissionGranted) {
|
||||
Alert.alert('Permission Required', 'Please grant microphone permission first');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
setRecordingStatus('recording');
|
||||
setLastRecordingSize(null);
|
||||
await audioRecorder.start();
|
||||
} catch (error: any) {
|
||||
console.error('[AudioTest] Start recording error:', error);
|
||||
Alert.alert('Recording Error', error.message);
|
||||
setRecordingStatus('idle');
|
||||
}
|
||||
}
|
||||
|
||||
async function handleStopRecording() {
|
||||
try {
|
||||
setRecordingStatus('processing');
|
||||
const audioBlob = await audioRecorder.stop();
|
||||
|
||||
setLastRecordingSize(audioBlob.size);
|
||||
setLastRecordedBlob(audioBlob);
|
||||
setRecordingStatus('idle');
|
||||
|
||||
Alert.alert(
|
||||
'Recording Complete',
|
||||
`Audio recorded successfully!\nSize: ${audioBlob.size} bytes\nType: ${audioBlob.type}`
|
||||
);
|
||||
} catch (error: any) {
|
||||
console.error('[AudioTest] Stop recording error:', error);
|
||||
Alert.alert('Recording Error', error.message);
|
||||
setRecordingStatus('idle');
|
||||
}
|
||||
}
|
||||
|
||||
async function handlePlayLastRecording() {
|
||||
if (!lastRecordedBlob) {
|
||||
Alert.alert('No Recording', 'Please record audio first');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
setPlaybackStatus('Playing last recording...');
|
||||
const duration = await audioPlayer.play(lastRecordedBlob);
|
||||
setPlaybackStatus(`Playback complete (${duration.toFixed(2)}s)`);
|
||||
} catch (error: any) {
|
||||
console.error('[AudioTest] Playback error:', error);
|
||||
Alert.alert('Playback Error', error.message);
|
||||
setPlaybackStatus('Playback failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function handlePlayTestAudio() {
|
||||
// Create a test audio blob (silent audio for testing)
|
||||
// In a real scenario, this would be audio from the server
|
||||
try {
|
||||
setPlaybackStatus('Playing test audio...');
|
||||
|
||||
// For now, we'll show an alert since we don't have a test audio file
|
||||
Alert.alert(
|
||||
'Test Audio',
|
||||
'This would play a test audio file. Try recording and playing back instead.'
|
||||
);
|
||||
setPlaybackStatus('Test audio not implemented');
|
||||
} catch (error: any) {
|
||||
console.error('[AudioTest] Test playback error:', error);
|
||||
Alert.alert('Playback Error', error.message);
|
||||
setPlaybackStatus('Test playback failed');
|
||||
}
|
||||
}
|
||||
|
||||
function handleStopPlayback() {
|
||||
audioPlayer.stop();
|
||||
setPlaybackStatus('Playback stopped');
|
||||
}
|
||||
|
||||
return (
|
||||
<ScrollView className="flex-1 bg-white dark:bg-black">
|
||||
<View className="p-6">
|
||||
{/* Header */}
|
||||
<Text className="text-3xl font-bold text-gray-900 dark:text-white mb-2">
|
||||
Audio Test
|
||||
</Text>
|
||||
<Text className="text-base text-gray-600 dark:text-gray-400 mb-8">
|
||||
Test audio recording and playback functionality
|
||||
</Text>
|
||||
|
||||
{/* Permission Status */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-lg font-semibold text-gray-900 dark:text-white mb-2">
|
||||
Permission Status
|
||||
</Text>
|
||||
<View className={`p-4 rounded-lg ${permissionGranted ? 'bg-green-100 dark:bg-green-900' : 'bg-red-100 dark:bg-red-900'}`}>
|
||||
<Text className={`text-base ${permissionGranted ? 'text-green-800 dark:text-green-200' : 'text-red-800 dark:text-red-200'}`}>
|
||||
{permissionGranted ? '✓ Microphone permission granted' : '✗ Microphone permission denied'}
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Recording Section */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-lg font-semibold text-gray-900 dark:text-white mb-4">
|
||||
Audio Recording
|
||||
</Text>
|
||||
|
||||
{/* Recording Status */}
|
||||
<View className="mb-4 p-4 bg-gray-100 dark:bg-gray-800 rounded-lg">
|
||||
<Text className="text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">
|
||||
Status
|
||||
</Text>
|
||||
<Text className="text-base text-gray-900 dark:text-white">
|
||||
{recordingStatus === 'idle' && 'Ready to record'}
|
||||
{recordingStatus === 'recording' && '🔴 Recording...'}
|
||||
{recordingStatus === 'processing' && 'Processing...'}
|
||||
</Text>
|
||||
</View>
|
||||
|
||||
{/* Recording Controls */}
|
||||
<View className="flex-row gap-3 mb-4">
|
||||
<Pressable
|
||||
onPress={handleStartRecording}
|
||||
disabled={recordingStatus !== 'idle' || !permissionGranted}
|
||||
className={`flex-1 p-4 rounded-lg items-center ${
|
||||
recordingStatus !== 'idle' || !permissionGranted
|
||||
? 'bg-gray-300 dark:bg-gray-700'
|
||||
: 'bg-blue-500 dark:bg-blue-600'
|
||||
}`}
|
||||
>
|
||||
<Text className={`text-base font-semibold ${
|
||||
recordingStatus !== 'idle' || !permissionGranted
|
||||
? 'text-gray-500 dark:text-gray-400'
|
||||
: 'text-white'
|
||||
}`}>
|
||||
Start Recording
|
||||
</Text>
|
||||
</Pressable>
|
||||
|
||||
<Pressable
|
||||
onPress={handleStopRecording}
|
||||
disabled={recordingStatus !== 'recording'}
|
||||
className={`flex-1 p-4 rounded-lg items-center ${
|
||||
recordingStatus !== 'recording'
|
||||
? 'bg-gray-300 dark:bg-gray-700'
|
||||
: 'bg-red-500 dark:bg-red-600'
|
||||
}`}
|
||||
>
|
||||
<Text className={`text-base font-semibold ${
|
||||
recordingStatus !== 'recording'
|
||||
? 'text-gray-500 dark:text-gray-400'
|
||||
: 'text-white'
|
||||
}`}>
|
||||
Stop Recording
|
||||
</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
|
||||
{/* Last Recording Info */}
|
||||
{lastRecordingSize !== null && (
|
||||
<View className="p-4 bg-blue-50 dark:bg-blue-950 rounded-lg">
|
||||
<Text className="text-sm font-medium text-blue-700 dark:text-blue-300 mb-1">
|
||||
Last Recording
|
||||
</Text>
|
||||
<Text className="text-base text-blue-900 dark:text-blue-100">
|
||||
Size: {lastRecordingSize} bytes
|
||||
</Text>
|
||||
<Text className="text-sm text-blue-600 dark:text-blue-400 mt-1">
|
||||
Type: audio/m4a
|
||||
</Text>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
|
||||
{/* Playback Section */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-lg font-semibold text-gray-900 dark:text-white mb-4">
|
||||
Audio Playback
|
||||
</Text>
|
||||
|
||||
{/* Playback Status */}
|
||||
<View className="mb-4 p-4 bg-gray-100 dark:bg-gray-800 rounded-lg">
|
||||
<Text className="text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">
|
||||
Status
|
||||
</Text>
|
||||
<Text className="text-base text-gray-900 dark:text-white">
|
||||
{playbackStatus}
|
||||
</Text>
|
||||
</View>
|
||||
|
||||
{/* Playback Controls */}
|
||||
<View className="gap-3">
|
||||
<Pressable
|
||||
onPress={handlePlayLastRecording}
|
||||
disabled={!lastRecordedBlob}
|
||||
className={`p-4 rounded-lg items-center ${
|
||||
!lastRecordedBlob
|
||||
? 'bg-gray-300 dark:bg-gray-700'
|
||||
: 'bg-green-500 dark:bg-green-600'
|
||||
}`}
|
||||
>
|
||||
<Text className={`text-base font-semibold ${
|
||||
!lastRecordedBlob
|
||||
? 'text-gray-500 dark:text-gray-400'
|
||||
: 'text-white'
|
||||
}`}>
|
||||
Play Last Recording
|
||||
</Text>
|
||||
</Pressable>
|
||||
|
||||
<Pressable
|
||||
onPress={handlePlayTestAudio}
|
||||
className="p-4 rounded-lg items-center bg-purple-500 dark:bg-purple-600"
|
||||
>
|
||||
<Text className="text-base font-semibold text-white">
|
||||
Play Test Audio
|
||||
</Text>
|
||||
</Pressable>
|
||||
|
||||
<Pressable
|
||||
onPress={handleStopPlayback}
|
||||
className="p-4 rounded-lg items-center bg-orange-500 dark:bg-orange-600"
|
||||
>
|
||||
<Text className="text-base font-semibold text-white">
|
||||
Stop Playback
|
||||
</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Audio Settings Info */}
|
||||
<View className="p-4 bg-gray-50 dark:bg-gray-900 rounded-lg">
|
||||
<Text className="text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
|
||||
Audio Configuration
|
||||
</Text>
|
||||
<Text className="text-sm text-gray-600 dark:text-gray-400 mb-1">
|
||||
• Sample Rate: 16000 Hz
|
||||
</Text>
|
||||
<Text className="text-sm text-gray-600 dark:text-gray-400 mb-1">
|
||||
• Channels: 1 (Mono)
|
||||
</Text>
|
||||
<Text className="text-sm text-gray-600 dark:text-gray-400 mb-1">
|
||||
• Format: M4A/AAC
|
||||
</Text>
|
||||
<Text className="text-sm text-gray-600 dark:text-gray-400">
|
||||
• Optimized for voice/speech
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
</ScrollView>
|
||||
);
|
||||
}
|
||||
939
packages/voice-mobile/app/index.tsx
Normal file
939
packages/voice-mobile/app/index.tsx
Normal file
@@ -0,0 +1,939 @@
|
||||
import { useEffect, useState, useRef } from 'react';
|
||||
import { View, ScrollView, KeyboardAvoidingView, Platform, Pressable, Text, TextInput, Keyboard } from 'react-native';
|
||||
import { router } from 'expo-router';
|
||||
import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake';
|
||||
import { useSafeAreaInsets } from 'react-native-safe-area-context';
|
||||
|
||||
// Simple unique ID generator
|
||||
let messageIdCounter = 0;
|
||||
function generateMessageId(): string {
|
||||
return `msg_${Date.now()}_${messageIdCounter++}`;
|
||||
}
|
||||
import { useWebSocket } from '@/hooks/use-websocket';
|
||||
import { useAudioRecorder } from '@/hooks/use-audio-recorder';
|
||||
import { useAudioPlayer } from '@/hooks/use-audio-player';
|
||||
import { useSettings } from '@/hooks/use-settings';
|
||||
import { ConnectionStatus } from '@/components/connection-status';
|
||||
import { UserMessage, AssistantMessage, ActivityLog, ToolCall } from '@/components/message';
|
||||
import { ArtifactDrawer, type Artifact } from '@/components/artifact-drawer';
|
||||
import { ActiveProcesses } from '@/components/active-processes';
|
||||
import { AgentStreamView } from '@/components/agent-stream-view';
|
||||
import { ConversationSelector } from '@/components/conversation-selector';
|
||||
import { MaterialIcons } from '@expo/vector-icons';
|
||||
import type {
|
||||
ActivityLogPayload,
|
||||
SessionInboundMessage,
|
||||
WSInboundMessage,
|
||||
} from '@voice-assistant/server/messages';
|
||||
import type { AgentStatus } from '@voice-assistant/server/acp/types';
|
||||
import type { SessionNotification } from '@agentclientprotocol/sdk';
|
||||
|
||||
type MessageEntry =
|
||||
| {
|
||||
type: 'user';
|
||||
id: string;
|
||||
timestamp: number;
|
||||
message: string;
|
||||
}
|
||||
| {
|
||||
type: 'assistant';
|
||||
id: string;
|
||||
timestamp: number;
|
||||
message: string;
|
||||
}
|
||||
| {
|
||||
type: 'activity';
|
||||
id: string;
|
||||
timestamp: number;
|
||||
activityType: 'system' | 'info' | 'success' | 'error';
|
||||
message: string;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
| {
|
||||
type: 'tool_call';
|
||||
id: string;
|
||||
timestamp: number;
|
||||
toolName: string;
|
||||
args: any;
|
||||
result?: any;
|
||||
error?: any;
|
||||
status: 'executing' | 'completed' | 'failed';
|
||||
};
|
||||
|
||||
type ViewMode = 'orchestrator' | 'agent';
|
||||
|
||||
interface Agent {
|
||||
id: string;
|
||||
status: AgentStatus;
|
||||
createdAt: Date;
|
||||
type: 'claude';
|
||||
sessionId?: string;
|
||||
error?: string;
|
||||
currentModeId?: string;
|
||||
availableModes?: Array<{ id: string; name: string; description?: string | null }>;
|
||||
}
|
||||
|
||||
interface Command {
|
||||
id: string;
|
||||
name: string;
|
||||
workingDirectory: string;
|
||||
currentCommand: string;
|
||||
isDead: boolean;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
interface AgentUpdate {
|
||||
timestamp: Date;
|
||||
notification: SessionNotification;
|
||||
}
|
||||
|
||||
export default function VoiceAssistantScreen() {
|
||||
const { settings, isLoading: settingsLoading } = useSettings();
|
||||
const [conversationId, setConversationId] = useState<string | null>(null);
|
||||
const ws = useWebSocket(settings.serverUrl, conversationId);
|
||||
const audioRecorder = useAudioRecorder();
|
||||
const audioPlayer = useAudioPlayer({ useSpeaker: settings.useSpeaker });
|
||||
const insets = useSafeAreaInsets();
|
||||
|
||||
const [messages, setMessages] = useState<MessageEntry[]>([]);
|
||||
const [currentAssistantMessage, setCurrentAssistantMessage] = useState('');
|
||||
const [isProcessingAudio, setIsProcessingAudio] = useState(false);
|
||||
const [isPlayingAudio, setIsPlayingAudio] = useState(false);
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [userInput, setUserInput] = useState('');
|
||||
|
||||
// Artifact state
|
||||
const [artifacts, setArtifacts] = useState<Map<string, Artifact>>(new Map());
|
||||
const [currentArtifact, setCurrentArtifact] = useState<Artifact | null>(null);
|
||||
|
||||
// Multi-view navigation state
|
||||
const [viewMode, setViewMode] = useState<ViewMode>('orchestrator');
|
||||
const [activeAgentId, setActiveAgentId] = useState<string | null>(null);
|
||||
|
||||
// Agent and command state
|
||||
const [agents, setAgents] = useState<Map<string, Agent>>(new Map());
|
||||
const [commands, setCommands] = useState<Map<string, Command>>(new Map());
|
||||
const [agentUpdates, setAgentUpdates] = useState<Map<string, AgentUpdate[]>>(new Map());
|
||||
|
||||
const scrollViewRef = useRef<ScrollView>(null);
|
||||
|
||||
// Keep screen awake if setting is enabled
|
||||
useEffect(() => {
|
||||
if (settings.keepScreenOn) {
|
||||
activateKeepAwakeAsync('voice-assistant');
|
||||
} else {
|
||||
deactivateKeepAwake('voice-assistant');
|
||||
}
|
||||
}, [settings.keepScreenOn]);
|
||||
|
||||
// Auto-scroll to bottom when messages update
|
||||
useEffect(() => {
|
||||
scrollViewRef.current?.scrollToEnd({ animated: true });
|
||||
}, [messages, currentAssistantMessage]);
|
||||
|
||||
// WebSocket message handlers
|
||||
useEffect(() => {
|
||||
// Session state handler - initial agents/commands
|
||||
const unsubSessionState = ws.on('session_state', (message) => {
|
||||
if (message.type !== 'session_state') return;
|
||||
const { agents: agentsList, commands: commandsList } = message.payload;
|
||||
|
||||
console.log('[App] Session state received:', agentsList.length, 'agents,', commandsList.length, 'commands');
|
||||
|
||||
// Update agents
|
||||
setAgents(new Map(agentsList.map((a) => [a.id, a as Agent])));
|
||||
|
||||
// Update commands
|
||||
setCommands(new Map(commandsList.map((c) => [c.id, c as Command])));
|
||||
});
|
||||
|
||||
// Agent created handler
|
||||
const unsubAgentCreated = ws.on('agent_created', (message) => {
|
||||
if (message.type !== 'agent_created') return;
|
||||
const { agentId, status, type, currentModeId, availableModes } = message.payload;
|
||||
|
||||
console.log('[App] Agent created:', agentId);
|
||||
|
||||
const agent: Agent = {
|
||||
id: agentId,
|
||||
status: status as AgentStatus,
|
||||
type,
|
||||
createdAt: new Date(),
|
||||
currentModeId,
|
||||
availableModes,
|
||||
};
|
||||
|
||||
setAgents((prev) => new Map(prev).set(agentId, agent));
|
||||
setAgentUpdates((prev) => new Map(prev).set(agentId, []));
|
||||
});
|
||||
|
||||
// Agent update handler
|
||||
const unsubAgentUpdate = ws.on('agent_update', (message) => {
|
||||
if (message.type !== 'agent_update') return;
|
||||
const { agentId, timestamp, notification } = message.payload;
|
||||
|
||||
console.log('[App] Agent update:', agentId);
|
||||
|
||||
setAgentUpdates((prev) => {
|
||||
const updated = new Map(prev);
|
||||
const updates = updated.get(agentId) || [];
|
||||
updated.set(agentId, [
|
||||
...updates,
|
||||
{
|
||||
timestamp: new Date(timestamp),
|
||||
notification,
|
||||
},
|
||||
]);
|
||||
return updated;
|
||||
});
|
||||
});
|
||||
|
||||
// Agent status handler
|
||||
const unsubAgentStatus = ws.on('agent_status', (message) => {
|
||||
if (message.type !== 'agent_status') return;
|
||||
const { agentId, status, info } = message.payload;
|
||||
|
||||
console.log('[App] Agent status changed:', agentId, status);
|
||||
|
||||
setAgents((prev) => {
|
||||
const updated = new Map(prev);
|
||||
const agent = updated.get(agentId);
|
||||
if (agent) {
|
||||
updated.set(agentId, {
|
||||
...agent,
|
||||
status: status as AgentStatus,
|
||||
sessionId: info.sessionId,
|
||||
error: info.error,
|
||||
});
|
||||
}
|
||||
return updated;
|
||||
});
|
||||
});
|
||||
|
||||
// Activity log handler
|
||||
const unsubActivity = ws.on('activity_log', (message) => {
|
||||
if (message.type !== 'activity_log') return;
|
||||
const data = message.payload;
|
||||
|
||||
// Handle tool calls
|
||||
if (data.type === 'tool_call' && data.metadata) {
|
||||
const {
|
||||
toolCallId,
|
||||
toolName,
|
||||
arguments: args,
|
||||
} = data.metadata as {
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
arguments: unknown;
|
||||
};
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'tool_call',
|
||||
id: toolCallId,
|
||||
timestamp: Date.now(),
|
||||
toolName,
|
||||
args,
|
||||
status: 'executing',
|
||||
},
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle tool results
|
||||
if (data.type === 'tool_result' && data.metadata) {
|
||||
const { toolCallId, result } = data.metadata as {
|
||||
toolCallId: string;
|
||||
result: unknown;
|
||||
};
|
||||
|
||||
setMessages((prev) =>
|
||||
prev.map((msg) =>
|
||||
msg.type === 'tool_call' && msg.id === toolCallId
|
||||
? { ...msg, result, status: 'completed' as const }
|
||||
: msg
|
||||
)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle tool errors
|
||||
if (
|
||||
data.type === 'error' &&
|
||||
data.metadata &&
|
||||
'toolCallId' in data.metadata
|
||||
) {
|
||||
const { toolCallId, error } = data.metadata as {
|
||||
toolCallId: string;
|
||||
error: unknown;
|
||||
};
|
||||
|
||||
setMessages((prev) =>
|
||||
prev.map((msg) =>
|
||||
msg.type === 'tool_call' && msg.id === toolCallId
|
||||
? { ...msg, error, status: 'failed' as const }
|
||||
: msg
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Map activity types to message types
|
||||
let activityType: 'system' | 'info' | 'success' | 'error' = 'info';
|
||||
if (data.type === 'error') activityType = 'error';
|
||||
|
||||
// Add user transcripts as user messages
|
||||
if (data.type === 'transcript') {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'user',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
message: data.content,
|
||||
},
|
||||
]);
|
||||
return;
|
||||
}
|
||||
|
||||
// Add assistant messages
|
||||
if (data.type === 'assistant') {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'assistant',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
message: data.content,
|
||||
},
|
||||
]);
|
||||
setCurrentAssistantMessage('');
|
||||
return;
|
||||
}
|
||||
|
||||
// Add activity log for other types
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType,
|
||||
message: data.content,
|
||||
metadata: data.metadata,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
// Assistant chunk handler (streaming)
|
||||
const unsubChunk = ws.on('assistant_chunk', (message) => {
|
||||
if (message.type !== 'assistant_chunk') return;
|
||||
setCurrentAssistantMessage((prev) => prev + message.payload.chunk);
|
||||
});
|
||||
|
||||
// Transcription result handler
|
||||
const unsubTranscription = ws.on('transcription_result', (message) => {
|
||||
if (message.type !== 'transcription_result') return;
|
||||
|
||||
setIsProcessingAudio(false);
|
||||
|
||||
const transcriptText = message.payload.text.trim();
|
||||
|
||||
if (!transcriptText) {
|
||||
// Empty transcription - false positive, resume playback
|
||||
console.log('[App] Empty transcription (false positive) - resuming playback');
|
||||
audioPlayer.resume();
|
||||
} else {
|
||||
// Has content - real speech detected, stop playback
|
||||
console.log('[App] Transcription received - stopping playback');
|
||||
audioPlayer.stop();
|
||||
setIsPlayingAudio(false);
|
||||
setCurrentAssistantMessage('');
|
||||
}
|
||||
});
|
||||
|
||||
// Audio output handler (TTS)
|
||||
const unsubAudioOutput = ws.on('audio_output', async (message) => {
|
||||
if (message.type !== 'audio_output') return;
|
||||
const data = message.payload;
|
||||
|
||||
try {
|
||||
setIsPlayingAudio(true);
|
||||
|
||||
// Create blob-like object with correct mime type (React Native compatible)
|
||||
const mimeType =
|
||||
data.format === 'mp3' ? 'audio/mpeg' : `audio/${data.format}`;
|
||||
const base64Audio = data.audio;
|
||||
|
||||
// Create a Blob-like object that works in React Native
|
||||
const audioBlob = {
|
||||
type: mimeType,
|
||||
size: Math.ceil(base64Audio.length * 3 / 4), // Approximate size from base64
|
||||
arrayBuffer: async () => {
|
||||
// Convert base64 to ArrayBuffer
|
||||
const binaryString = atob(base64Audio);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return bytes.buffer;
|
||||
},
|
||||
} as Blob;
|
||||
|
||||
// Play audio
|
||||
await audioPlayer.play(audioBlob);
|
||||
|
||||
// Send confirmation back to server (properly typed)
|
||||
const confirmMessage: WSInboundMessage = {
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'audio_played',
|
||||
id: data.id,
|
||||
},
|
||||
};
|
||||
ws.send(confirmMessage);
|
||||
|
||||
setIsPlayingAudio(false);
|
||||
} catch (error: any) {
|
||||
console.error('[App] Audio playback error:', error);
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'error',
|
||||
message: `Audio playback failed: ${error.message}`,
|
||||
},
|
||||
]);
|
||||
setIsPlayingAudio(false);
|
||||
}
|
||||
});
|
||||
|
||||
// Status handler
|
||||
const unsubStatus = ws.on('status', (message) => {
|
||||
if (message.type !== 'status') return;
|
||||
const msg =
|
||||
'message' in message.payload
|
||||
? String(message.payload.message)
|
||||
: `Status: ${message.payload.status}`;
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'info',
|
||||
message: msg,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
// Conversation loaded handler
|
||||
const unsubConversationLoaded = ws.on('conversation_loaded', (message) => {
|
||||
if (message.type !== 'conversation_loaded') return;
|
||||
const { conversationId, messageCount } = message.payload;
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'success',
|
||||
message: `Loaded conversation with ${messageCount} messages`,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
// Artifact handler
|
||||
const unsubArtifact = ws.on('artifact', (message) => {
|
||||
if (message.type !== 'artifact') return;
|
||||
const artifactData = message.payload;
|
||||
|
||||
console.log('[App] Received artifact:', artifactData.id, artifactData.type, artifactData.title);
|
||||
|
||||
// Store artifact
|
||||
setArtifacts((prev) => {
|
||||
const updated = new Map(prev);
|
||||
updated.set(artifactData.id, artifactData);
|
||||
return updated;
|
||||
});
|
||||
|
||||
// Show drawer immediately
|
||||
setCurrentArtifact(artifactData);
|
||||
});
|
||||
|
||||
return () => {
|
||||
unsubSessionState();
|
||||
unsubAgentCreated();
|
||||
unsubAgentUpdate();
|
||||
unsubAgentStatus();
|
||||
unsubActivity();
|
||||
unsubChunk();
|
||||
unsubTranscription();
|
||||
unsubAudioOutput();
|
||||
unsubStatus();
|
||||
unsubConversationLoaded();
|
||||
unsubArtifact();
|
||||
};
|
||||
}, [ws, audioPlayer]);
|
||||
|
||||
// Connection status handler
|
||||
useEffect(() => {
|
||||
if (ws.isConnected) {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'success',
|
||||
message: 'WebSocket connected',
|
||||
},
|
||||
]);
|
||||
} else if (messages.length > 0) {
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'error',
|
||||
message: 'WebSocket disconnected',
|
||||
},
|
||||
]);
|
||||
}
|
||||
}, [ws.isConnected]);
|
||||
|
||||
// Voice button handler
|
||||
async function handleVoicePress() {
|
||||
if (!ws.isConnected) return;
|
||||
|
||||
// If recording, stop and send
|
||||
if (isRecording) {
|
||||
try {
|
||||
console.log('[App] Stopping recording...');
|
||||
const audioBlob = await audioRecorder.stop();
|
||||
setIsRecording(false);
|
||||
|
||||
const format = audioBlob.type || 'audio/m4a';
|
||||
console.log(
|
||||
`[App] Recording complete: ${audioBlob.size} bytes, format: ${format}`
|
||||
);
|
||||
|
||||
setIsProcessingAudio(true);
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'info',
|
||||
message: 'Sending audio to server...',
|
||||
},
|
||||
]);
|
||||
|
||||
// Convert to base64
|
||||
const arrayBuffer = await audioBlob.arrayBuffer();
|
||||
const base64Audio = btoa(
|
||||
new Uint8Array(arrayBuffer).reduce(
|
||||
(data, byte) => data + String.fromCharCode(byte),
|
||||
''
|
||||
)
|
||||
);
|
||||
|
||||
// Send to server (properly typed)
|
||||
const audioMessage: WSInboundMessage = {
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'audio_chunk',
|
||||
audio: base64Audio,
|
||||
format: format,
|
||||
isLast: true,
|
||||
},
|
||||
};
|
||||
ws.send(audioMessage);
|
||||
|
||||
console.log(
|
||||
`[App] Sent audio: ${audioBlob.size} bytes, format: ${format}`
|
||||
);
|
||||
} catch (error: any) {
|
||||
console.error('[App] Recording error:', error);
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'error',
|
||||
message: `Failed to record audio: ${error.message}`,
|
||||
},
|
||||
]);
|
||||
setIsRecording(false);
|
||||
}
|
||||
} else {
|
||||
// Start recording
|
||||
try {
|
||||
console.log('[App] Starting recording...');
|
||||
|
||||
// Stop any currently playing audio
|
||||
audioPlayer.stop();
|
||||
setIsPlayingAudio(false);
|
||||
|
||||
await audioRecorder.start();
|
||||
setIsRecording(true);
|
||||
} catch (error: any) {
|
||||
console.error('[App] Failed to start recording:', error);
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'error',
|
||||
message: `Failed to start recording: ${error.message}`,
|
||||
},
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle artifact click from activity log
|
||||
function handleArtifactClick(artifactId: string) {
|
||||
const artifact = artifacts.get(artifactId);
|
||||
if (artifact) {
|
||||
console.log('[App] Opening artifact:', artifactId);
|
||||
setCurrentArtifact(artifact);
|
||||
} else {
|
||||
console.warn('[App] Artifact not found:', artifactId);
|
||||
}
|
||||
}
|
||||
|
||||
// Close artifact drawer
|
||||
function handleCloseArtifact() {
|
||||
setCurrentArtifact(null);
|
||||
}
|
||||
|
||||
// Handle agent selection
|
||||
function handleSelectAgent(agentId: string) {
|
||||
setActiveAgentId(agentId);
|
||||
setViewMode('agent');
|
||||
}
|
||||
|
||||
// Handle back to orchestrator
|
||||
function handleBackToOrchestrator() {
|
||||
setActiveAgentId(null);
|
||||
setViewMode('orchestrator');
|
||||
}
|
||||
|
||||
// Agent control handlers
|
||||
function handleKillAgent(agentId: string) {
|
||||
console.log('[App] Kill agent:', agentId);
|
||||
// TODO: Implement kill agent API call
|
||||
}
|
||||
|
||||
function handleCancelAgent(agentId: string) {
|
||||
console.log('[App] Cancel agent:', agentId);
|
||||
// TODO: Implement cancel agent API call
|
||||
}
|
||||
|
||||
// Text message handlers
|
||||
function handleSendMessage() {
|
||||
if (!userInput.trim() || !ws.isConnected) return;
|
||||
|
||||
// Stop any currently playing audio
|
||||
audioPlayer.stop();
|
||||
setIsPlayingAudio(false);
|
||||
|
||||
// Send message to server using the hook's method
|
||||
ws.sendUserMessage(userInput);
|
||||
|
||||
// Clear input and reset streaming state
|
||||
setUserInput('');
|
||||
setCurrentAssistantMessage('');
|
||||
Keyboard.dismiss();
|
||||
}
|
||||
|
||||
function handleCancel() {
|
||||
console.log('[App] Cancelling operations...');
|
||||
|
||||
// Stop audio playback
|
||||
audioPlayer.stop();
|
||||
setIsPlayingAudio(false);
|
||||
|
||||
// Clear streaming state
|
||||
setCurrentAssistantMessage('');
|
||||
|
||||
// Reset processing state
|
||||
setIsProcessingAudio(false);
|
||||
|
||||
// Send abort request to server (properly typed)
|
||||
const abortMessage: WSInboundMessage = {
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'abort_request',
|
||||
},
|
||||
};
|
||||
ws.send(abortMessage);
|
||||
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
type: 'activity',
|
||||
id: generateMessageId(),
|
||||
timestamp: Date.now(),
|
||||
activityType: 'info',
|
||||
message: 'Operations cancelled',
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
// Compute if we're processing
|
||||
const isInProgress =
|
||||
isProcessingAudio || isPlayingAudio || currentAssistantMessage.length > 0;
|
||||
|
||||
function handleButtonClick() {
|
||||
if (isRecording) {
|
||||
// Stop recording and send the audio
|
||||
handleVoicePress();
|
||||
} else if (isInProgress) {
|
||||
// Cancel processing/playback
|
||||
handleCancel();
|
||||
} else if (userInput.trim()) {
|
||||
// Send text message
|
||||
handleSendMessage();
|
||||
} else {
|
||||
// Start recording
|
||||
handleVoicePress();
|
||||
}
|
||||
}
|
||||
|
||||
// Conversation selection handler
|
||||
function handleSelectConversation(newConversationId: string | null) {
|
||||
// Clear all state
|
||||
setMessages([]);
|
||||
setCurrentAssistantMessage('');
|
||||
setUserInput('');
|
||||
setArtifacts(new Map());
|
||||
setCurrentArtifact(null);
|
||||
setAgents(new Map());
|
||||
setCommands(new Map());
|
||||
setAgentUpdates(new Map());
|
||||
setViewMode('orchestrator');
|
||||
setActiveAgentId(null);
|
||||
|
||||
// Stop any ongoing operations
|
||||
if (isRecording) {
|
||||
audioRecorder.stop().catch(console.error);
|
||||
setIsRecording(false);
|
||||
}
|
||||
audioPlayer.stop();
|
||||
setIsPlayingAudio(false);
|
||||
setIsProcessingAudio(false);
|
||||
|
||||
// Update conversation ID (will trigger WebSocket reconnection)
|
||||
setConversationId(newConversationId);
|
||||
}
|
||||
|
||||
// Render agent stream view
|
||||
if (viewMode === 'agent' && activeAgentId) {
|
||||
const agent = agents.get(activeAgentId);
|
||||
const updates = agentUpdates.get(activeAgentId) || [];
|
||||
|
||||
if (!agent) {
|
||||
return (
|
||||
<View className="flex-1 bg-black items-center justify-center">
|
||||
<Text className="text-white">Agent not found</Text>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<AgentStreamView
|
||||
agentId={activeAgentId}
|
||||
agent={agent}
|
||||
updates={updates}
|
||||
onBack={handleBackToOrchestrator}
|
||||
onKillAgent={handleKillAgent}
|
||||
onCancelAgent={handleCancelAgent}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
// Render orchestrator view (main chat)
|
||||
return (
|
||||
<KeyboardAvoidingView
|
||||
behavior="padding"
|
||||
className="flex-1 bg-black"
|
||||
>
|
||||
<View className="flex-1">
|
||||
{/* Connection status header with buttons */}
|
||||
<View style={{ paddingTop: insets.top + 16 }}>
|
||||
<View className="flex-row items-center justify-between px-6 pb-4">
|
||||
<View className="flex-1">
|
||||
<ConnectionStatus isConnected={ws.isConnected} />
|
||||
</View>
|
||||
<View className="flex-row items-center gap-2">
|
||||
<ConversationSelector
|
||||
currentConversationId={conversationId}
|
||||
onSelectConversation={handleSelectConversation}
|
||||
websocket={ws}
|
||||
/>
|
||||
<Pressable
|
||||
onPress={() => router.push('/settings')}
|
||||
className="bg-zinc-800 p-3 rounded-lg"
|
||||
>
|
||||
<MaterialIcons name="settings" size={20} color="white" />
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Active processes bar */}
|
||||
<ActiveProcesses
|
||||
agents={Array.from(agents.values())}
|
||||
commands={Array.from(commands.values())}
|
||||
activeProcessId={activeAgentId}
|
||||
activeProcessType={activeAgentId ? 'agent' : null}
|
||||
onSelectAgent={handleSelectAgent}
|
||||
onBackToOrchestrator={handleBackToOrchestrator}
|
||||
/>
|
||||
|
||||
{/* Messages */}
|
||||
<ScrollView
|
||||
ref={scrollViewRef}
|
||||
className="flex-1"
|
||||
contentContainerClassName="pb-4"
|
||||
>
|
||||
{messages.map((msg) => {
|
||||
if (msg.type === 'user') {
|
||||
return (
|
||||
<UserMessage
|
||||
key={msg.id}
|
||||
message={msg.message}
|
||||
timestamp={msg.timestamp}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (msg.type === 'assistant') {
|
||||
return (
|
||||
<AssistantMessage
|
||||
key={msg.id}
|
||||
message={msg.message}
|
||||
timestamp={msg.timestamp}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (msg.type === 'activity') {
|
||||
return (
|
||||
<ActivityLog
|
||||
key={msg.id}
|
||||
type={msg.activityType}
|
||||
message={msg.message}
|
||||
timestamp={msg.timestamp}
|
||||
metadata={msg.metadata}
|
||||
onArtifactClick={handleArtifactClick}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (msg.type === 'tool_call') {
|
||||
return (
|
||||
<ToolCall
|
||||
key={msg.id}
|
||||
toolName={msg.toolName}
|
||||
args={msg.args}
|
||||
result={msg.result}
|
||||
error={msg.error}
|
||||
status={msg.status}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
})}
|
||||
|
||||
{/* Streaming assistant message */}
|
||||
{currentAssistantMessage && (
|
||||
<AssistantMessage
|
||||
message={currentAssistantMessage}
|
||||
timestamp={Date.now()}
|
||||
isStreaming={true}
|
||||
/>
|
||||
)}
|
||||
</ScrollView>
|
||||
|
||||
{/* Input area */}
|
||||
<View className="pt-4 px-6 border-t border-zinc-800" style={{ paddingBottom: Math.max(insets.bottom, 32) }}>
|
||||
{/* Text input */}
|
||||
<TextInput
|
||||
value={userInput}
|
||||
onChangeText={setUserInput}
|
||||
placeholder="Say something..."
|
||||
placeholderTextColor="#71717a"
|
||||
className="bg-zinc-800 text-white rounded-2xl px-4 py-3 mb-4 max-h-32"
|
||||
multiline
|
||||
editable={!isRecording && ws.isConnected}
|
||||
/>
|
||||
|
||||
{/* Buttons */}
|
||||
<View className="flex-row items-center justify-center gap-4">
|
||||
{/* Realtime mode button (placeholder) */}
|
||||
<Pressable
|
||||
disabled={true}
|
||||
className="w-16 h-16 rounded-full bg-zinc-800 items-center justify-center opacity-50"
|
||||
>
|
||||
<View className="w-8 h-8 items-center justify-center">
|
||||
<View className="w-6 h-6 rounded-full border-2 border-white" />
|
||||
<View className="absolute w-3 h-3 rounded-full border-2 border-white" />
|
||||
</View>
|
||||
</Pressable>
|
||||
|
||||
{/* Main action button */}
|
||||
<Pressable
|
||||
onPress={handleButtonClick}
|
||||
disabled={!ws.isConnected}
|
||||
className={`w-20 h-20 rounded-full items-center justify-center ${
|
||||
!ws.isConnected ? 'opacity-50' : 'opacity-100'
|
||||
} ${
|
||||
isRecording
|
||||
? 'bg-red-500'
|
||||
: isInProgress
|
||||
? 'bg-red-600'
|
||||
: userInput.trim()
|
||||
? 'bg-blue-600'
|
||||
: 'bg-zinc-700'
|
||||
}`}
|
||||
>
|
||||
{isInProgress ? (
|
||||
<View className="relative w-6 h-6">
|
||||
<View className="absolute w-6 h-0.5 bg-white rotate-45" style={{top: 11}} />
|
||||
<View className="absolute w-6 h-0.5 bg-white -rotate-45" style={{top: 11}} />
|
||||
</View>
|
||||
) : isRecording ? (
|
||||
<View className="w-4 h-4 bg-white rounded-full" />
|
||||
) : userInput.trim() ? (
|
||||
<Text className="text-white text-xl">▶</Text>
|
||||
) : (
|
||||
<View className="w-6 h-8 relative">
|
||||
<View className="absolute bottom-0 left-1/2 -ml-2 w-4 h-6 bg-white rounded-t-full" />
|
||||
<View className="absolute bottom-0 left-1/2 -ml-3 w-6 h-1.5 bg-white rounded-full" />
|
||||
</View>
|
||||
)}
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Artifact drawer */}
|
||||
<ArtifactDrawer
|
||||
artifact={currentArtifact}
|
||||
onClose={handleCloseArtifact}
|
||||
/>
|
||||
</View>
|
||||
</KeyboardAvoidingView>
|
||||
);
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
import { Link } from 'expo-router';
|
||||
import { StyleSheet } from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
|
||||
export default function ModalScreen() {
|
||||
return (
|
||||
<ThemedView style={styles.container}>
|
||||
<ThemedText type="title">This is a modal</ThemedText>
|
||||
<Link href="/" dismissTo style={styles.link}>
|
||||
<ThemedText type="link">Go to home screen</ThemedText>
|
||||
</Link>
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
container: {
|
||||
flex: 1,
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
padding: 20,
|
||||
},
|
||||
link: {
|
||||
marginTop: 15,
|
||||
paddingVertical: 15,
|
||||
},
|
||||
});
|
||||
367
packages/voice-mobile/app/settings.tsx
Normal file
367
packages/voice-mobile/app/settings.tsx
Normal file
@@ -0,0 +1,367 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { View, Text, ScrollView, TextInput, Switch, Pressable, Alert, ActivityIndicator } from 'react-native';
|
||||
import { router } from 'expo-router';
|
||||
import { useSafeAreaInsets } from 'react-native-safe-area-context';
|
||||
import { useSettings } from '@/hooks/use-settings';
|
||||
|
||||
export default function SettingsScreen() {
|
||||
const { settings, isLoading, updateSettings, resetSettings } = useSettings();
|
||||
const insets = useSafeAreaInsets();
|
||||
|
||||
const [serverUrl, setServerUrl] = useState(settings.serverUrl);
|
||||
const [useSpeaker, setUseSpeaker] = useState(settings.useSpeaker);
|
||||
const [keepScreenOn, setKeepScreenOn] = useState(settings.keepScreenOn);
|
||||
const [theme, setTheme] = useState<'dark' | 'light' | 'auto'>(settings.theme);
|
||||
const [hasChanges, setHasChanges] = useState(false);
|
||||
const [isTesting, setIsTesting] = useState(false);
|
||||
const [testResult, setTestResult] = useState<{ success: boolean; message: string } | null>(null);
|
||||
|
||||
// Update local state when settings load
|
||||
useEffect(() => {
|
||||
setServerUrl(settings.serverUrl);
|
||||
setUseSpeaker(settings.useSpeaker);
|
||||
setKeepScreenOn(settings.keepScreenOn);
|
||||
setTheme(settings.theme);
|
||||
}, [settings]);
|
||||
|
||||
// Track changes
|
||||
useEffect(() => {
|
||||
const changed =
|
||||
serverUrl !== settings.serverUrl ||
|
||||
useSpeaker !== settings.useSpeaker ||
|
||||
keepScreenOn !== settings.keepScreenOn ||
|
||||
theme !== settings.theme;
|
||||
setHasChanges(changed);
|
||||
}, [serverUrl, useSpeaker, keepScreenOn, theme, settings]);
|
||||
|
||||
function validateServerUrl(url: string): boolean {
|
||||
try {
|
||||
const urlObj = new URL(url);
|
||||
return urlObj.protocol === 'ws:' || urlObj.protocol === 'wss:';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function handleSave() {
|
||||
// Validate server URL
|
||||
if (!validateServerUrl(serverUrl)) {
|
||||
Alert.alert(
|
||||
'Invalid URL',
|
||||
'Server URL must be a valid WebSocket URL (ws:// or wss://)',
|
||||
[{ text: 'OK' }]
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await updateSettings({
|
||||
serverUrl,
|
||||
useSpeaker,
|
||||
keepScreenOn,
|
||||
theme,
|
||||
});
|
||||
|
||||
Alert.alert(
|
||||
'Settings Saved',
|
||||
'Your settings have been saved successfully.',
|
||||
[
|
||||
{
|
||||
text: 'OK',
|
||||
onPress: () => router.back(),
|
||||
},
|
||||
]
|
||||
);
|
||||
} catch (error) {
|
||||
Alert.alert(
|
||||
'Error',
|
||||
'Failed to save settings. Please try again.',
|
||||
[{ text: 'OK' }]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleReset() {
|
||||
Alert.alert(
|
||||
'Reset Settings',
|
||||
'Are you sure you want to reset all settings to defaults?',
|
||||
[
|
||||
{ text: 'Cancel', style: 'cancel' },
|
||||
{
|
||||
text: 'Reset',
|
||||
style: 'destructive',
|
||||
onPress: async () => {
|
||||
try {
|
||||
await resetSettings();
|
||||
Alert.alert('Settings Reset', 'All settings have been reset to defaults.');
|
||||
} catch (error) {
|
||||
Alert.alert('Error', 'Failed to reset settings. Please try again.');
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
function handleCancel() {
|
||||
if (hasChanges) {
|
||||
Alert.alert(
|
||||
'Discard Changes',
|
||||
'You have unsaved changes. Are you sure you want to go back?',
|
||||
[
|
||||
{ text: 'Stay', style: 'cancel' },
|
||||
{
|
||||
text: 'Discard',
|
||||
style: 'destructive',
|
||||
onPress: () => router.back(),
|
||||
},
|
||||
]
|
||||
);
|
||||
} else {
|
||||
router.back();
|
||||
}
|
||||
}
|
||||
|
||||
async function handleTestConnection() {
|
||||
if (!validateServerUrl(serverUrl)) {
|
||||
Alert.alert(
|
||||
'Invalid URL',
|
||||
'Server URL must be a valid WebSocket URL (ws:// or wss://)',
|
||||
[{ text: 'OK' }]
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
setIsTesting(true);
|
||||
setTestResult(null);
|
||||
|
||||
try {
|
||||
const ws = new WebSocket(serverUrl);
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
ws.close();
|
||||
setTestResult({
|
||||
success: false,
|
||||
message: 'Connection timeout - server did not respond',
|
||||
});
|
||||
setIsTesting(false);
|
||||
}, 5000);
|
||||
|
||||
ws.onopen = () => {
|
||||
clearTimeout(timeout);
|
||||
ws.close();
|
||||
setTestResult({
|
||||
success: true,
|
||||
message: 'Connection successful',
|
||||
});
|
||||
setIsTesting(false);
|
||||
};
|
||||
|
||||
ws.onerror = () => {
|
||||
clearTimeout(timeout);
|
||||
setTestResult({
|
||||
success: false,
|
||||
message: 'Connection failed - check URL and network',
|
||||
});
|
||||
setIsTesting(false);
|
||||
};
|
||||
} catch (error) {
|
||||
setTestResult({
|
||||
success: false,
|
||||
message: 'Failed to create connection',
|
||||
});
|
||||
setIsTesting(false);
|
||||
}
|
||||
}
|
||||
|
||||
if (isLoading) {
|
||||
return (
|
||||
<View className="flex-1 bg-black items-center justify-center">
|
||||
<Text className="text-white text-lg">Loading settings...</Text>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<View className="flex-1 bg-black">
|
||||
{/* Header */}
|
||||
<View className="px-6 pb-4 border-b border-gray-800" style={{ paddingTop: insets.top + 16 }}>
|
||||
<View className="flex-row items-center justify-between">
|
||||
<Text className="text-white text-3xl font-bold">Settings</Text>
|
||||
<Pressable onPress={handleCancel}>
|
||||
<Text className="text-blue-500 text-base font-semibold">Cancel</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
<ScrollView className="flex-1">
|
||||
<View className="p-6">
|
||||
{/* Server Configuration */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-white text-lg font-semibold mb-4">
|
||||
Server Configuration
|
||||
</Text>
|
||||
|
||||
<Text className="text-gray-400 text-sm mb-2">WebSocket URL</Text>
|
||||
<TextInput
|
||||
className="bg-zinc-900 text-white p-4 rounded-lg mb-2"
|
||||
placeholder="wss://example.com/ws"
|
||||
placeholderTextColor="#6b7280"
|
||||
value={serverUrl}
|
||||
onChangeText={(text) => {
|
||||
setServerUrl(text);
|
||||
setTestResult(null);
|
||||
}}
|
||||
autoCapitalize="none"
|
||||
autoCorrect={false}
|
||||
keyboardType="url"
|
||||
/>
|
||||
<Text className="text-gray-500 text-xs mb-3">
|
||||
Must be a valid WebSocket URL (ws:// or wss://)
|
||||
</Text>
|
||||
|
||||
{/* Test Connection Button */}
|
||||
<Pressable
|
||||
onPress={handleTestConnection}
|
||||
disabled={isTesting || !validateServerUrl(serverUrl)}
|
||||
className={`p-3 rounded-lg mb-3 flex-row items-center justify-center ${
|
||||
isTesting || !validateServerUrl(serverUrl) ? 'bg-zinc-800' : 'bg-blue-600 active:bg-blue-700'
|
||||
}`}
|
||||
>
|
||||
{isTesting ? (
|
||||
<>
|
||||
<ActivityIndicator size="small" color="#fff" />
|
||||
<Text className="text-white font-semibold ml-2">Testing...</Text>
|
||||
</>
|
||||
) : (
|
||||
<Text className="text-white font-semibold">Test Connection</Text>
|
||||
)}
|
||||
</Pressable>
|
||||
|
||||
{/* Test Result */}
|
||||
{testResult && (
|
||||
<View
|
||||
className={`p-3 rounded-lg ${
|
||||
testResult.success
|
||||
? 'bg-green-900/30 border border-green-700'
|
||||
: 'bg-red-900/30 border border-red-700'
|
||||
}`}
|
||||
>
|
||||
<Text className={testResult.success ? 'text-green-400' : 'text-red-400'}>
|
||||
{testResult.message}
|
||||
</Text>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
|
||||
{/* Audio Settings */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-white text-lg font-semibold mb-4">
|
||||
Audio
|
||||
</Text>
|
||||
|
||||
<View className="bg-zinc-900 rounded-lg p-4 mb-3">
|
||||
<View className="flex-row justify-between items-center">
|
||||
<View className="flex-1">
|
||||
<Text className="text-white text-base mb-1">Use Speaker</Text>
|
||||
<Text className="text-gray-400 text-sm">
|
||||
Play audio through speaker instead of earpiece
|
||||
</Text>
|
||||
</View>
|
||||
<Switch
|
||||
value={useSpeaker}
|
||||
onValueChange={setUseSpeaker}
|
||||
trackColor={{ false: '#374151', true: '#3b82f6' }}
|
||||
thumbColor={useSpeaker ? '#60a5fa' : '#d1d5db'}
|
||||
/>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
<View className="bg-zinc-900 rounded-lg p-4">
|
||||
<View className="flex-row justify-between items-center">
|
||||
<View className="flex-1">
|
||||
<Text className="text-white text-base mb-1">Keep Screen On</Text>
|
||||
<Text className="text-gray-400 text-sm">
|
||||
Prevent screen from sleeping during voice sessions
|
||||
</Text>
|
||||
</View>
|
||||
<Switch
|
||||
value={keepScreenOn}
|
||||
onValueChange={setKeepScreenOn}
|
||||
trackColor={{ false: '#374151', true: '#3b82f6' }}
|
||||
thumbColor={keepScreenOn ? '#60a5fa' : '#d1d5db'}
|
||||
/>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Theme Settings */}
|
||||
<View className="mb-8">
|
||||
<Text className="text-white text-lg font-semibold mb-4">
|
||||
Theme
|
||||
</Text>
|
||||
|
||||
<View className="bg-zinc-900 rounded-lg p-4 opacity-50">
|
||||
<Text className="text-gray-400 text-sm mb-3">
|
||||
Theme selection (coming soon)
|
||||
</Text>
|
||||
|
||||
{(['dark', 'light', 'auto'] as const).map((themeOption) => (
|
||||
<Pressable
|
||||
key={themeOption}
|
||||
disabled
|
||||
className="flex-row items-center py-2"
|
||||
>
|
||||
<View className={`w-5 h-5 rounded-full border-2 ${
|
||||
theme === themeOption ? 'border-blue-500' : 'border-gray-600'
|
||||
} mr-3 items-center justify-center`}>
|
||||
{theme === themeOption && (
|
||||
<View className="w-3 h-3 rounded-full bg-blue-500" />
|
||||
)}
|
||||
</View>
|
||||
<Text className="text-gray-400 text-base capitalize">
|
||||
{themeOption}
|
||||
</Text>
|
||||
</Pressable>
|
||||
))}
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Action Buttons */}
|
||||
<View className="mb-8">
|
||||
<Pressable
|
||||
className={`p-4 rounded-lg mb-3 ${
|
||||
hasChanges ? 'bg-blue-500' : 'bg-blue-500/50'
|
||||
}`}
|
||||
onPress={handleSave}
|
||||
disabled={!hasChanges}
|
||||
>
|
||||
<Text className="text-white text-center text-base font-semibold">
|
||||
Save Settings
|
||||
</Text>
|
||||
</Pressable>
|
||||
|
||||
<Pressable
|
||||
className="p-4 rounded-lg border border-red-500/30"
|
||||
onPress={handleReset}
|
||||
>
|
||||
<Text className="text-red-500 text-center text-base font-semibold">
|
||||
Reset to Defaults
|
||||
</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
|
||||
{/* App Info */}
|
||||
<View className="border-t border-gray-800 pt-6">
|
||||
<Text className="text-gray-500 text-sm text-center">
|
||||
Voice Assistant Mobile
|
||||
</Text>
|
||||
<Text className="text-gray-600 text-xs text-center mt-1">
|
||||
Version 1.0.0
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
</ScrollView>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
169
packages/voice-mobile/components/active-processes.tsx
Normal file
169
packages/voice-mobile/components/active-processes.tsx
Normal file
@@ -0,0 +1,169 @@
|
||||
import { View, Text, Pressable, ScrollView } from 'react-native';
|
||||
import type { AgentStatus } from '@voice-assistant/server/acp/types';
|
||||
|
||||
export interface ActiveProcessesProps {
|
||||
agents: Array<{
|
||||
id: string;
|
||||
status: AgentStatus;
|
||||
type: 'claude';
|
||||
currentModeId?: string;
|
||||
availableModes?: Array<{ id: string; name: string; description?: string | null }>;
|
||||
}>;
|
||||
commands: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
workingDirectory: string;
|
||||
currentCommand: string;
|
||||
isDead: boolean;
|
||||
exitCode: number | null;
|
||||
}>;
|
||||
activeProcessId: string | null;
|
||||
activeProcessType: 'agent' | null;
|
||||
onSelectAgent: (id: string) => void;
|
||||
onBackToOrchestrator: () => void;
|
||||
}
|
||||
|
||||
function getAgentStatusColor(status: AgentStatus): string {
|
||||
switch (status) {
|
||||
case 'initializing':
|
||||
return '#f59e0b'; // orange
|
||||
case 'ready':
|
||||
return '#3b82f6'; // blue
|
||||
case 'processing':
|
||||
return '#fbbf24'; // yellow
|
||||
case 'completed':
|
||||
return '#22c55e'; // green
|
||||
case 'failed':
|
||||
return '#ef4444'; // red
|
||||
case 'killed':
|
||||
return '#9ca3af'; // gray
|
||||
default:
|
||||
return '#9ca3af';
|
||||
}
|
||||
}
|
||||
|
||||
function getModeName(modeId?: string, availableModes?: Array<{ id: string; name: string }>): string {
|
||||
if (!modeId) return 'unknown';
|
||||
const mode = availableModes?.find((m) => m.id === modeId);
|
||||
return mode?.name || modeId;
|
||||
}
|
||||
|
||||
function getModeColor(modeId?: string): string {
|
||||
if (!modeId) return '#9ca3af'; // gray
|
||||
|
||||
// Color based on common mode types
|
||||
if (modeId.includes('ask')) return '#f59e0b'; // orange - asks permission
|
||||
if (modeId.includes('code')) return '#22c55e'; // green - writes code
|
||||
if (modeId.includes('architect') || modeId.includes('plan')) return '#3b82f6'; // blue - plans
|
||||
|
||||
return '#9ca3af'; // gray - unknown
|
||||
}
|
||||
|
||||
export function ActiveProcesses({
|
||||
agents,
|
||||
commands,
|
||||
activeProcessId,
|
||||
activeProcessType,
|
||||
onSelectAgent,
|
||||
onBackToOrchestrator,
|
||||
}: ActiveProcessesProps) {
|
||||
const hasActiveProcess = activeProcessId !== null && activeProcessType !== null;
|
||||
|
||||
if (agents.length === 0 && commands.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<View className="bg-zinc-900 border-b border-zinc-800">
|
||||
{/* Header with Back button */}
|
||||
{hasActiveProcess && (
|
||||
<View className="px-4 py-3 border-b border-zinc-800">
|
||||
<Pressable
|
||||
onPress={onBackToOrchestrator}
|
||||
className="bg-zinc-800 px-4 py-2 rounded-lg active:bg-zinc-700"
|
||||
>
|
||||
<Text className="text-white text-sm font-semibold text-center">
|
||||
Back to Chat
|
||||
</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
)}
|
||||
|
||||
{/* Scrollable process list */}
|
||||
<ScrollView
|
||||
horizontal
|
||||
showsHorizontalScrollIndicator={false}
|
||||
className="px-4 py-3"
|
||||
contentContainerStyle={{ gap: 8 }}
|
||||
>
|
||||
{/* Agents */}
|
||||
{agents.map((agent) => {
|
||||
const modeName = getModeName(agent.currentModeId, agent.availableModes);
|
||||
const isActive = activeProcessType === 'agent' && activeProcessId === agent.id;
|
||||
|
||||
return (
|
||||
<Pressable
|
||||
key={`agent-${agent.id}`}
|
||||
onPress={() => onSelectAgent(agent.id)}
|
||||
className={`flex-row items-center gap-2 px-3 py-2 rounded-lg ${
|
||||
isActive ? 'bg-blue-600' : 'bg-zinc-800'
|
||||
} active:opacity-70`}
|
||||
>
|
||||
{/* Agent icon */}
|
||||
<View className="w-3 h-3 rounded-full bg-blue-500" />
|
||||
|
||||
{/* Agent ID (shortened) */}
|
||||
<Text className="text-white text-xs font-medium">
|
||||
{agent.id.substring(0, 8)}
|
||||
</Text>
|
||||
|
||||
{/* Status indicator */}
|
||||
<View
|
||||
className="w-2 h-2 rounded-full"
|
||||
style={{ backgroundColor: getAgentStatusColor(agent.status) }}
|
||||
/>
|
||||
|
||||
{/* Mode indicator */}
|
||||
{agent.currentModeId && (
|
||||
<View
|
||||
className="w-1.5 h-1.5 rounded-full opacity-30"
|
||||
style={{ backgroundColor: getModeColor(agent.currentModeId) }}
|
||||
/>
|
||||
)}
|
||||
</Pressable>
|
||||
);
|
||||
})}
|
||||
|
||||
{/* Commands */}
|
||||
{commands.map((command) => {
|
||||
const statusColor = command.isDead
|
||||
? command.exitCode === 0
|
||||
? '#22c55e'
|
||||
: '#ef4444'
|
||||
: '#3b82f6';
|
||||
|
||||
return (
|
||||
<View
|
||||
key={`command-${command.id}`}
|
||||
className="flex-row items-center gap-2 px-3 py-2 rounded-lg bg-zinc-800"
|
||||
>
|
||||
{/* Command icon */}
|
||||
<View className="w-3 h-3 rounded-sm bg-purple-500" />
|
||||
|
||||
{/* Command name */}
|
||||
<Text className="text-white text-xs font-medium" numberOfLines={1}>
|
||||
{command.name || command.currentCommand.substring(0, 20)}
|
||||
</Text>
|
||||
|
||||
{/* Status indicator */}
|
||||
<View
|
||||
className="w-2 h-2 rounded-full"
|
||||
style={{ backgroundColor: statusColor }}
|
||||
/>
|
||||
</View>
|
||||
);
|
||||
})}
|
||||
</ScrollView>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
286
packages/voice-mobile/components/agent-stream-view.tsx
Normal file
286
packages/voice-mobile/components/agent-stream-view.tsx
Normal file
@@ -0,0 +1,286 @@
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import { View, Text, ScrollView, Pressable } from 'react-native';
|
||||
import { useSafeAreaInsets } from 'react-native-safe-area-context';
|
||||
import type { AgentStatus } from '@voice-assistant/server/acp/types';
|
||||
import type { SessionNotification } from '@agentclientprotocol/sdk';
|
||||
|
||||
export interface AgentStreamViewProps {
|
||||
agentId: string;
|
||||
agent: {
|
||||
id: string;
|
||||
status: AgentStatus;
|
||||
createdAt: Date;
|
||||
type: 'claude';
|
||||
};
|
||||
updates: Array<{
|
||||
timestamp: Date;
|
||||
notification: SessionNotification;
|
||||
}>;
|
||||
onBack: () => void;
|
||||
onKillAgent: (agentId: string) => void;
|
||||
onCancelAgent: (agentId: string) => void;
|
||||
}
|
||||
|
||||
function getStatusColor(status: AgentStatus): string {
|
||||
switch (status) {
|
||||
case 'initializing':
|
||||
return '#f59e0b';
|
||||
case 'ready':
|
||||
return '#3b82f6';
|
||||
case 'processing':
|
||||
return '#fbbf24';
|
||||
case 'completed':
|
||||
return '#22c55e';
|
||||
case 'failed':
|
||||
return '#ef4444';
|
||||
case 'killed':
|
||||
return '#9ca3af';
|
||||
default:
|
||||
return '#9ca3af';
|
||||
}
|
||||
}
|
||||
|
||||
function getStatusIcon(status: AgentStatus): string {
|
||||
switch (status) {
|
||||
case 'initializing':
|
||||
case 'processing':
|
||||
return '⏳';
|
||||
case 'ready':
|
||||
case 'completed':
|
||||
return '✓';
|
||||
case 'failed':
|
||||
case 'killed':
|
||||
return '✗';
|
||||
default:
|
||||
return '•';
|
||||
}
|
||||
}
|
||||
|
||||
function formatTimestamp(date: Date): string {
|
||||
return new Intl.DateTimeFormat('en-US', {
|
||||
hour: 'numeric',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
hour12: true,
|
||||
}).format(date);
|
||||
}
|
||||
|
||||
function AgentUpdate({
|
||||
update,
|
||||
}: {
|
||||
update: { timestamp: Date; notification: SessionNotification };
|
||||
}) {
|
||||
const [isExpanded, setIsExpanded] = useState(true);
|
||||
const notification = update.notification as any;
|
||||
|
||||
// Parse different notification types
|
||||
if (notification.type === 'sessionUpdate') {
|
||||
const sessionUpdate = notification.sessionUpdate || {};
|
||||
|
||||
// Handle message chunks
|
||||
if (sessionUpdate.kind === 'agent_message_chunk') {
|
||||
return (
|
||||
<View className="bg-zinc-800 rounded-lg p-3 mb-2">
|
||||
<Text className="text-zinc-500 text-xs mb-1">
|
||||
{formatTimestamp(update.timestamp)}
|
||||
</Text>
|
||||
<Text className="text-white text-sm">{sessionUpdate.chunk || ''}</Text>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
// Handle tool calls
|
||||
if (sessionUpdate.kind === 'tool_call') {
|
||||
return (
|
||||
<View className="bg-zinc-800 rounded-lg mb-2 overflow-hidden">
|
||||
<Pressable
|
||||
onPress={() => setIsExpanded(!isExpanded)}
|
||||
className="px-3 py-2 active:bg-zinc-700"
|
||||
>
|
||||
<Text className="text-zinc-500 text-xs mb-1">
|
||||
{formatTimestamp(update.timestamp)}
|
||||
</Text>
|
||||
<Text className="text-blue-400 text-sm font-medium">
|
||||
Tool Call: {sessionUpdate.toolName || 'unknown'}
|
||||
</Text>
|
||||
</Pressable>
|
||||
{isExpanded && sessionUpdate.arguments && (
|
||||
<View className="px-3 pb-3 border-t border-zinc-700">
|
||||
<Text className="text-zinc-400 text-xs font-mono mt-2">
|
||||
{JSON.stringify(sessionUpdate.arguments, null, 2)}
|
||||
</Text>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
// Handle tool call updates
|
||||
if (sessionUpdate.kind === 'tool_call_update') {
|
||||
return (
|
||||
<View className="bg-zinc-800 rounded-lg p-3 mb-2">
|
||||
<Text className="text-zinc-500 text-xs mb-1">
|
||||
{formatTimestamp(update.timestamp)}
|
||||
</Text>
|
||||
<Text className="text-yellow-400 text-sm">
|
||||
{sessionUpdate.status || 'updating'}
|
||||
</Text>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
// Handle available commands
|
||||
if (sessionUpdate.kind === 'available_commands_update') {
|
||||
const commands = sessionUpdate.commands || [];
|
||||
return (
|
||||
<View className="bg-zinc-800 rounded-lg mb-2 overflow-hidden">
|
||||
<Pressable
|
||||
onPress={() => setIsExpanded(!isExpanded)}
|
||||
className="px-3 py-2 active:bg-zinc-700"
|
||||
>
|
||||
<Text className="text-zinc-500 text-xs mb-1">
|
||||
{formatTimestamp(update.timestamp)}
|
||||
</Text>
|
||||
<Text className="text-green-400 text-sm font-medium">
|
||||
Available Commands ({commands.length})
|
||||
</Text>
|
||||
</Pressable>
|
||||
{isExpanded && (
|
||||
<View className="px-3 pb-3 border-t border-zinc-700">
|
||||
{commands.map((cmd: any, idx: number) => (
|
||||
<Text key={idx} className="text-zinc-400 text-xs mt-1">
|
||||
• {cmd.name || cmd}
|
||||
</Text>
|
||||
))}
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
// Generic session update
|
||||
return (
|
||||
<View className="bg-zinc-800 rounded-lg p-3 mb-2">
|
||||
<Text className="text-zinc-500 text-xs mb-1">
|
||||
{formatTimestamp(update.timestamp)}
|
||||
</Text>
|
||||
<Text className="text-zinc-400 text-xs font-mono">
|
||||
{JSON.stringify(sessionUpdate, null, 2)}
|
||||
</Text>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
// Fallback for unknown notification types
|
||||
return (
|
||||
<View className="bg-zinc-800 rounded-lg p-3 mb-2">
|
||||
<Text className="text-zinc-500 text-xs mb-1">
|
||||
{formatTimestamp(update.timestamp)}
|
||||
</Text>
|
||||
<Text className="text-zinc-400 text-xs font-mono">
|
||||
{JSON.stringify(notification, null, 2)}
|
||||
</Text>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
export function AgentStreamView({
|
||||
agentId,
|
||||
agent,
|
||||
updates,
|
||||
onBack,
|
||||
onKillAgent,
|
||||
onCancelAgent,
|
||||
}: AgentStreamViewProps) {
|
||||
const scrollViewRef = useRef<ScrollView>(null);
|
||||
const insets = useSafeAreaInsets();
|
||||
|
||||
// Auto-scroll to bottom when new updates arrive
|
||||
useEffect(() => {
|
||||
scrollViewRef.current?.scrollToEnd({ animated: true });
|
||||
}, [updates]);
|
||||
|
||||
const canCancel = agent.status === 'processing';
|
||||
const canKill = agent.status !== 'killed' && agent.status !== 'completed';
|
||||
|
||||
return (
|
||||
<View className="flex-1 bg-black">
|
||||
{/* Header */}
|
||||
<View className="bg-zinc-900 border-b border-zinc-800 px-4 pb-4" style={{ paddingTop: insets.top + 16 }}>
|
||||
{/* Back button */}
|
||||
<Pressable
|
||||
onPress={onBack}
|
||||
className="bg-zinc-800 px-4 py-2 rounded-lg mb-4 active:bg-zinc-700"
|
||||
>
|
||||
<Text className="text-white text-sm font-semibold text-center">
|
||||
← Back to Chat
|
||||
</Text>
|
||||
</Pressable>
|
||||
|
||||
{/* Agent info */}
|
||||
<View className="flex-row items-center justify-between mb-3">
|
||||
<View className="flex-row items-center gap-2">
|
||||
<Text className="text-zinc-400 text-sm">Agent:</Text>
|
||||
<Text className="text-white text-sm font-mono">
|
||||
{agentId.substring(0, 8)}
|
||||
</Text>
|
||||
</View>
|
||||
<View
|
||||
className="flex-row items-center gap-2 px-3 py-1 rounded-full"
|
||||
style={{ backgroundColor: getStatusColor(agent.status) }}
|
||||
>
|
||||
<Text className="text-white text-xs">{getStatusIcon(agent.status)}</Text>
|
||||
<Text className="text-white text-xs font-medium">{agent.status}</Text>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
<Text className="text-zinc-500 text-xs">
|
||||
Created: {formatTimestamp(agent.createdAt)}
|
||||
</Text>
|
||||
|
||||
{/* Control buttons */}
|
||||
{(canCancel || canKill) && (
|
||||
<View className="flex-row gap-2 mt-4">
|
||||
{canCancel && (
|
||||
<Pressable
|
||||
onPress={() => onCancelAgent(agentId)}
|
||||
className="flex-1 bg-orange-600 px-4 py-2 rounded-lg active:bg-orange-700"
|
||||
>
|
||||
<Text className="text-white text-sm font-semibold text-center">
|
||||
Cancel
|
||||
</Text>
|
||||
</Pressable>
|
||||
)}
|
||||
{canKill && (
|
||||
<Pressable
|
||||
onPress={() => onKillAgent(agentId)}
|
||||
className="flex-1 bg-red-600 px-4 py-2 rounded-lg active:bg-red-700"
|
||||
>
|
||||
<Text className="text-white text-sm font-semibold text-center">
|
||||
Kill
|
||||
</Text>
|
||||
</Pressable>
|
||||
)}
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
|
||||
{/* Updates list */}
|
||||
<ScrollView
|
||||
ref={scrollViewRef}
|
||||
className="flex-1 px-4"
|
||||
contentContainerStyle={{ paddingTop: 16, paddingBottom: Math.max(insets.bottom, 32) }}
|
||||
>
|
||||
{updates.length === 0 ? (
|
||||
<View className="flex-1 items-center justify-center py-12">
|
||||
<Text className="text-zinc-500 text-sm text-center">
|
||||
No updates yet.{'\n'}Waiting for agent activity...
|
||||
</Text>
|
||||
</View>
|
||||
) : (
|
||||
updates.map((update, idx) => <AgentUpdate key={idx} update={update} />)
|
||||
)}
|
||||
</ScrollView>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
126
packages/voice-mobile/components/artifact-drawer.tsx
Normal file
126
packages/voice-mobile/components/artifact-drawer.tsx
Normal file
@@ -0,0 +1,126 @@
|
||||
import { View, Text, ScrollView, Pressable, Modal } from 'react-native';
|
||||
import { useEffect } from 'react';
|
||||
|
||||
export interface Artifact {
|
||||
id: string;
|
||||
type: 'markdown' | 'diff' | 'image' | 'code';
|
||||
title: string;
|
||||
content: string;
|
||||
isBase64: boolean;
|
||||
}
|
||||
|
||||
interface ArtifactDrawerProps {
|
||||
artifact: Artifact | null;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) {
|
||||
useEffect(() => {
|
||||
if (!artifact) return;
|
||||
console.log('[ArtifactDrawer] Showing artifact:', artifact.id, artifact.type, artifact.title);
|
||||
}, [artifact]);
|
||||
|
||||
if (!artifact) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Decode content if base64
|
||||
const content = artifact.isBase64 ? atob(artifact.content) : artifact.content;
|
||||
|
||||
// Type badge colors
|
||||
const typeColors = {
|
||||
markdown: 'bg-blue-600',
|
||||
diff: 'bg-purple-600',
|
||||
image: 'bg-green-600',
|
||||
code: 'bg-orange-600',
|
||||
};
|
||||
|
||||
return (
|
||||
<Modal
|
||||
visible={!!artifact}
|
||||
animationType="slide"
|
||||
presentationStyle="pageSheet"
|
||||
onRequestClose={onClose}
|
||||
>
|
||||
<View className="flex-1 bg-black">
|
||||
{/* Header */}
|
||||
<View className="pt-16 pb-4 px-4 border-b border-zinc-800">
|
||||
<View className="flex-row items-center justify-between">
|
||||
<View className="flex-1 mr-4">
|
||||
<Text className="text-white text-xl font-bold" numberOfLines={2}>
|
||||
{artifact.title}
|
||||
</Text>
|
||||
</View>
|
||||
<View className="flex-row items-center gap-2">
|
||||
<View className={`${typeColors[artifact.type]} px-3 py-1 rounded-full`}>
|
||||
<Text className="text-white text-xs font-semibold uppercase">
|
||||
{artifact.type}
|
||||
</Text>
|
||||
</View>
|
||||
<Pressable
|
||||
onPress={onClose}
|
||||
className="bg-zinc-800 w-10 h-10 rounded-full items-center justify-center"
|
||||
>
|
||||
<Text className="text-white text-xl font-bold">×</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{/* Content */}
|
||||
<ScrollView className="flex-1" contentContainerClassName="p-4">
|
||||
{artifact.type === 'image' ? (
|
||||
<View className="items-center justify-center">
|
||||
<Text className="text-zinc-400 text-sm">
|
||||
Image viewing not yet implemented
|
||||
</Text>
|
||||
<Text className="text-zinc-600 text-xs mt-2">
|
||||
Base64 image data received
|
||||
</Text>
|
||||
</View>
|
||||
) : (
|
||||
<View className="bg-zinc-900 rounded-lg p-4 border border-zinc-800">
|
||||
<ScrollView horizontal showsHorizontalScrollIndicator={true}>
|
||||
<Text
|
||||
className="text-zinc-300 text-sm font-mono"
|
||||
style={{ fontFamily: 'monospace' }}
|
||||
>
|
||||
{content}
|
||||
</Text>
|
||||
</ScrollView>
|
||||
</View>
|
||||
)}
|
||||
|
||||
{/* Metadata */}
|
||||
<View className="mt-4 bg-zinc-900 rounded-lg p-3 border border-zinc-800">
|
||||
<Text className="text-zinc-500 text-xs font-semibold mb-2">METADATA</Text>
|
||||
<View className="space-y-1">
|
||||
<View className="flex-row">
|
||||
<Text className="text-zinc-500 text-xs w-20">ID:</Text>
|
||||
<Text className="text-zinc-400 text-xs flex-1 font-mono">
|
||||
{artifact.id}
|
||||
</Text>
|
||||
</View>
|
||||
<View className="flex-row">
|
||||
<Text className="text-zinc-500 text-xs w-20">Type:</Text>
|
||||
<Text className="text-zinc-400 text-xs">{artifact.type}</Text>
|
||||
</View>
|
||||
<View className="flex-row">
|
||||
<Text className="text-zinc-500 text-xs w-20">Encoding:</Text>
|
||||
<Text className="text-zinc-400 text-xs">
|
||||
{artifact.isBase64 ? 'Base64' : 'Plain text'}
|
||||
</Text>
|
||||
</View>
|
||||
<View className="flex-row">
|
||||
<Text className="text-zinc-500 text-xs w-20">Size:</Text>
|
||||
<Text className="text-zinc-400 text-xs">
|
||||
{content.length.toLocaleString()} characters
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
</ScrollView>
|
||||
</View>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
18
packages/voice-mobile/components/connection-status.tsx
Normal file
18
packages/voice-mobile/components/connection-status.tsx
Normal file
@@ -0,0 +1,18 @@
|
||||
import { View, Text } from 'react-native';
|
||||
|
||||
interface ConnectionStatusProps {
|
||||
isConnected: boolean;
|
||||
}
|
||||
|
||||
export function ConnectionStatus({ isConnected }: ConnectionStatusProps) {
|
||||
return (
|
||||
<View className="px-4 py-3 border-b border-zinc-800">
|
||||
<View className="flex-row items-center">
|
||||
<View className={`w-2 h-2 rounded-full mr-2 ${isConnected ? 'bg-green-500' : 'bg-red-500'}`} />
|
||||
<Text className={isConnected ? 'text-green-500 text-sm' : 'text-red-500 text-sm'}>
|
||||
{isConnected ? 'Connected' : 'Disconnected'}
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
270
packages/voice-mobile/components/conversation-selector.tsx
Normal file
270
packages/voice-mobile/components/conversation-selector.tsx
Normal file
@@ -0,0 +1,270 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { Modal, View, Text, Pressable, ScrollView, Alert } from 'react-native';
|
||||
import { MaterialIcons } from '@expo/vector-icons';
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage';
|
||||
import type { UseWebSocketReturn } from '../hooks/use-websocket';
|
||||
|
||||
const STORAGE_KEY = '@voice-assistant:conversation-id';
|
||||
|
||||
interface Conversation {
|
||||
id: string;
|
||||
lastUpdated: string;
|
||||
messageCount: number;
|
||||
}
|
||||
|
||||
interface ConversationSelectorProps {
|
||||
currentConversationId: string | null;
|
||||
onSelectConversation: (conversationId: string | null) => void;
|
||||
websocket: UseWebSocketReturn;
|
||||
}
|
||||
|
||||
export function ConversationSelector({
|
||||
currentConversationId,
|
||||
onSelectConversation,
|
||||
websocket,
|
||||
}: ConversationSelectorProps) {
|
||||
const [conversations, setConversations] = useState<Conversation[]>([]);
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
|
||||
// Listen for conversation list responses
|
||||
useEffect(() => {
|
||||
const unsubscribe = websocket.on('list_conversations_response', (message) => {
|
||||
console.log('[ConversationSelector] Received conversations:', message.payload.conversations.length);
|
||||
setConversations(message.payload.conversations);
|
||||
setIsLoading(false);
|
||||
});
|
||||
|
||||
return unsubscribe;
|
||||
}, [websocket]);
|
||||
|
||||
// Listen for delete conversation responses
|
||||
useEffect(() => {
|
||||
const unsubscribe = websocket.on('delete_conversation_response', (message) => {
|
||||
console.log('[ConversationSelector] Delete response:', message.payload);
|
||||
if (message.payload.success) {
|
||||
// Refresh conversations list
|
||||
fetchConversations();
|
||||
|
||||
// If we deleted the current conversation, start a new one
|
||||
if (message.payload.conversationId === currentConversationId) {
|
||||
handleNewConversation();
|
||||
}
|
||||
} else {
|
||||
Alert.alert('Error', `Failed to delete conversation: ${message.payload.error}`);
|
||||
}
|
||||
});
|
||||
|
||||
return unsubscribe;
|
||||
}, [websocket, currentConversationId]);
|
||||
|
||||
// Fetch conversations when modal opens
|
||||
useEffect(() => {
|
||||
if (isOpen) {
|
||||
fetchConversations();
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
function fetchConversations() {
|
||||
setIsLoading(true);
|
||||
console.log('[ConversationSelector] Requesting conversations via WebSocket');
|
||||
websocket.send({
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'list_conversations_request',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function handleDeleteConversation(id: string) {
|
||||
Alert.alert(
|
||||
'Delete Conversation',
|
||||
'Are you sure you want to delete this conversation?',
|
||||
[
|
||||
{ text: 'Cancel', style: 'cancel' },
|
||||
{
|
||||
text: 'Delete',
|
||||
style: 'destructive',
|
||||
onPress: () => {
|
||||
console.log('[ConversationSelector] Deleting conversation via WebSocket:', id);
|
||||
websocket.send({
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'delete_conversation_request',
|
||||
conversationId: id,
|
||||
},
|
||||
});
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
function handleClearAll() {
|
||||
Alert.alert(
|
||||
'Clear All Conversations',
|
||||
'Are you sure you want to delete all conversations?',
|
||||
[
|
||||
{ text: 'Cancel', style: 'cancel' },
|
||||
{
|
||||
text: 'Clear All',
|
||||
style: 'destructive',
|
||||
onPress: () => {
|
||||
console.log('[ConversationSelector] Clearing all conversations via WebSocket');
|
||||
// Delete all conversations
|
||||
conversations.forEach((conv) => {
|
||||
websocket.send({
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'delete_conversation_request',
|
||||
conversationId: conv.id,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
// Clear local state
|
||||
setConversations([]);
|
||||
|
||||
// Start new conversation
|
||||
handleNewConversation();
|
||||
setIsOpen(false);
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
async function handleSelectConversation(id: string) {
|
||||
try {
|
||||
// Save to AsyncStorage for persistence
|
||||
await AsyncStorage.setItem(STORAGE_KEY, id);
|
||||
onSelectConversation(id);
|
||||
setIsOpen(false);
|
||||
} catch (error) {
|
||||
console.error('[ConversationSelector] Failed to save conversation ID:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleNewConversation() {
|
||||
try {
|
||||
// Clear saved conversation ID
|
||||
await AsyncStorage.removeItem(STORAGE_KEY);
|
||||
onSelectConversation(null);
|
||||
setIsOpen(false);
|
||||
} catch (error) {
|
||||
console.error('[ConversationSelector] Failed to clear conversation ID:', error);
|
||||
}
|
||||
}
|
||||
|
||||
function formatDate(dateString: string): string {
|
||||
const date = new Date(dateString);
|
||||
const now = new Date();
|
||||
const diffMs = now.getTime() - date.getTime();
|
||||
const diffMins = Math.floor(diffMs / 60000);
|
||||
const diffHours = Math.floor(diffMs / 3600000);
|
||||
const diffDays = Math.floor(diffMs / 86400000);
|
||||
|
||||
if (diffMins < 1) return 'Just now';
|
||||
if (diffMins < 60) return `${diffMins}m ago`;
|
||||
if (diffHours < 24) return `${diffHours}h ago`;
|
||||
if (diffDays < 7) return `${diffDays}d ago`;
|
||||
return date.toLocaleDateString();
|
||||
}
|
||||
|
||||
return (
|
||||
<View>
|
||||
<Pressable
|
||||
onPress={() => setIsOpen(true)}
|
||||
className="bg-zinc-800 px-4 py-2 rounded-lg"
|
||||
>
|
||||
<MaterialIcons name="chat-bubble-outline" size={20} color="white" />
|
||||
</Pressable>
|
||||
|
||||
<Modal
|
||||
visible={isOpen}
|
||||
animationType="slide"
|
||||
transparent={true}
|
||||
onRequestClose={() => setIsOpen(false)}
|
||||
>
|
||||
<View className="flex-1 bg-black/50">
|
||||
<Pressable
|
||||
className="flex-1"
|
||||
onPress={() => setIsOpen(false)}
|
||||
/>
|
||||
<View className="bg-zinc-900 rounded-t-3xl max-h-[80%]">
|
||||
{/* Header */}
|
||||
<View className="flex-row items-center justify-between p-6 border-b border-zinc-800">
|
||||
<Text className="text-white text-lg font-semibold">Conversations</Text>
|
||||
<Pressable onPress={() => setIsOpen(false)}>
|
||||
<MaterialIcons name="close" size={24} color="white" />
|
||||
</Pressable>
|
||||
</View>
|
||||
|
||||
{/* New Conversation Button */}
|
||||
<View className="p-4 border-b border-zinc-800">
|
||||
<Pressable
|
||||
onPress={handleNewConversation}
|
||||
className="flex-row items-center justify-center gap-2 bg-zinc-800 py-3 rounded-lg"
|
||||
>
|
||||
<MaterialIcons name="add" size={20} color="white" />
|
||||
<Text className="text-white font-semibold">New Conversation</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
|
||||
{/* Conversations List */}
|
||||
<ScrollView className="flex-1">
|
||||
{isLoading ? (
|
||||
<View className="p-8 items-center">
|
||||
<Text className="text-zinc-400">Loading...</Text>
|
||||
</View>
|
||||
) : conversations.length > 0 ? (
|
||||
<>
|
||||
{conversations.map((conversation) => (
|
||||
<Pressable
|
||||
key={conversation.id}
|
||||
onPress={() => handleSelectConversation(conversation.id)}
|
||||
className={`flex-row items-center justify-between p-4 border-b border-zinc-800 ${
|
||||
conversation.id === currentConversationId ? 'bg-zinc-800' : ''
|
||||
}`}
|
||||
>
|
||||
<View className="flex-1">
|
||||
<Text className="text-white font-semibold">
|
||||
{conversation.messageCount} messages
|
||||
</Text>
|
||||
<Text className="text-zinc-400 text-sm mt-1">
|
||||
{formatDate(conversation.lastUpdated)}
|
||||
</Text>
|
||||
</View>
|
||||
<Pressable
|
||||
onPress={() => handleDeleteConversation(conversation.id)}
|
||||
className="p-2"
|
||||
hitSlop={{ top: 10, bottom: 10, left: 10, right: 10 }}
|
||||
>
|
||||
<MaterialIcons name="delete-outline" size={20} color="#ef4444" />
|
||||
</Pressable>
|
||||
</Pressable>
|
||||
))}
|
||||
|
||||
{/* Clear All Button */}
|
||||
<View className="p-4">
|
||||
<Pressable
|
||||
onPress={handleClearAll}
|
||||
className="flex-row items-center justify-center gap-2 bg-red-900/20 py-3 rounded-lg border border-red-900"
|
||||
>
|
||||
<MaterialIcons name="delete-outline" size={18} color="#ef4444" />
|
||||
<Text className="text-red-500 font-semibold">Clear All</Text>
|
||||
</Pressable>
|
||||
</View>
|
||||
</>
|
||||
) : (
|
||||
<View className="p-8 items-center">
|
||||
<Text className="text-zinc-400">No saved conversations</Text>
|
||||
</View>
|
||||
)}
|
||||
</ScrollView>
|
||||
</View>
|
||||
</View>
|
||||
</Modal>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import { Href, Link } from 'expo-router';
|
||||
import { openBrowserAsync, WebBrowserPresentationStyle } from 'expo-web-browser';
|
||||
import { type ComponentProps } from 'react';
|
||||
|
||||
type Props = Omit<ComponentProps<typeof Link>, 'href'> & { href: Href & string };
|
||||
|
||||
export function ExternalLink({ href, ...rest }: Props) {
|
||||
return (
|
||||
<Link
|
||||
target="_blank"
|
||||
{...rest}
|
||||
href={href}
|
||||
onPress={async (event) => {
|
||||
if (process.env.EXPO_OS !== 'web') {
|
||||
// Prevent the default behavior of linking to the default browser on native.
|
||||
event.preventDefault();
|
||||
// Open the link in an in-app browser.
|
||||
await openBrowserAsync(href, {
|
||||
presentationStyle: WebBrowserPresentationStyle.AUTOMATIC,
|
||||
});
|
||||
}
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
import { BottomTabBarButtonProps } from '@react-navigation/bottom-tabs';
|
||||
import { PlatformPressable } from '@react-navigation/elements';
|
||||
import * as Haptics from 'expo-haptics';
|
||||
|
||||
export function HapticTab(props: BottomTabBarButtonProps) {
|
||||
return (
|
||||
<PlatformPressable
|
||||
{...props}
|
||||
onPressIn={(ev) => {
|
||||
if (process.env.EXPO_OS === 'ios') {
|
||||
// Add a soft haptic feedback when pressing down on the tabs.
|
||||
Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Light);
|
||||
}
|
||||
props.onPressIn?.(ev);
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import Animated from 'react-native-reanimated';
|
||||
|
||||
export function HelloWave() {
|
||||
return (
|
||||
<Animated.Text
|
||||
style={{
|
||||
fontSize: 28,
|
||||
lineHeight: 32,
|
||||
marginTop: -6,
|
||||
animationName: {
|
||||
'50%': { transform: [{ rotate: '25deg' }] },
|
||||
},
|
||||
animationIterationCount: 4,
|
||||
animationDuration: '300ms',
|
||||
}}>
|
||||
👋
|
||||
</Animated.Text>
|
||||
);
|
||||
}
|
||||
288
packages/voice-mobile/components/message.tsx
Normal file
288
packages/voice-mobile/components/message.tsx
Normal file
@@ -0,0 +1,288 @@
|
||||
import { View, Text, Pressable, Animated } from 'react-native';
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
|
||||
interface UserMessageProps {
|
||||
message: string;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export function UserMessage({ message, timestamp }: UserMessageProps) {
|
||||
const time = new Date(timestamp).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
|
||||
|
||||
return (
|
||||
<View className="flex-row justify-end mb-3 px-4">
|
||||
<View className="bg-blue-600 rounded-2xl rounded-tr-sm px-4 py-3 max-w-[80%]">
|
||||
<Text className="text-white text-base leading-5">{message}</Text>
|
||||
<Text className="text-blue-200 text-xs mt-1">{time}</Text>
|
||||
</View>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
interface AssistantMessageProps {
|
||||
message: string;
|
||||
timestamp: number;
|
||||
isStreaming?: boolean;
|
||||
}
|
||||
|
||||
export function AssistantMessage({ message, timestamp, isStreaming = false }: AssistantMessageProps) {
|
||||
const time = new Date(timestamp).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
|
||||
const fadeAnim = useRef(new Animated.Value(0.3)).current;
|
||||
|
||||
useEffect(() => {
|
||||
if (isStreaming) {
|
||||
Animated.loop(
|
||||
Animated.sequence([
|
||||
Animated.timing(fadeAnim, {
|
||||
toValue: 1,
|
||||
duration: 600,
|
||||
useNativeDriver: true,
|
||||
}),
|
||||
Animated.timing(fadeAnim, {
|
||||
toValue: 0.3,
|
||||
duration: 600,
|
||||
useNativeDriver: true,
|
||||
}),
|
||||
])
|
||||
).start();
|
||||
} else {
|
||||
fadeAnim.stopAnimation();
|
||||
fadeAnim.setValue(1);
|
||||
}
|
||||
}, [isStreaming, fadeAnim]);
|
||||
|
||||
return (
|
||||
<View className="flex-row justify-start mb-3 px-4">
|
||||
<View className="bg-teal-800 rounded-2xl rounded-tl-sm px-4 py-3 max-w-[80%]">
|
||||
<Text className="text-teal-50 text-base leading-5">{message}</Text>
|
||||
<View className="flex-row items-center justify-between mt-1">
|
||||
<Text className="text-teal-200 text-xs">{time}</Text>
|
||||
{isStreaming && (
|
||||
<Animated.View style={{ opacity: fadeAnim }}>
|
||||
<Text className="text-teal-200 text-xs ml-2 font-bold">...</Text>
|
||||
</Animated.View>
|
||||
)}
|
||||
</View>
|
||||
</View>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
|
||||
interface ActivityLogProps {
|
||||
type: 'system' | 'info' | 'success' | 'error' | 'artifact';
|
||||
message: string;
|
||||
timestamp: number;
|
||||
metadata?: Record<string, unknown>;
|
||||
artifactId?: string;
|
||||
artifactType?: string;
|
||||
title?: string;
|
||||
onArtifactClick?: (artifactId: string) => void;
|
||||
}
|
||||
|
||||
export function ActivityLog({
|
||||
type,
|
||||
message,
|
||||
timestamp,
|
||||
metadata,
|
||||
artifactId,
|
||||
artifactType,
|
||||
title,
|
||||
onArtifactClick
|
||||
}: ActivityLogProps) {
|
||||
const [isExpanded, setIsExpanded] = useState(false);
|
||||
|
||||
const typeConfig = {
|
||||
system: { bg: 'bg-zinc-800/50', text: 'text-zinc-400', icon: '●' },
|
||||
info: { bg: 'bg-blue-900/30', text: 'text-blue-400', icon: 'i' },
|
||||
success: { bg: 'bg-green-900/30', text: 'text-green-400', icon: '✓' },
|
||||
error: { bg: 'bg-red-900/30', text: 'text-red-400', icon: '✗' },
|
||||
artifact: { bg: 'bg-blue-900/40', text: 'text-blue-300', icon: '📋' },
|
||||
};
|
||||
|
||||
const config = typeConfig[type];
|
||||
|
||||
const handlePress = () => {
|
||||
if (type === 'artifact' && artifactId && onArtifactClick) {
|
||||
onArtifactClick(artifactId);
|
||||
} else if (metadata) {
|
||||
setIsExpanded(!isExpanded);
|
||||
}
|
||||
};
|
||||
|
||||
const displayMessage = type === 'artifact' && artifactType && title
|
||||
? `${artifactType}: ${title}`
|
||||
: message;
|
||||
|
||||
return (
|
||||
<Pressable
|
||||
onPress={handlePress}
|
||||
disabled={type !== 'artifact' && !metadata}
|
||||
className={`mx-2 mb-1 rounded-md overflow-hidden ${config.bg} ${
|
||||
(type === 'artifact' || metadata) ? 'active:opacity-70' : ''
|
||||
}`}
|
||||
>
|
||||
<View className="px-3 py-2.5">
|
||||
<View className="flex-row items-start gap-2">
|
||||
<Text className={`${config.text} text-sm leading-5`}>
|
||||
{config.icon}
|
||||
</Text>
|
||||
<View className="flex-1">
|
||||
<Text className={`${config.text} text-sm leading-5`}>
|
||||
{displayMessage}
|
||||
</Text>
|
||||
{metadata && (
|
||||
<View className="flex-row items-center mt-1">
|
||||
<Text className="text-zinc-500 text-xs">Details</Text>
|
||||
<Text className="text-zinc-500 text-xs ml-1">
|
||||
{isExpanded ? '▼' : '▶'}
|
||||
</Text>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
</View>
|
||||
{isExpanded && metadata && (
|
||||
<View className="mt-2 bg-black/50 rounded p-2 border border-zinc-700">
|
||||
<Text className="text-zinc-300 text-xs font-mono leading-4">
|
||||
{JSON.stringify(metadata, null, 2)}
|
||||
</Text>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
interface ToolCallProps {
|
||||
toolName: string;
|
||||
args: any;
|
||||
result?: any;
|
||||
error?: any;
|
||||
status: 'executing' | 'completed' | 'failed';
|
||||
}
|
||||
|
||||
export function ToolCall({ toolName, args, result, error, status }: ToolCallProps) {
|
||||
const [isExpanded, setIsExpanded] = useState(false);
|
||||
const spinAnim = useRef(new Animated.Value(0)).current;
|
||||
|
||||
useEffect(() => {
|
||||
if (status === 'executing') {
|
||||
Animated.loop(
|
||||
Animated.timing(spinAnim, {
|
||||
toValue: 1,
|
||||
duration: 1000,
|
||||
useNativeDriver: true,
|
||||
})
|
||||
).start();
|
||||
} else {
|
||||
spinAnim.stopAnimation();
|
||||
spinAnim.setValue(0);
|
||||
}
|
||||
}, [status, spinAnim]);
|
||||
|
||||
const spin = spinAnim.interpolate({
|
||||
inputRange: [0, 1],
|
||||
outputRange: ['0deg', '360deg'],
|
||||
});
|
||||
|
||||
const statusConfig = {
|
||||
executing: {
|
||||
border: 'border-amber-500',
|
||||
bg: 'bg-amber-500/20',
|
||||
text: 'text-amber-300',
|
||||
icon: '⟳',
|
||||
label: 'executing',
|
||||
},
|
||||
completed: {
|
||||
border: 'border-green-500',
|
||||
bg: 'bg-green-500/20',
|
||||
text: 'text-green-300',
|
||||
icon: '✓',
|
||||
label: 'completed',
|
||||
},
|
||||
failed: {
|
||||
border: 'border-red-500',
|
||||
bg: 'bg-red-500/20',
|
||||
text: 'text-red-300',
|
||||
icon: '✗',
|
||||
label: 'failed',
|
||||
},
|
||||
};
|
||||
|
||||
const config = statusConfig[status];
|
||||
|
||||
return (
|
||||
<Pressable
|
||||
onPress={() => setIsExpanded(!isExpanded)}
|
||||
className={`mx-2 mb-2 bg-zinc-900 rounded-lg border ${config.border} overflow-hidden active:opacity-80`}
|
||||
>
|
||||
<View className="p-3">
|
||||
<View className="flex-row items-center">
|
||||
<Text className="text-zinc-400 text-sm mr-2">
|
||||
{isExpanded ? '▼' : '▶'}
|
||||
</Text>
|
||||
<Text className="text-slate-200 font-mono font-medium text-sm flex-1">
|
||||
{toolName}
|
||||
</Text>
|
||||
<View className={`flex-row items-center gap-1.5 px-2 py-1 rounded ${config.bg}`}>
|
||||
{status === 'executing' ? (
|
||||
<Animated.Text
|
||||
style={{ transform: [{ rotate: spin }] }}
|
||||
className={`${config.text} text-xs`}
|
||||
>
|
||||
{config.icon}
|
||||
</Animated.Text>
|
||||
) : (
|
||||
<Text className={`${config.text} text-xs`}>
|
||||
{config.icon}
|
||||
</Text>
|
||||
)}
|
||||
<Text className={`${config.text} text-xs font-medium uppercase`}>
|
||||
{config.label}
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{isExpanded && (
|
||||
<View className="mt-3 gap-2">
|
||||
<View>
|
||||
<Text className="text-zinc-400 text-xs font-semibold uppercase tracking-wide mb-1.5">
|
||||
Arguments
|
||||
</Text>
|
||||
<View className="bg-black rounded border border-zinc-700 p-2">
|
||||
<Text className="text-slate-200 text-xs font-mono leading-4">
|
||||
{JSON.stringify(args, null, 2)}
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
|
||||
{result !== undefined && (
|
||||
<View>
|
||||
<Text className="text-zinc-400 text-xs font-semibold uppercase tracking-wide mb-1.5">
|
||||
Result
|
||||
</Text>
|
||||
<View className="bg-black rounded border border-zinc-700 p-2">
|
||||
<Text className="text-slate-200 text-xs font-mono leading-4">
|
||||
{JSON.stringify(result, null, 2)}
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
)}
|
||||
|
||||
{error !== undefined && (
|
||||
<View>
|
||||
<Text className="text-red-400 text-xs font-semibold uppercase tracking-wide mb-1.5">
|
||||
Error
|
||||
</Text>
|
||||
<View className="bg-black rounded border border-red-800 p-2">
|
||||
<Text className="text-slate-200 text-xs font-mono leading-4">
|
||||
{JSON.stringify(error, null, 2)}
|
||||
</Text>
|
||||
</View>
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
)}
|
||||
</View>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
import type { PropsWithChildren, ReactElement } from 'react';
|
||||
import { StyleSheet } from 'react-native';
|
||||
import Animated, {
|
||||
interpolate,
|
||||
useAnimatedRef,
|
||||
useAnimatedStyle,
|
||||
useScrollOffset,
|
||||
} from 'react-native-reanimated';
|
||||
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { useColorScheme } from '@/hooks/use-color-scheme';
|
||||
import { useThemeColor } from '@/hooks/use-theme-color';
|
||||
|
||||
const HEADER_HEIGHT = 250;
|
||||
|
||||
type Props = PropsWithChildren<{
|
||||
headerImage: ReactElement;
|
||||
headerBackgroundColor: { dark: string; light: string };
|
||||
}>;
|
||||
|
||||
export default function ParallaxScrollView({
|
||||
children,
|
||||
headerImage,
|
||||
headerBackgroundColor,
|
||||
}: Props) {
|
||||
const backgroundColor = useThemeColor({}, 'background');
|
||||
const colorScheme = useColorScheme() ?? 'light';
|
||||
const scrollRef = useAnimatedRef<Animated.ScrollView>();
|
||||
const scrollOffset = useScrollOffset(scrollRef);
|
||||
const headerAnimatedStyle = useAnimatedStyle(() => {
|
||||
return {
|
||||
transform: [
|
||||
{
|
||||
translateY: interpolate(
|
||||
scrollOffset.value,
|
||||
[-HEADER_HEIGHT, 0, HEADER_HEIGHT],
|
||||
[-HEADER_HEIGHT / 2, 0, HEADER_HEIGHT * 0.75]
|
||||
),
|
||||
},
|
||||
{
|
||||
scale: interpolate(scrollOffset.value, [-HEADER_HEIGHT, 0, HEADER_HEIGHT], [2, 1, 1]),
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
return (
|
||||
<Animated.ScrollView
|
||||
ref={scrollRef}
|
||||
style={{ backgroundColor, flex: 1 }}
|
||||
scrollEventThrottle={16}>
|
||||
<Animated.View
|
||||
style={[
|
||||
styles.header,
|
||||
{ backgroundColor: headerBackgroundColor[colorScheme] },
|
||||
headerAnimatedStyle,
|
||||
]}>
|
||||
{headerImage}
|
||||
</Animated.View>
|
||||
<ThemedView style={styles.content}>{children}</ThemedView>
|
||||
</Animated.ScrollView>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
container: {
|
||||
flex: 1,
|
||||
},
|
||||
header: {
|
||||
height: HEADER_HEIGHT,
|
||||
overflow: 'hidden',
|
||||
},
|
||||
content: {
|
||||
flex: 1,
|
||||
padding: 32,
|
||||
gap: 16,
|
||||
overflow: 'hidden',
|
||||
},
|
||||
});
|
||||
@@ -1,60 +0,0 @@
|
||||
import { StyleSheet, Text, type TextProps } from 'react-native';
|
||||
|
||||
import { useThemeColor } from '@/hooks/use-theme-color';
|
||||
|
||||
export type ThemedTextProps = TextProps & {
|
||||
lightColor?: string;
|
||||
darkColor?: string;
|
||||
type?: 'default' | 'title' | 'defaultSemiBold' | 'subtitle' | 'link';
|
||||
};
|
||||
|
||||
export function ThemedText({
|
||||
style,
|
||||
lightColor,
|
||||
darkColor,
|
||||
type = 'default',
|
||||
...rest
|
||||
}: ThemedTextProps) {
|
||||
const color = useThemeColor({ light: lightColor, dark: darkColor }, 'text');
|
||||
|
||||
return (
|
||||
<Text
|
||||
style={[
|
||||
{ color },
|
||||
type === 'default' ? styles.default : undefined,
|
||||
type === 'title' ? styles.title : undefined,
|
||||
type === 'defaultSemiBold' ? styles.defaultSemiBold : undefined,
|
||||
type === 'subtitle' ? styles.subtitle : undefined,
|
||||
type === 'link' ? styles.link : undefined,
|
||||
style,
|
||||
]}
|
||||
{...rest}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
default: {
|
||||
fontSize: 16,
|
||||
lineHeight: 24,
|
||||
},
|
||||
defaultSemiBold: {
|
||||
fontSize: 16,
|
||||
lineHeight: 24,
|
||||
fontWeight: '600',
|
||||
},
|
||||
title: {
|
||||
fontSize: 32,
|
||||
fontWeight: 'bold',
|
||||
lineHeight: 32,
|
||||
},
|
||||
subtitle: {
|
||||
fontSize: 20,
|
||||
fontWeight: 'bold',
|
||||
},
|
||||
link: {
|
||||
lineHeight: 30,
|
||||
fontSize: 16,
|
||||
color: '#0a7ea4',
|
||||
},
|
||||
});
|
||||
@@ -1,14 +0,0 @@
|
||||
import { View, type ViewProps } from 'react-native';
|
||||
|
||||
import { useThemeColor } from '@/hooks/use-theme-color';
|
||||
|
||||
export type ThemedViewProps = ViewProps & {
|
||||
lightColor?: string;
|
||||
darkColor?: string;
|
||||
};
|
||||
|
||||
export function ThemedView({ style, lightColor, darkColor, ...otherProps }: ThemedViewProps) {
|
||||
const backgroundColor = useThemeColor({ light: lightColor, dark: darkColor }, 'background');
|
||||
|
||||
return <View style={[{ backgroundColor }, style]} {...otherProps} />;
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
import { PropsWithChildren, useState } from 'react';
|
||||
import { StyleSheet, TouchableOpacity } from 'react-native';
|
||||
|
||||
import { ThemedText } from '@/components/themed-text';
|
||||
import { ThemedView } from '@/components/themed-view';
|
||||
import { IconSymbol } from '@/components/ui/icon-symbol';
|
||||
import { Colors } from '@/constants/theme';
|
||||
import { useColorScheme } from '@/hooks/use-color-scheme';
|
||||
|
||||
export function Collapsible({ children, title }: PropsWithChildren & { title: string }) {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const theme = useColorScheme() ?? 'light';
|
||||
|
||||
return (
|
||||
<ThemedView>
|
||||
<TouchableOpacity
|
||||
style={styles.heading}
|
||||
onPress={() => setIsOpen((value) => !value)}
|
||||
activeOpacity={0.8}>
|
||||
<IconSymbol
|
||||
name="chevron.right"
|
||||
size={18}
|
||||
weight="medium"
|
||||
color={theme === 'light' ? Colors.light.icon : Colors.dark.icon}
|
||||
style={{ transform: [{ rotate: isOpen ? '90deg' : '0deg' }] }}
|
||||
/>
|
||||
|
||||
<ThemedText type="defaultSemiBold">{title}</ThemedText>
|
||||
</TouchableOpacity>
|
||||
{isOpen && <ThemedView style={styles.content}>{children}</ThemedView>}
|
||||
</ThemedView>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = StyleSheet.create({
|
||||
heading: {
|
||||
flexDirection: 'row',
|
||||
alignItems: 'center',
|
||||
gap: 6,
|
||||
},
|
||||
content: {
|
||||
marginTop: 6,
|
||||
marginLeft: 24,
|
||||
},
|
||||
});
|
||||
@@ -1,32 +0,0 @@
|
||||
import { SymbolView, SymbolViewProps, SymbolWeight } from 'expo-symbols';
|
||||
import { StyleProp, ViewStyle } from 'react-native';
|
||||
|
||||
export function IconSymbol({
|
||||
name,
|
||||
size = 24,
|
||||
color,
|
||||
style,
|
||||
weight = 'regular',
|
||||
}: {
|
||||
name: SymbolViewProps['name'];
|
||||
size?: number;
|
||||
color: string;
|
||||
style?: StyleProp<ViewStyle>;
|
||||
weight?: SymbolWeight;
|
||||
}) {
|
||||
return (
|
||||
<SymbolView
|
||||
weight={weight}
|
||||
tintColor={color}
|
||||
resizeMode="scaleAspectFit"
|
||||
name={name}
|
||||
style={[
|
||||
{
|
||||
width: size,
|
||||
height: size,
|
||||
},
|
||||
style,
|
||||
]}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
// Fallback for using MaterialIcons on Android and web.
|
||||
|
||||
import MaterialIcons from '@expo/vector-icons/MaterialIcons';
|
||||
import { SymbolWeight, SymbolViewProps } from 'expo-symbols';
|
||||
import { ComponentProps } from 'react';
|
||||
import { OpaqueColorValue, type StyleProp, type TextStyle } from 'react-native';
|
||||
|
||||
type IconMapping = Record<SymbolViewProps['name'], ComponentProps<typeof MaterialIcons>['name']>;
|
||||
type IconSymbolName = keyof typeof MAPPING;
|
||||
|
||||
/**
|
||||
* Add your SF Symbols to Material Icons mappings here.
|
||||
* - see Material Icons in the [Icons Directory](https://icons.expo.fyi).
|
||||
* - see SF Symbols in the [SF Symbols](https://developer.apple.com/sf-symbols/) app.
|
||||
*/
|
||||
const MAPPING = {
|
||||
'house.fill': 'home',
|
||||
'paperplane.fill': 'send',
|
||||
'chevron.left.forwardslash.chevron.right': 'code',
|
||||
'chevron.right': 'chevron-right',
|
||||
} as IconMapping;
|
||||
|
||||
/**
|
||||
* An icon component that uses native SF Symbols on iOS, and Material Icons on Android and web.
|
||||
* This ensures a consistent look across platforms, and optimal resource usage.
|
||||
* Icon `name`s are based on SF Symbols and require manual mapping to Material Icons.
|
||||
*/
|
||||
export function IconSymbol({
|
||||
name,
|
||||
size = 24,
|
||||
color,
|
||||
style,
|
||||
}: {
|
||||
name: IconSymbolName;
|
||||
size?: number;
|
||||
color: string | OpaqueColorValue;
|
||||
style?: StyleProp<TextStyle>;
|
||||
weight?: SymbolWeight;
|
||||
}) {
|
||||
return <MaterialIcons color={color} size={size} name={MAPPING[name]} style={style} />;
|
||||
}
|
||||
109
packages/voice-mobile/components/voice-button.tsx
Normal file
109
packages/voice-mobile/components/voice-button.tsx
Normal file
@@ -0,0 +1,109 @@
|
||||
import { Pressable, View, Text, Animated } from 'react-native';
|
||||
import { useEffect, useRef } from 'react';
|
||||
|
||||
interface VoiceButtonProps {
|
||||
state: 'idle' | 'recording' | 'processing' | 'playing';
|
||||
onPress: () => void;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
export function VoiceButton({ state, onPress, disabled = false }: VoiceButtonProps) {
|
||||
const pulseAnim = useRef(new Animated.Value(1)).current;
|
||||
|
||||
useEffect(() => {
|
||||
if (state === 'recording') {
|
||||
Animated.loop(
|
||||
Animated.sequence([
|
||||
Animated.timing(pulseAnim, {
|
||||
toValue: 1.2,
|
||||
duration: 800,
|
||||
useNativeDriver: true,
|
||||
}),
|
||||
Animated.timing(pulseAnim, {
|
||||
toValue: 1,
|
||||
duration: 800,
|
||||
useNativeDriver: true,
|
||||
}),
|
||||
])
|
||||
).start();
|
||||
} else {
|
||||
pulseAnim.setValue(1);
|
||||
}
|
||||
}, [state, pulseAnim]);
|
||||
|
||||
const getButtonStyle = () => {
|
||||
switch (state) {
|
||||
case 'recording':
|
||||
return 'bg-red-500';
|
||||
case 'processing':
|
||||
return 'bg-blue-500';
|
||||
case 'playing':
|
||||
return 'bg-green-500';
|
||||
default:
|
||||
return 'bg-zinc-700';
|
||||
}
|
||||
};
|
||||
|
||||
const getIcon = () => {
|
||||
switch (state) {
|
||||
case 'recording':
|
||||
return (
|
||||
<View className="w-6 h-6 bg-white rounded" />
|
||||
);
|
||||
case 'processing':
|
||||
return (
|
||||
<View className="w-6 h-6">
|
||||
<View className="w-1.5 h-1.5 bg-white rounded-full absolute top-0 left-3" />
|
||||
<View className="w-1.5 h-1.5 bg-white rounded-full absolute top-3 right-0" />
|
||||
<View className="w-1.5 h-1.5 bg-white rounded-full absolute bottom-0 left-3" />
|
||||
</View>
|
||||
);
|
||||
case 'playing':
|
||||
return (
|
||||
<View className="flex-row items-center gap-1">
|
||||
<View className="w-1 h-4 bg-white rounded" />
|
||||
<View className="w-1 h-6 bg-white rounded" />
|
||||
<View className="w-1 h-3 bg-white rounded" />
|
||||
</View>
|
||||
);
|
||||
default:
|
||||
return (
|
||||
<View className="w-6 h-8 relative">
|
||||
<View className="absolute bottom-0 left-1/2 -translate-x-1/2 w-4 h-6 bg-white rounded-t-full" />
|
||||
<View className="absolute bottom-0 left-1/2 -translate-x-1/2 w-6 h-1.5 bg-white rounded-full" />
|
||||
</View>
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const getLabel = () => {
|
||||
switch (state) {
|
||||
case 'recording':
|
||||
return 'Recording...';
|
||||
case 'processing':
|
||||
return 'Processing...';
|
||||
case 'playing':
|
||||
return 'Playing...';
|
||||
default:
|
||||
return 'Tap to speak';
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<View className="items-center gap-4">
|
||||
<Pressable
|
||||
onPress={onPress}
|
||||
disabled={disabled}
|
||||
className={`${disabled ? 'opacity-50' : 'opacity-100'}`}
|
||||
>
|
||||
<Animated.View
|
||||
style={{ transform: [{ scale: pulseAnim }] }}
|
||||
className={`w-20 h-20 rounded-full ${getButtonStyle()} items-center justify-center shadow-lg`}
|
||||
>
|
||||
{getIcon()}
|
||||
</Animated.View>
|
||||
</Pressable>
|
||||
<Text className="text-zinc-400 text-sm">{getLabel()}</Text>
|
||||
</View>
|
||||
);
|
||||
}
|
||||
22
packages/voice-mobile/eas.json
Normal file
22
packages/voice-mobile/eas.json
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"cli": {
|
||||
"version": ">= 14.0.0"
|
||||
},
|
||||
"build": {
|
||||
"development": {
|
||||
"developmentClient": true,
|
||||
"distribution": "internal",
|
||||
"channel": "development"
|
||||
},
|
||||
"preview": {
|
||||
"distribution": "internal",
|
||||
"channel": "preview"
|
||||
},
|
||||
"production": {
|
||||
"channel": "production"
|
||||
}
|
||||
},
|
||||
"submit": {
|
||||
"production": {}
|
||||
}
|
||||
}
|
||||
259
packages/voice-mobile/hooks/use-audio-player.ts
Normal file
259
packages/voice-mobile/hooks/use-audio-player.ts
Normal file
@@ -0,0 +1,259 @@
|
||||
import { useAudioPlayer as useExpoAudioPlayer, setAudioModeAsync } from 'expo-audio';
|
||||
import { Paths, File } from 'expo-file-system';
|
||||
import { useState, useRef, useEffect } from 'react';
|
||||
import { Platform } from 'react-native';
|
||||
|
||||
interface QueuedAudio {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
interface AudioPlayerOptions {
|
||||
useSpeaker?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert Blob to file URI for expo-audio playback
|
||||
*/
|
||||
async function blobToFile(blob: Blob): Promise<File> {
|
||||
// Read blob as array buffer
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
|
||||
// Create temporary file in cache directory
|
||||
const fileName = `audio_${Date.now()}.mp3`;
|
||||
const file = new File(Paths.cache, fileName);
|
||||
|
||||
// Write array buffer to file
|
||||
const uint8Array = new Uint8Array(arrayBuffer);
|
||||
file.create();
|
||||
const stream = file.writableStream();
|
||||
const writer = stream.getWriter();
|
||||
await writer.write(uint8Array);
|
||||
await writer.close();
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio playback with queue system matching web version functionality
|
||||
*/
|
||||
export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
const player = useExpoAudioPlayer();
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [isPaused, setIsPaused] = useState(false);
|
||||
const queueRef = useRef<QueuedAudio[]>([]);
|
||||
const isProcessingQueueRef = useRef(false);
|
||||
const currentRejectRef = useRef<((error: Error) => void) | null>(null);
|
||||
const currentFileRef = useRef<File | null>(null);
|
||||
|
||||
// Configure audio mode based on speaker/earpiece preference
|
||||
useEffect(() => {
|
||||
async function configureAudioMode() {
|
||||
try {
|
||||
if (Platform.OS === 'android') {
|
||||
// On Android, use shouldRouteThroughEarpiece
|
||||
await setAudioModeAsync({
|
||||
shouldRouteThroughEarpiece: !options?.useSpeaker,
|
||||
});
|
||||
} else if (Platform.OS === 'ios') {
|
||||
// On iOS, the audio routing is controlled by the AVAudioSession category
|
||||
// The default category should work, but we can configure it if needed
|
||||
// For now, we rely on the iOS default behavior which routes to the receiver
|
||||
// unless the user has changed the route (e.g., via speaker button in Control Center)
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('[AudioPlayer] Failed to configure audio mode:', error);
|
||||
}
|
||||
}
|
||||
|
||||
configureAudioMode();
|
||||
}, [options?.useSpeaker]);
|
||||
|
||||
// Monitor playback status
|
||||
useEffect(() => {
|
||||
if (!player.playing && isPlaying && !isPaused) {
|
||||
// Playback finished
|
||||
setIsPlaying(false);
|
||||
processNextInQueue();
|
||||
}
|
||||
}, [player.playing, isPlaying, isPaused]);
|
||||
|
||||
async function play(audioData: Blob): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Add to queue with its promise handlers
|
||||
queueRef.current.push({ audioData, resolve, reject });
|
||||
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueueRef.current) {
|
||||
processQueue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = true;
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
async function processNextInQueue(): Promise<void> {
|
||||
if (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
} else {
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<number> {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
currentRejectRef.current = reject;
|
||||
|
||||
try {
|
||||
console.log(
|
||||
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
|
||||
);
|
||||
|
||||
// Convert blob to file
|
||||
const file = await blobToFile(audioData);
|
||||
currentFileRef.current = file;
|
||||
|
||||
setIsPlaying(true);
|
||||
setIsPaused(false);
|
||||
|
||||
// Load and play audio
|
||||
await player.replace({ uri: file.uri });
|
||||
player.play();
|
||||
|
||||
// Wait for playback to finish
|
||||
const checkInterval = setInterval(() => {
|
||||
if (!player.playing && player.duration > 0) {
|
||||
clearInterval(checkInterval);
|
||||
|
||||
const duration = player.duration;
|
||||
console.log(`[AudioPlayer] Playback finished (duration: ${duration}s)`);
|
||||
|
||||
setIsPlaying(false);
|
||||
currentRejectRef.current = null;
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
resolve(duration);
|
||||
}
|
||||
}, 100);
|
||||
|
||||
} catch (error) {
|
||||
console.error('[AudioPlayer] Error playing audio:', error);
|
||||
setIsPlaying(false);
|
||||
currentRejectRef.current = null;
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
// Stop currently playing audio
|
||||
if (player) {
|
||||
player.pause();
|
||||
}
|
||||
setIsPlaying(false);
|
||||
setIsPaused(false);
|
||||
|
||||
// Clean up temporary file
|
||||
if (currentFileRef.current) {
|
||||
try {
|
||||
currentFileRef.current.delete();
|
||||
} catch (err: any) {
|
||||
console.warn('[AudioPlayer] Failed to delete temp file:', err);
|
||||
}
|
||||
currentFileRef.current = null;
|
||||
}
|
||||
|
||||
// Reject the current playing promise if it exists
|
||||
if (currentRejectRef.current) {
|
||||
currentRejectRef.current(new Error('Playback stopped'));
|
||||
currentRejectRef.current = null;
|
||||
}
|
||||
|
||||
// Reject all pending promises in the queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error('Playback stopped'));
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
function pause(): void {
|
||||
if (player && isPlaying && !isPaused) {
|
||||
player.pause();
|
||||
setIsPaused(true);
|
||||
console.log('[AudioPlayer] Playback paused');
|
||||
}
|
||||
}
|
||||
|
||||
function resume(): void {
|
||||
if (player && isPaused) {
|
||||
player.play();
|
||||
setIsPaused(false);
|
||||
console.log('[AudioPlayer] Playback resumed');
|
||||
}
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
// Reject all pending promises in the queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error('Queue cleared'));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
play,
|
||||
stop,
|
||||
pause,
|
||||
resume,
|
||||
isPlaying: () => isPlaying,
|
||||
isPaused: () => isPaused,
|
||||
clearQueue,
|
||||
};
|
||||
}
|
||||
277
packages/voice-mobile/hooks/use-audio-recorder.ts
Normal file
277
packages/voice-mobile/hooks/use-audio-recorder.ts
Normal file
@@ -0,0 +1,277 @@
|
||||
import { useAudioRecorder as useExpoAudioRecorder, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio';
|
||||
import { Paths, File, Directory, FileInfo } from 'expo-file-system';
|
||||
import { useState } from 'react';
|
||||
|
||||
export interface AudioCaptureConfig {
|
||||
sampleRate?: number;
|
||||
numberOfChannels?: number;
|
||||
bitRate?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Workaround for Expo SDK 54 Android bug where audioRecorder.uri returns empty/zero-byte file
|
||||
* https://github.com/expo/expo/issues/39646
|
||||
*/
|
||||
async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
|
||||
try {
|
||||
console.log('[AudioRecorder] Searching for recording file created at:', createdAt.toISOString());
|
||||
|
||||
const audioDir = new Directory(Paths.cache, 'Audio');
|
||||
console.log('[AudioRecorder] Audio cache directory URI:', audioDir.uri);
|
||||
console.log('[AudioRecorder] Directory exists:', audioDir.exists);
|
||||
|
||||
if (!audioDir.exists) {
|
||||
console.log('[AudioRecorder] Audio cache directory does not exist');
|
||||
return null;
|
||||
}
|
||||
|
||||
const files = audioDir.list();
|
||||
console.log('[AudioRecorder] Found files in Audio cache:', files.length);
|
||||
|
||||
if (!files.length) {
|
||||
console.log('[AudioRecorder] No files found in Audio cache directory');
|
||||
return null;
|
||||
}
|
||||
|
||||
const validFiles = files
|
||||
.map(file => {
|
||||
const info = file.info();
|
||||
console.log('[AudioRecorder] File info:', {
|
||||
uri: info.uri,
|
||||
size: info.size,
|
||||
creationTime: info.creationTime ? new Date(info.creationTime).toISOString() : null,
|
||||
});
|
||||
return info;
|
||||
})
|
||||
.filter(f => f.size && f.size > 0);
|
||||
|
||||
console.log('[AudioRecorder] Valid files (size > 0):', validFiles.length);
|
||||
|
||||
if (validFiles.length === 0) {
|
||||
console.log('[AudioRecorder] No valid files found (all are zero-byte)');
|
||||
return null;
|
||||
}
|
||||
|
||||
let closest: FileInfo | null = null;
|
||||
let minDiff = Infinity;
|
||||
|
||||
for (const file of validFiles) {
|
||||
if (!file.creationTime || !file.uri) continue;
|
||||
const diff = Math.abs(file.creationTime - createdAt.getTime());
|
||||
console.log('[AudioRecorder] Time diff for file:', {
|
||||
uri: file.uri,
|
||||
diffMs: diff,
|
||||
});
|
||||
if (diff < minDiff) {
|
||||
closest = file;
|
||||
minDiff = diff;
|
||||
}
|
||||
}
|
||||
|
||||
if (closest) {
|
||||
const resultUri = closest.uri?.slice(0, -1) ?? null;
|
||||
console.log('[AudioRecorder] Found closest file:', {
|
||||
uri: resultUri,
|
||||
size: closest.size,
|
||||
timeDiffMs: minDiff,
|
||||
});
|
||||
return resultUri;
|
||||
}
|
||||
|
||||
console.log('[AudioRecorder] No closest file found');
|
||||
return null;
|
||||
} catch (e) {
|
||||
console.error('[AudioRecorder] Error finding actual recording file:', e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert audio file URI to Blob for WebSocket transmission
|
||||
* Returns a Blob-like object that works in React Native
|
||||
*/
|
||||
async function uriToBlob(uri: string): Promise<Blob> {
|
||||
// Use expo-file-system to read the file
|
||||
const file = new File(uri);
|
||||
const base64 = await file.base64();
|
||||
const size = file.size;
|
||||
|
||||
// React Native doesn't support creating Blobs from binary data
|
||||
// Create a Blob-like object that has the methods we need
|
||||
const blobLike = {
|
||||
type: 'audio/m4a',
|
||||
size: size,
|
||||
arrayBuffer: async () => {
|
||||
// Convert base64 to ArrayBuffer
|
||||
const binaryString = atob(base64);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return bytes.buffer;
|
||||
},
|
||||
} as Blob;
|
||||
|
||||
return blobLike;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio recording with configuration matching web version
|
||||
* Matches the web app's audio constraints:
|
||||
* - 16000 sample rate (optimal for speech/Whisper)
|
||||
* - 1 channel (mono)
|
||||
* - Echo cancellation, noise suppression, auto gain control (voice_communication on Android)
|
||||
*/
|
||||
export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [recordingStartTime, setRecordingStartTime] = useState<Date | null>(null);
|
||||
|
||||
// Create recording options matching web app constraints
|
||||
const recordingOptions: RecordingOptions = {
|
||||
...RecordingPresets.HIGH_QUALITY,
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
numberOfChannels: config?.numberOfChannels || 1,
|
||||
bitRate: config?.bitRate || 128000,
|
||||
extension: '.m4a',
|
||||
android: {
|
||||
extension: '.m4a',
|
||||
outputFormat: 'mpeg4',
|
||||
audioEncoder: 'aac',
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
audioSource: 'voice_communication', // Enables echo cancellation, noise suppression, auto gain control
|
||||
},
|
||||
ios: {
|
||||
extension: '.m4a',
|
||||
audioQuality: 127, // High quality
|
||||
sampleRate: config?.sampleRate || 16000,
|
||||
linearPCMBitDepth: 16,
|
||||
linearPCMIsBigEndian: false,
|
||||
linearPCMIsFloat: false,
|
||||
},
|
||||
web: {
|
||||
mimeType: 'audio/webm;codecs=opus',
|
||||
bitsPerSecond: config?.bitRate || 128000,
|
||||
},
|
||||
};
|
||||
|
||||
const audioRecorder = useExpoAudioRecorder(recordingOptions);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (isRecording) {
|
||||
throw new Error('Already recording');
|
||||
}
|
||||
|
||||
try {
|
||||
// Request microphone permissions
|
||||
console.log('[AudioRecorder] Requesting recording permissions...');
|
||||
const permissionResponse = await requestRecordingPermissionsAsync();
|
||||
|
||||
if (!permissionResponse.granted) {
|
||||
throw new Error('Microphone permission denied. Please enable microphone access in your device settings.');
|
||||
}
|
||||
|
||||
// Configure audio mode for recording
|
||||
console.log('[AudioRecorder] Configuring audio mode...');
|
||||
await setAudioModeAsync({
|
||||
playsInSilentMode: true,
|
||||
allowsRecording: true,
|
||||
});
|
||||
|
||||
console.log('[AudioRecorder] Starting recording with options:', {
|
||||
sampleRate: recordingOptions.sampleRate,
|
||||
numberOfChannels: recordingOptions.numberOfChannels,
|
||||
bitRate: recordingOptions.bitRate,
|
||||
});
|
||||
|
||||
const startTime = new Date();
|
||||
setRecordingStartTime(startTime);
|
||||
|
||||
// Prepare the recorder before recording (required step)
|
||||
console.log('[AudioRecorder] Preparing recorder...');
|
||||
await audioRecorder.prepareToRecordAsync();
|
||||
|
||||
console.log('[AudioRecorder] Starting recording...');
|
||||
await audioRecorder.record();
|
||||
setIsRecording(true);
|
||||
|
||||
console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
|
||||
console.log('[AudioRecorder] Recorder isRecording:', audioRecorder.isRecording);
|
||||
} catch (error: any) {
|
||||
console.error('[AudioRecorder] Failed to start recording:', error);
|
||||
throw new Error(`Failed to start audio recording: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<Blob> {
|
||||
if (!isRecording) {
|
||||
throw new Error('Not recording');
|
||||
}
|
||||
|
||||
try {
|
||||
// Stop recording
|
||||
await audioRecorder.stop();
|
||||
setIsRecording(false);
|
||||
|
||||
// Get URI from recorder
|
||||
let uri = audioRecorder.uri;
|
||||
console.log('[AudioRecorder] Initial URI from recorder:', uri);
|
||||
|
||||
// Workaround for Expo SDK 54 Android bug - find actual recording file
|
||||
if (recordingStartTime && (!uri || uri === '')) {
|
||||
console.log('[AudioRecorder] Using workaround to find actual recording file...');
|
||||
const actualUri = await getActualRecordingUri(recordingStartTime);
|
||||
if (actualUri) {
|
||||
uri = actualUri;
|
||||
console.log('[AudioRecorder] Found actual recording URI:', uri);
|
||||
}
|
||||
}
|
||||
|
||||
if (!uri || uri === '') {
|
||||
throw new Error('No recording URI found');
|
||||
}
|
||||
|
||||
// Get file info
|
||||
const file = new File(uri);
|
||||
const exists = file.exists;
|
||||
console.log('[AudioRecorder] File exists:', exists);
|
||||
|
||||
if (!exists) {
|
||||
throw new Error('Recording file does not exist');
|
||||
}
|
||||
|
||||
// Convert URI to Blob
|
||||
const audioBlob = await uriToBlob(uri);
|
||||
|
||||
console.log('[AudioRecorder] Recording converted to blob:', {
|
||||
size: audioBlob.size,
|
||||
type: audioBlob.type,
|
||||
});
|
||||
|
||||
// Clean up the temporary file
|
||||
file.delete();
|
||||
|
||||
// Reset start time
|
||||
setRecordingStartTime(null);
|
||||
|
||||
return audioBlob;
|
||||
} catch (error: any) {
|
||||
setIsRecording(false);
|
||||
setRecordingStartTime(null);
|
||||
console.error('[AudioRecorder] Failed to stop recording:', error);
|
||||
throw new Error(`Failed to stop audio recording: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function getSupportedMimeType(): string | null {
|
||||
// On native platforms, expo-audio uses m4a/AAC
|
||||
// On web, it can use webm/opus
|
||||
return 'audio/m4a';
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
isRecording: () => isRecording,
|
||||
getSupportedMimeType,
|
||||
};
|
||||
}
|
||||
78
packages/voice-mobile/hooks/use-settings.ts
Normal file
78
packages/voice-mobile/hooks/use-settings.ts
Normal file
@@ -0,0 +1,78 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import AsyncStorage from '@react-native-async-storage/async-storage';
|
||||
|
||||
const STORAGE_KEY = '@voice-assistant:settings';
|
||||
|
||||
export interface Settings {
|
||||
serverUrl: string;
|
||||
useSpeaker: boolean;
|
||||
keepScreenOn: boolean;
|
||||
theme: 'dark' | 'light' | 'auto';
|
||||
}
|
||||
|
||||
const DEFAULT_SETTINGS: Settings = {
|
||||
serverUrl: 'wss://mohameds-macbook-pro.tail8fe838.ts.net/ws',
|
||||
useSpeaker: true,
|
||||
keepScreenOn: true,
|
||||
theme: 'dark',
|
||||
};
|
||||
|
||||
export interface UseSettingsReturn {
|
||||
settings: Settings;
|
||||
isLoading: boolean;
|
||||
updateSettings: (updates: Partial<Settings>) => Promise<void>;
|
||||
resetSettings: () => Promise<void>;
|
||||
}
|
||||
|
||||
export function useSettings(): UseSettingsReturn {
|
||||
const [settings, setSettings] = useState<Settings>(DEFAULT_SETTINGS);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
|
||||
// Load settings from AsyncStorage on mount
|
||||
useEffect(() => {
|
||||
loadSettings();
|
||||
}, []);
|
||||
|
||||
async function loadSettings() {
|
||||
try {
|
||||
const stored = await AsyncStorage.getItem(STORAGE_KEY);
|
||||
if (stored) {
|
||||
const parsed = JSON.parse(stored) as Partial<Settings>;
|
||||
setSettings({ ...DEFAULT_SETTINGS, ...parsed });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to load settings:', error);
|
||||
// Continue with default settings
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
const updateSettings = useCallback(async (updates: Partial<Settings>) => {
|
||||
try {
|
||||
const newSettings = { ...settings, ...updates };
|
||||
setSettings(newSettings);
|
||||
await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(newSettings));
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to save settings:', error);
|
||||
throw error;
|
||||
}
|
||||
}, [settings]);
|
||||
|
||||
const resetSettings = useCallback(async () => {
|
||||
try {
|
||||
setSettings(DEFAULT_SETTINGS);
|
||||
await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(DEFAULT_SETTINGS));
|
||||
} catch (error) {
|
||||
console.error('[Settings] Failed to reset settings:', error);
|
||||
throw error;
|
||||
}
|
||||
}, []);
|
||||
|
||||
return {
|
||||
settings,
|
||||
isLoading,
|
||||
updateSettings,
|
||||
resetSettings,
|
||||
};
|
||||
}
|
||||
161
packages/voice-mobile/hooks/use-websocket.ts
Normal file
161
packages/voice-mobile/hooks/use-websocket.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
import { useEffect, useRef, useState, useCallback } from 'react';
|
||||
import type {
|
||||
WSInboundMessage,
|
||||
WSOutboundMessage,
|
||||
SessionOutboundMessage
|
||||
} from '@voice-assistant/server/messages';
|
||||
|
||||
export interface UseWebSocketReturn {
|
||||
isConnected: boolean;
|
||||
conversationId: string | null;
|
||||
send: (message: WSInboundMessage) => void;
|
||||
on: (type: SessionOutboundMessage['type'], handler: (message: SessionOutboundMessage) => void) => () => void;
|
||||
sendPing: () => void;
|
||||
sendUserMessage: (message: string) => void;
|
||||
}
|
||||
|
||||
export function useWebSocket(url: string, conversationId?: string | null): UseWebSocketReturn {
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const [currentConversationId, setCurrentConversationId] = useState<string | null>(null);
|
||||
const wsRef = useRef<WebSocket | null>(null);
|
||||
const handlersRef = useRef<Map<SessionOutboundMessage['type'], Set<(message: SessionOutboundMessage) => void>>>(new Map());
|
||||
const reconnectTimeoutRef = useRef<ReturnType<typeof setTimeout> | undefined>(undefined);
|
||||
|
||||
const connect = useCallback(() => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Add conversation ID to URL if provided
|
||||
const wsUrl = conversationId ? `${url}?conversationId=${conversationId}` : url;
|
||||
const ws = new WebSocket(wsUrl);
|
||||
|
||||
ws.onopen = () => {
|
||||
console.log('[WS] Connected to server');
|
||||
setIsConnected(true);
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
console.log('[WS] Disconnected from server');
|
||||
setIsConnected(false);
|
||||
|
||||
// Attempt to reconnect after 3 seconds
|
||||
reconnectTimeoutRef.current = setTimeout(() => {
|
||||
console.log('[WS] Attempting to reconnect...');
|
||||
connect();
|
||||
}, 3000);
|
||||
};
|
||||
|
||||
ws.onerror = (error) => {
|
||||
console.error('[WS] Error:', error);
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
try {
|
||||
const wsMessage: WSOutboundMessage = JSON.parse(event.data);
|
||||
|
||||
// Only session messages trigger handlers
|
||||
if (wsMessage.type === 'session') {
|
||||
const sessionMessage = wsMessage.message;
|
||||
console.log(`[WS] Received session message type: ${sessionMessage.type}`);
|
||||
|
||||
// Track conversation ID when loaded
|
||||
if (sessionMessage.type === 'conversation_loaded') {
|
||||
setCurrentConversationId(sessionMessage.payload.conversationId);
|
||||
}
|
||||
|
||||
// Call all registered handlers for this message type
|
||||
const handlers = handlersRef.current.get(sessionMessage.type);
|
||||
if (handlers) {
|
||||
handlers.forEach((handler) => {
|
||||
try {
|
||||
handler(sessionMessage);
|
||||
} catch (err) {
|
||||
console.error(`[WS] Error in handler for ${sessionMessage.type}:`, err);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// pong - just log
|
||||
console.log(`[WS] Received ${wsMessage.type}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[WS] Failed to parse message:', err);
|
||||
}
|
||||
};
|
||||
|
||||
wsRef.current = ws;
|
||||
} catch (err) {
|
||||
console.error('[WS] Failed to create WebSocket:', err);
|
||||
}
|
||||
}, [url, conversationId]);
|
||||
|
||||
useEffect(() => {
|
||||
connect();
|
||||
|
||||
return () => {
|
||||
if (reconnectTimeoutRef.current) {
|
||||
clearTimeout(reconnectTimeoutRef.current);
|
||||
}
|
||||
if (wsRef.current) {
|
||||
wsRef.current.close();
|
||||
}
|
||||
};
|
||||
}, [connect]);
|
||||
|
||||
const send = useCallback((message: WSInboundMessage) => {
|
||||
if (wsRef.current?.readyState === WebSocket.OPEN) {
|
||||
wsRef.current.send(JSON.stringify(message));
|
||||
} else {
|
||||
console.warn('[WS] Cannot send message - not connected');
|
||||
}
|
||||
}, []);
|
||||
|
||||
const on = useCallback((
|
||||
type: SessionOutboundMessage['type'],
|
||||
handler: (message: SessionOutboundMessage) => void
|
||||
) => {
|
||||
if (!handlersRef.current.has(type)) {
|
||||
handlersRef.current.set(type, new Set());
|
||||
}
|
||||
handlersRef.current.get(type)!.add(handler);
|
||||
|
||||
// Return cleanup function
|
||||
return () => {
|
||||
const handlers = handlersRef.current.get(type);
|
||||
if (handlers) {
|
||||
handlers.delete(handler);
|
||||
if (handlers.size === 0) {
|
||||
handlersRef.current.delete(type);
|
||||
}
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const sendPing = useCallback(() => {
|
||||
send({ type: 'ping' });
|
||||
}, [send]);
|
||||
|
||||
const sendUserMessage = useCallback(
|
||||
(message: string) => {
|
||||
send({
|
||||
type: 'session',
|
||||
message: {
|
||||
type: 'user_text',
|
||||
text: message,
|
||||
},
|
||||
});
|
||||
},
|
||||
[send]
|
||||
);
|
||||
|
||||
return {
|
||||
isConnected,
|
||||
conversationId: currentConversationId,
|
||||
send,
|
||||
on,
|
||||
sendPing,
|
||||
sendUserMessage,
|
||||
};
|
||||
}
|
||||
4326
packages/voice-mobile/package-lock.json
generated
4326
packages/voice-mobile/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -5,27 +5,34 @@
|
||||
"scripts": {
|
||||
"start": "expo start",
|
||||
"reset-project": "node ./scripts/reset-project.js",
|
||||
"android": "expo start --android",
|
||||
"ios": "expo start --ios",
|
||||
"android": "expo run:android",
|
||||
"ios": "expo run:ios",
|
||||
"web": "expo start --web",
|
||||
"lint": "expo lint"
|
||||
},
|
||||
"dependencies": {
|
||||
"@expo/vector-icons": "^15.0.2",
|
||||
"@react-native-async-storage/async-storage": "2.2.0",
|
||||
"@react-navigation/bottom-tabs": "^7.4.0",
|
||||
"@react-navigation/elements": "^2.6.3",
|
||||
"@react-navigation/native": "^7.1.8",
|
||||
"expo": "~54.0.15",
|
||||
"expo-audio": "~1.0.13",
|
||||
"expo-av": "^16.0.7",
|
||||
"expo-constants": "~18.0.9",
|
||||
"expo-dev-client": "^6.0.15",
|
||||
"expo-file-system": "~19.0.17",
|
||||
"expo-font": "~14.0.9",
|
||||
"expo-haptics": "~15.0.7",
|
||||
"expo-image": "~3.0.10",
|
||||
"expo-keep-awake": "^15.0.7",
|
||||
"expo-linking": "~8.0.8",
|
||||
"expo-router": "~6.0.13",
|
||||
"expo-splash-screen": "~31.0.10",
|
||||
"expo-status-bar": "~3.0.8",
|
||||
"expo-symbols": "~1.0.7",
|
||||
"expo-system-ui": "~6.0.7",
|
||||
"expo-updates": "~29.0.12",
|
||||
"expo-web-browser": "~15.0.8",
|
||||
"nativewind": "^5.0.0-preview.2",
|
||||
"react": "19.1.0",
|
||||
@@ -42,13 +49,14 @@
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4.1.15",
|
||||
"@types/react": "~19.1.0",
|
||||
"eas-cli": "^16.24.1",
|
||||
"eslint": "^9.25.0",
|
||||
"eslint-config-expo": "~10.0.0",
|
||||
"postcss": "^8.4.49",
|
||||
"tailwindcss": "^4.1.15",
|
||||
"typescript": "~5.9.2"
|
||||
},
|
||||
"resolutions": {
|
||||
"overrides": {
|
||||
"lightningcss": "1.30.1"
|
||||
},
|
||||
"private": true
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
"paths": {
|
||||
"@/*": [
|
||||
"./*"
|
||||
],
|
||||
"@voice-assistant/*": [
|
||||
"../voice-assistant/src/*"
|
||||
]
|
||||
}
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user