puter/paseo
1
0
Fork 0
mirror of https://github.com/getpaseo/paseo.git synced 2026-07-29 12:01:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: migrate to @boudra/expo-two-way-audio and add tool call details bottom sheet

Browse Source
This commit is contained in:
Mohamed Boudra
2025-10-23 20:01:52 +02:00
parent 74fffcd8a3
commit 15320819a7
9 changed files with 417 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/app/package.json
View File

@@ -17,7 +17,7 @@
"@react-navigation/bottom-tabs": "^7.4.0",
"@react-navigation/elements": "^2.6.3",
"@react-navigation/native": "^7.1.8",
"@speechmatics/expo-two-way-audio": "^0.1.2",
"@boudra/expo-two-way-audio": "^0.1.3",
"buffer": "^6.0.3",
"expo": "^54.0.18",
"expo-audio": "~1.0.13",

38
packages/app/src/components/agent-stream-view.tsx
View File

@@ -1,9 +1,11 @@
import { useEffect, useRef } from 'react';
import { useEffect, useRef, useState } from 'react';
import { View, Text, ScrollView, Pressable } from 'react-native';
import { useSafeAreaInsets } from 'react-native-safe-area-context';
import { StyleSheet, useUnistyles } from 'react-native-unistyles';
import { BottomSheetModal } from '@gorhom/bottom-sheet';
import type { AgentStatus } from '@server/server/acp/types';
import { AssistantMessage, UserMessage, ActivityLog, ToolCall } from './message';
import { ToolCallBottomSheet } from './tool-call-bottom-sheet';
import type { StreamItem } from '@/types/stream';
export interface AgentStreamViewProps {
@@ -40,13 +42,32 @@ export function AgentStreamView({
onPermissionResponse,
}: AgentStreamViewProps) {
const scrollViewRef = useRef<ScrollView>(null);
const bottomSheetRef = useRef<BottomSheetModal>(null);
const insets = useSafeAreaInsets();
const [selectedToolCall, setSelectedToolCall] = useState<{
toolName: string;
status: 'executing' | 'completed' | 'failed';
args: any;
result?: any;
error?: any;
} | null>(null);
// Auto-scroll to bottom when new items arrive
useEffect(() => {
scrollViewRef.current?.scrollToEnd({ animated: true });
}, [streamItems]);
function handleOpenToolCallDetails(toolCall: {
toolName: string;
status: 'executing' | 'completed' | 'failed';
args: any;
result?: any;
error?: any;
}) {
setSelectedToolCall(toolCall);
bottomSheetRef.current?.present();
}
return (
<View style={stylesheet.container}>
{/* Content list */}
@@ -107,6 +128,12 @@ export function AgentStreamView({
args={item.rawInput}
result={item.rawOutput}
status={toolStatus}
onOpenDetails={() => handleOpenToolCallDetails({
toolName: item.title,
status: toolStatus,
args: item.rawInput,
result: item.rawOutput,
})}
/>
);
}
@@ -137,6 +164,15 @@ export function AgentStreamView({
/>
))}
</ScrollView>
<ToolCallBottomSheet
bottomSheetRef={bottomSheetRef}
toolName={selectedToolCall?.toolName}
status={selectedToolCall?.status}
args={selectedToolCall?.args}
result={selectedToolCall?.result}
error={selectedToolCall?.error}
/>
</View>
);
}

33
packages/app/src/components/create-agent-modal.tsx
View File

@@ -1,5 +1,12 @@
import { useState, useRef, useEffect, useMemo, useCallback } from "react";
import { View, Text, Pressable, ScrollView, ActivityIndicator } from "react-native";
import {
View,
Text,
Pressable,
ScrollView,
ActivityIndicator,
InteractionManager,
} from "react-native";
import { useSafeAreaInsets } from "react-native-safe-area-context";
import { useReanimatedKeyboardAnimation } from "react-native-keyboard-controller";
import Animated, { useAnimatedStyle } from "react-native-reanimated";
@@ -57,6 +64,9 @@ export function CreateAgentModal({
const [isLoading, setIsLoading] = useState(false);
const [pendingRequestId, setPendingRequestId] = useState<string | null>(null);
// Use ref instead of state to survive state resets in onDismiss
const pendingNavigationAgentIdRef = useRef<string | null>(null);
const snapPoints = useMemo(() => ["90%"], []);
// Keyboard animation for footer
@@ -132,12 +142,11 @@ export function CreateAgentModal({
setIsLoading(false);
setPendingRequestId(null);
// Navigate to the agent page BEFORE closing modal
// This prevents race condition on Android where router.push() happens
// while the modal is unmounting, causing NullPointerException
router.push(`/agent/${agentId}`);
// Store the agent ID in ref for navigation after modal dismisses
// Using ref instead of state because state gets reset in onDismiss
pendingNavigationAgentIdRef.current = agentId;
// Close modal after navigation starts
// Close modal - navigation will happen in handleDismiss
handleClose();
}
});
@@ -145,7 +154,7 @@ export function CreateAgentModal({
return () => {
unsubscribe();
};
}, [pendingRequestId, ws, router]);
}, [pendingRequestId, ws]);
async function handleCreate() {
if (!workingDir.trim()) {
@@ -194,6 +203,8 @@ export function CreateAgentModal({
}
function handleDismiss() {
const agentId = pendingNavigationAgentIdRef.current;
// Reset all state
setWorkingDir("");
setSelectedMode("plan");
@@ -201,6 +212,14 @@ export function CreateAgentModal({
setIsLoading(false);
setPendingRequestId(null);
onClose();
// Navigate after interactions complete to avoid race with dismiss animation
if (agentId) {
InteractionManager.runAfterInteractions(() => {
router.push(`/agent/${agentId}`);
pendingNavigationAgentIdRef.current = null;
});
}
}
return (

54
packages/app/src/components/message.tsx
View File

@@ -322,6 +322,7 @@ interface ToolCallProps {
result?: any;
error?: any;
status: 'executing' | 'completed' | 'failed';
onOpenDetails?: () => void;
}
const toolCallStylesheet = StyleSheet.create((theme) => ({
@@ -420,8 +421,7 @@ const toolCallStylesheet = StyleSheet.create((theme) => ({
},
}));
export function ToolCall({ toolName, args, result, error, status }: ToolCallProps) {
const [isExpanded, setIsExpanded] = useState(false);
export function ToolCall({ toolName, args, result, error, status, onOpenDetails }: ToolCallProps) {
const spinAnim = useRef(new Animated.Value(0)).current;
useEffect(() => {
@@ -469,18 +469,11 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp
return (
<Pressable
onPress={() => setIsExpanded(!isExpanded)}
onPress={onOpenDetails}
style={[toolCallStylesheet.pressable, toolCallStylesheet.pressableActive, config.border]}
>
<View style={toolCallStylesheet.content}>
<View style={toolCallStylesheet.headerRow}>
<View style={toolCallStylesheet.chevronContainer}>
{isExpanded ? (
<ChevronDown size={16} color="#9ca3af" />
) : (
<ChevronRight size={16} color="#9ca3af" />
)}
</View>
<Text style={toolCallStylesheet.toolName}>
{toolName}
</Text>
@@ -499,47 +492,6 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp
</Text>
</View>
</View>
{isExpanded && (
<View style={toolCallStylesheet.expandedContent}>
<View style={toolCallStylesheet.section}>
<Text style={toolCallStylesheet.sectionTitle}>
Arguments
</Text>
<View style={toolCallStylesheet.sectionContent}>
<Text style={toolCallStylesheet.sectionText}>
{JSON.stringify(args, null, 2)}
</Text>
</View>
</View>
{result !== undefined && (
<View style={toolCallStylesheet.section}>
<Text style={toolCallStylesheet.sectionTitle}>
Result
</Text>
<View style={toolCallStylesheet.sectionContent}>
<Text style={toolCallStylesheet.sectionText}>
{JSON.stringify(result, null, 2)}
</Text>
</View>
</View>
)}
{error !== undefined && (
<View style={toolCallStylesheet.section}>
<Text style={[toolCallStylesheet.sectionTitle, toolCallStylesheet.errorSectionTitle]}>
Error
</Text>
<View style={[toolCallStylesheet.sectionContent, toolCallStylesheet.errorSectionContent]}>
<Text style={toolCallStylesheet.sectionText}>
{JSON.stringify(error, null, 2)}
</Text>
</View>
</View>
)}
</View>
)}
</View>
</Pressable>
);

55
packages/app/src/components/orchestrator-messages-view.tsx
View File

@@ -1,12 +1,14 @@
import { View, ScrollView } from "react-native";
import { StyleSheet, useUnistyles } from "react-native-unistyles";
import { forwardRef } from "react";
import { forwardRef, useRef, useState } from "react";
import { BottomSheetModal } from "@gorhom/bottom-sheet";
import {
UserMessage,
AssistantMessage,
ActivityLog,
ToolCall,
} from "@/components/message";
import { ToolCallBottomSheet } from "./tool-call-bottom-sheet";
import type { MessageEntry } from "@/contexts/session-context";
interface OrchestratorMessagesViewProps {
@@ -18,15 +20,35 @@ interface OrchestratorMessagesViewProps {
export const OrchestratorMessagesView = forwardRef<ScrollView, OrchestratorMessagesViewProps>(
function OrchestratorMessagesView({ messages, currentAssistantMessage, onArtifactClick }, ref) {
const { theme } = useUnistyles();
const bottomSheetRef = useRef<BottomSheetModal>(null);
const [selectedToolCall, setSelectedToolCall] = useState<{
toolName: string;
status: 'executing' | 'completed' | 'failed';
args: any;
result?: any;
error?: any;
} | null>(null);
function handleOpenToolCallDetails(toolCall: {
toolName: string;
status: 'executing' | 'completed' | 'failed';
args: any;
result?: any;
error?: any;
}) {
setSelectedToolCall(toolCall);
bottomSheetRef.current?.present();
}
return (
<ScrollView
ref={ref}
style={styles.scrollView}
contentContainerStyle={styles.scrollContent}
keyboardShouldPersistTaps="handled"
keyboardDismissMode="on-drag"
>
<>
<ScrollView
ref={ref}
style={styles.scrollView}
contentContainerStyle={styles.scrollContent}
keyboardShouldPersistTaps="handled"
keyboardDismissMode="on-drag"
>
{messages.map((msg) => {
if (msg.type === "user") {
return (
@@ -85,6 +107,13 @@ export const OrchestratorMessagesView = forwardRef<ScrollView, OrchestratorMessa
result={msg.result}
error={msg.error}
status={msg.status}
onOpenDetails={() => handleOpenToolCallDetails({
toolName: msg.toolName,
status: msg.status,
args: msg.args,
result: msg.result,
error: msg.error,
})}
/>
);
}
@@ -101,6 +130,16 @@ export const OrchestratorMessagesView = forwardRef<ScrollView, OrchestratorMessa
/>
)}
</ScrollView>
<ToolCallBottomSheet
bottomSheetRef={bottomSheetRef}
toolName={selectedToolCall?.toolName}
status={selectedToolCall?.status}
args={selectedToolCall?.args}
result={selectedToolCall?.result}
error={selectedToolCall?.error}
/>
</>
);
}
);

207
packages/app/src/components/tool-call-bottom-sheet.tsx Normal file
View File

@@ -0,0 +1,207 @@
import React, { useCallback, useMemo } from "react";
import { View, Text, StyleSheet } from "react-native";
import {
BottomSheetModal,
BottomSheetView,
BottomSheetScrollView,
BottomSheetBackdrop,
BottomSheetBackdropProps,
} from "@gorhom/bottom-sheet";
interface ToolCallBottomSheetProps {
bottomSheetRef: React.RefObject<BottomSheetModal>;
toolName?: string;
status?: "pending" | "in_progress" | "executing" | "completed" | "failed";
args?: any;
result?: any;
error?: any;
}
export function ToolCallBottomSheet({
bottomSheetRef,
toolName,
status,
args,
result,
error,
}: ToolCallBottomSheetProps) {
const snapPoints = useMemo(() => ["80%"], []);
const renderBackdrop = useCallback(
(props: BottomSheetBackdropProps) => (
<BottomSheetBackdrop
{...props}
disappearsOnIndex={-1}
appearsOnIndex={0}
opacity={0.5}
/>
),
[]
);
const statusColor = useMemo(() => {
switch (status) {
case "completed":
return "#22c55e";
case "failed":
return "#ef4444";
case "executing":
case "in_progress":
return "#3b82f6";
case "pending":
default:
return "#6b7280";
}
}, [status]);
const statusLabel = useMemo(() => {
switch (status) {
case "in_progress":
case "executing":
return "Executing";
case "completed":
return "Completed";
case "failed":
return "Failed";
case "pending":
default:
return "Pending";
}
}, [status]);
return (
<BottomSheetModal
ref={bottomSheetRef}
snapPoints={snapPoints}
backdropComponent={renderBackdrop}
enablePanDownToClose={true}
handleIndicatorStyle={styles.handleIndicator}
backgroundStyle={styles.background}
>
<BottomSheetView style={styles.container}>
<View style={styles.header}>
<Text style={styles.toolName}>{toolName || "Tool Call"}</Text>
<View style={[styles.statusBadge, { backgroundColor: statusColor }]}>
<Text style={styles.statusText}>{statusLabel}</Text>
</View>
</View>
<BottomSheetScrollView
style={styles.scrollView}
contentContainerStyle={styles.scrollContent}
>
{args !== undefined && (
<View style={styles.section}>
<Text style={styles.sectionTitle}>Arguments</Text>
<View style={styles.jsonContainer}>
<Text style={styles.jsonText}>
{JSON.stringify(args, null, 2)}
</Text>
</View>
</View>
)}
{result !== undefined && (
<View style={styles.section}>
<Text style={styles.sectionTitle}>Result</Text>
<View style={styles.jsonContainer}>
<Text style={styles.jsonText}>
{JSON.stringify(result, null, 2)}
</Text>
</View>
</View>
)}
{error !== undefined && (
<View style={styles.section}>
<Text style={styles.sectionTitle}>Error</Text>
<View style={[styles.jsonContainer, styles.errorContainer]}>
<Text style={[styles.jsonText, styles.errorText]}>
{JSON.stringify(error, null, 2)}
</Text>
</View>
</View>
)}
</BottomSheetScrollView>
</BottomSheetView>
</BottomSheetModal>
);
}
const styles = StyleSheet.create({
container: {
flex: 1,
backgroundColor: "#1f2937",
},
handleIndicator: {
backgroundColor: "#4b5563",
},
background: {
backgroundColor: "#1f2937",
},
header: {
flexDirection: "row",
alignItems: "center",
justifyContent: "space-between",
paddingHorizontal: 20,
paddingVertical: 16,
borderBottomWidth: 1,
borderBottomColor: "#374151",
},
toolName: {
fontSize: 18,
fontWeight: "600",
color: "#f9fafb",
flex: 1,
},
statusBadge: {
paddingHorizontal: 8,
paddingVertical: 4,
borderRadius: 4,
marginLeft: 12,
},
statusText: {
fontSize: 12,
fontWeight: "600",
color: "#fff",
textTransform: "capitalize",
},
scrollView: {
flex: 1,
},
scrollContent: {
paddingHorizontal: 20,
paddingVertical: 16,
},
section: {
marginBottom: 20,
},
sectionTitle: {
fontSize: 14,
fontWeight: "600",
color: "#9ca3af",
marginBottom: 8,
textTransform: "uppercase",
letterSpacing: 0.5,
},
jsonContainer: {
backgroundColor: "#111827",
borderRadius: 8,
padding: 12,
borderWidth: 1,
borderColor: "#374151",
},
jsonText: {
fontFamily: "monospace",
fontSize: 12,
color: "#e5e7eb",
lineHeight: 18,
},
errorContainer: {
borderColor: "#ef4444",
backgroundColor: "#1f1416",
},
errorText: {
color: "#fca5a5",
},
});

71
packages/app/src/hooks/use-audio-player.ts
View File

@@ -1,5 +1,11 @@
import { useState, useRef } from 'react';
import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio';
import { useState, useRef } from "react";
import {
initialize,
playPCMData,
stopPlayback,
pausePlayback,
resumePlayback,
} from "@boudra/expo-two-way-audio";
interface QueuedAudio {
audioData: Blob;
@@ -14,7 +20,7 @@ interface QueuedAudio {
function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
// PCM16 = 2 bytes per sample
const samples24k = pcm24k.length / 2;
const samples16k = Math.floor(samples24k * 16000 / 24000);
const samples16k = Math.floor((samples24k * 16000) / 24000);
const pcm16k = new Uint8Array(samples16k * 2);
const ratio = 24000 / 16000; // 1.5
@@ -27,10 +33,10 @@ function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array {
}
}
console.log('[AudioPlayer] Resampled PCM:', {
console.log("[AudioPlayer] Resampled PCM:", {
input24k: pcm24k.length,
output16k: pcm16k.length,
durationMs: (samples16k / 16),
durationMs: samples16k / 16,
});
return pcm16k;
@@ -44,6 +50,7 @@ export function useAudioPlayer() {
const [audioInitialized, setAudioInitialized] = useState(false);
const queueRef = useRef<QueuedAudio[]>([]);
const isProcessingQueueRef = useRef(false);
const playbackTimeoutRef = useRef<NodeJS.Timeout | null>(null);
async function play(audioData: Blob): Promise<number> {
return new Promise((resolve, reject) => {
@@ -100,12 +107,19 @@ export function useAudioPlayer() {
// Initialize audio if not already initialized
if (!audioInitialized) {
console.log('[AudioPlayer] Initializing audio...');
console.log("[AudioPlayer] Initializing audio...");
await initialize();
setAudioInitialized(true);
console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)');
console.log(
"[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)"
);
}
// Workaround: Resume playback before playing new audio to ensure the audio engine is ready
// This fixes the issue where playback doesn't work after calling stopPlayback()
console.log("[AudioPlayer] Resuming playback engine...");
resumePlayback();
// Get PCM data from blob (server now sends PCM format)
const arrayBuffer = await audioData.arrayBuffer();
let pcm24k = new Uint8Array(arrayBuffer);
@@ -118,22 +132,40 @@ export function useAudioPlayer() {
const durationSec = samples / 16000; // 16kHz sample rate
const audioSizeKb = (pcm16k.length / 1024).toFixed(2);
console.log('[AudioPlayer] 🔊 Playing audio:', audioSizeKb, 'KB, duration:', durationSec.toFixed(2), 's');
console.log(
"[AudioPlayer] 🔊 Playing audio:",
audioSizeKb,
"KB, duration:",
durationSec.toFixed(2),
"s"
);
setIsPlaying(true);
// Play entire PCM data at once through Speechmatics
playPCMData(pcm16k);
// Clear any existing timeout
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
}
// Wait for playback to finish (estimate based on duration)
setTimeout(() => {
console.log('[AudioPlayer] ✅ Playback finished');
playbackTimeoutRef.current = setTimeout(() => {
console.log("[AudioPlayer] ✅ Playback finished");
setIsPlaying(false);
playbackTimeoutRef.current = null;
resolve(durationSec);
}, durationSec * 1000);
} catch (error) {
console.error('[AudioPlayer] Error playing audio:', error);
console.error("[AudioPlayer] Error playing audio:", error);
// Clear timeout on error
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
playbackTimeoutRef.current = null;
}
setIsPlaying(false);
reject(error);
}
@@ -142,7 +174,16 @@ export function useAudioPlayer() {
function stop(): void {
if (isPlaying) {
console.log('[AudioPlayer] 🛑 Stopping playback (interrupted)');
console.log("[AudioPlayer] 🛑 Stopping playback (interrupted)");
// Stop native playback
stopPlayback();
// Clear playback timeout
if (playbackTimeoutRef.current) {
clearTimeout(playbackTimeoutRef.current);
playbackTimeoutRef.current = null;
}
}
setIsPlaying(false);
@@ -150,7 +191,7 @@ export function useAudioPlayer() {
// Reject all pending promises in the queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error('Playback stopped'));
item.reject(new Error("Playback stopped"));
}
isProcessingQueueRef.current = false;
@@ -160,7 +201,7 @@ export function useAudioPlayer() {
// Reject all pending promises in the queue
while (queueRef.current.length > 0) {
const item = queueRef.current.shift()!;
item.reject(new Error('Queue cleared'));
item.reject(new Error("Queue cleared"));
}
}

36
packages/app/src/hooks/use-speechmatics-audio.ts
View File

@@ -6,7 +6,7 @@ import {
useExpoTwoWayAudioEventListener,
type MicrophoneDataCallback,
type VolumeLevelCallback,
} from "@speechmatics/expo-two-way-audio";
} from "@boudra/expo-two-way-audio";
export interface SpeechmaticsAudioConfig {
onAudioSegment?: (audioData: string) => void;
@@ -200,7 +200,11 @@ export function useSpeechmaticsAudio(
// console.log('[SpeechmaticsAudio] Volume:', volumeLevel.toFixed(6), 'Threshold:', VOLUME_THRESHOLD);
if (speechDetected && !isSpeakingRef.current && !speechConfirmedRef.current) {
if (
speechDetected &&
!isSpeakingRef.current &&
!speechConfirmedRef.current
) {
// Initial speech detection - start tracking
if (speechDetectionStartRef.current === null) {
console.log(
@@ -233,10 +237,18 @@ export function useSpeechmaticsAudio(
config.onSpeechStart?.();
}
}
} else if (speechDetected && isSpeakingRef.current && speechConfirmedRef.current) {
} else if (
speechDetected &&
isSpeakingRef.current &&
speechConfirmedRef.current
) {
// Continuing confirmed speech
silenceStartRef.current = null;
} else if (!speechDetected && !speechConfirmedRef.current && speechDetectionStartRef.current !== null) {
} else if (
!speechDetected &&
!speechConfirmedRef.current &&
speechDetectionStartRef.current !== null
) {
// Volume dropped during detection phase - apply grace period
if (detectionSilenceStartRef.current === null) {
detectionSilenceStartRef.current = Date.now();
@@ -255,7 +267,11 @@ export function useSpeechmaticsAudio(
setIsDetecting(false);
}
}
} else if (!speechDetected && isSpeakingRef.current && speechConfirmedRef.current) {
} else if (
!speechDetected &&
isSpeakingRef.current &&
speechConfirmedRef.current
) {
// Potential speech END
if (silenceStartRef.current === null) {
silenceStartRef.current = Date.now();
@@ -289,7 +305,15 @@ export function useSpeechmaticsAudio(
}
}
},
[isActive, isMuted, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config]
[
isActive,
isMuted,
VOLUME_THRESHOLD,
SILENCE_DURATION_MS,
SPEECH_CONFIRMATION_MS,
DETECTION_GRACE_PERIOD_MS,
config,
]
)
);

13
packages/server/src/server/session.ts
View File

@@ -259,7 +259,7 @@ export class Session {
/**
* Maybe trigger title generation for an agent
* Only generates title once after first meaningful activity
* Only generates title once after first user message chunk and some initial agent activity
*/
private maybeTriggerTitleGeneration(agentId: string): void {
// Skip if title generator not initialized
@@ -275,8 +275,15 @@ export class Session {
// Get agent updates
const updates = this.agentManager.getAgentUpdates(agentId);
// Need at least 15 updates before generating title
if (updates.length < 15) {
// Find first user message chunk
const hasUserMessage = updates.some(
(update) =>
update.notification.type === "session" &&
update.notification.notification.update.sessionUpdate === "user_message_chunk"
);
// Need at least one user message and some additional updates (3-5) for context
if (!hasUserMessage || updates.length < 5) {
return;
}