diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 0a404bd2f..01dd23db8 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -13,3 +13,11 @@ This is an npm workspace monorepo: - **Root**: Workspace configuration and shared TypeScript config - **packages/server**: Backend server (Express + WebSocket API) - **packages/app**: Cross-platform app (Expo - iOS, Android, Web) + +## Running and checking logs + +We run the mobile app and server in the tmux session `voice-dev`: + +tmux capture-pane -t voice-dev:mobile -p + +tmux capture-pane -t voice-dev:server -p diff --git a/package-lock.json b/package-lock.json index 601611292..9f66b3a4b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -101,19 +101,6 @@ "zod": "^3.25.76 || ^4.1.8" } }, - "node_modules/@alloc/quick-lru": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", - "integrity": "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/@babel/code-frame": { "version": "7.10.4", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", @@ -6384,7 +6371,6 @@ "version": "0.76.2", "resolved": "https://registry.npmjs.org/@react-native/normalize-colors/-/normalize-colors-0.76.2.tgz", "integrity": "sha512-ICoOpaTLPsFQjNLSM00NgQr6wal300cZZonHVSDXKntX+BfkLeuCHRtr/Mn+klTtW+/1v2/2FRm9dXjvyGf9Dw==", - "dev": true, "license": "MIT" }, "node_modules/@react-native/virtualized-lists": { @@ -6906,283 +6892,23 @@ "react-native": "*" } }, + "node_modules/@speechmatics/expo-two-way-audio": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@speechmatics/expo-two-way-audio/-/expo-two-way-audio-0.1.2.tgz", + "integrity": "sha512-ecg4y/fkd2v1oWH2BTNyG5GBRqDuL2j9b86xNXGXHX87Xx87pw9CIPmIUIJWkk2aApXsWdkATz2txEcV41JMkg==", + "license": "MIT", + "peerDependencies": { + "expo": "*", + "react": "*", + "react-native": "*" + } + }, "node_modules/@standard-schema/spec": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.0.0.tgz", "integrity": "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==", "license": "MIT" }, - "node_modules/@tailwindcss/node": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.1.15.tgz", - "integrity": "sha512-HF4+7QxATZWY3Jr8OlZrBSXmwT3Watj0OogeDvdUY/ByXJHQ+LBtqA2brDb3sBxYslIFx6UP94BJ4X6a4L9Bmw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/remapping": "^2.3.4", - "enhanced-resolve": "^5.18.3", - "jiti": "^2.6.0", - "lightningcss": "1.30.2", - "magic-string": "^0.30.19", - "source-map-js": "^1.2.1", - "tailwindcss": "4.1.15" - } - }, - "node_modules/@tailwindcss/oxide": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.1.15.tgz", - "integrity": "sha512-krhX+UOOgnsUuks2SR7hFafXmLQrKxB4YyRTERuCE59JlYL+FawgaAlSkOYmDRJdf1Q+IFNDMl9iRnBW7QBDfQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 10" - }, - "optionalDependencies": { - "@tailwindcss/oxide-android-arm64": "4.1.15", - "@tailwindcss/oxide-darwin-arm64": "4.1.15", - "@tailwindcss/oxide-darwin-x64": "4.1.15", - "@tailwindcss/oxide-freebsd-x64": "4.1.15", - "@tailwindcss/oxide-linux-arm-gnueabihf": "4.1.15", - "@tailwindcss/oxide-linux-arm64-gnu": "4.1.15", - "@tailwindcss/oxide-linux-arm64-musl": "4.1.15", - "@tailwindcss/oxide-linux-x64-gnu": "4.1.15", - "@tailwindcss/oxide-linux-x64-musl": "4.1.15", - "@tailwindcss/oxide-wasm32-wasi": "4.1.15", - "@tailwindcss/oxide-win32-arm64-msvc": "4.1.15", - "@tailwindcss/oxide-win32-x64-msvc": "4.1.15" - } - }, - "node_modules/@tailwindcss/oxide-android-arm64": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.1.15.tgz", - "integrity": "sha512-TkUkUgAw8At4cBjCeVCRMc/guVLKOU1D+sBPrHt5uVcGhlbVKxrCaCW9OKUIBv1oWkjh4GbunD/u/Mf0ql6kEA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-darwin-arm64": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.1.15.tgz", - "integrity": "sha512-xt5XEJpn2piMSfvd1UFN6jrWXyaKCwikP4Pidcf+yfHTSzSpYhG3dcMktjNkQO3JiLCp+0bG0HoWGvz97K162w==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-darwin-x64": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.1.15.tgz", - "integrity": "sha512-TnWaxP6Bx2CojZEXAV2M01Yl13nYPpp0EtGpUrY+LMciKfIXiLL2r/SiSRpagE5Fp2gX+rflp/Os1VJDAyqymg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-freebsd-x64": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.1.15.tgz", - "integrity": "sha512-quISQDWqiB6Cqhjc3iWptXVZHNVENsWoI77L1qgGEHNIdLDLFnw3/AfY7DidAiiCIkGX/MjIdB3bbBZR/G2aJg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.1.15.tgz", - "integrity": "sha512-ObG76+vPlab65xzVUQbExmDU9FIeYLQ5k2LrQdR2Ud6hboR+ZobXpDoKEYXf/uOezOfIYmy2Ta3w0ejkTg9yxg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.1.15.tgz", - "integrity": "sha512-4WbBacRmk43pkb8/xts3wnOZMDKsPFyEH/oisCm2q3aLZND25ufvJKcDUpAu0cS+CBOL05dYa8D4U5OWECuH/Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-linux-arm64-musl": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.1.15.tgz", - "integrity": "sha512-AbvmEiteEj1nf42nE8skdHv73NoR+EwXVSgPY6l39X12Ex8pzOwwfi3Kc8GAmjsnsaDEbk+aj9NyL3UeyHcTLg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-linux-x64-gnu": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.1.15.tgz", - "integrity": "sha512-+rzMVlvVgrXtFiS+ES78yWgKqpThgV19ISKD58Ck+YO5pO5KjyxLt7AWKsWMbY0R9yBDC82w6QVGz837AKQcHg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-linux-x64-musl": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.1.15.tgz", - "integrity": "sha512-fPdEy7a8eQN9qOIK3Em9D3TO1z41JScJn8yxl/76mp4sAXFDfV4YXxsiptJcOwy6bGR+70ZSwFIZhTXzQeqwQg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-wasm32-wasi": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.1.15.tgz", - "integrity": "sha512-sJ4yd6iXXdlgIMfIBXuVGp/NvmviEoMVWMOAGxtxhzLPp9LOj5k0pMEMZdjeMCl4C6Up+RM8T3Zgk+BMQ0bGcQ==", - "bundleDependencies": [ - "@napi-rs/wasm-runtime", - "@emnapi/core", - "@emnapi/runtime", - "@tybys/wasm-util", - "@emnapi/wasi-threads", - "tslib" - ], - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "^1.5.0", - "@emnapi/runtime": "^1.5.0", - "@emnapi/wasi-threads": "^1.1.0", - "@napi-rs/wasm-runtime": "^1.0.7", - "@tybys/wasm-util": "^0.10.1", - "tslib": "^2.4.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.15.tgz", - "integrity": "sha512-sJGE5faXnNQ1iXeqmRin7Ds/ru2fgCiaQZQQz3ZGIDtvbkeV85rAZ0QJFMDg0FrqsffZG96H1U9AQlNBRLsHVg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/oxide-win32-x64-msvc": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.1.15.tgz", - "integrity": "sha512-NLeHE7jUV6HcFKS504bpOohyi01zPXi2PXmjFfkzTph8xRxDdxkRsXm/xDO5uV5K3brrE1cCwbUYmFUSHR3u1w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tailwindcss/postcss": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/@tailwindcss/postcss/-/postcss-4.1.15.tgz", - "integrity": "sha512-IZh8IT76KujRz6d15wZw4eoeViT4TqmzVWNNfpuNCTKiaZUwgr5vtPqO4HjuYDyx3MgGR5qgPt1HMzTeLJyA3g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@alloc/quick-lru": "^5.2.0", - "@tailwindcss/node": "4.1.15", - "@tailwindcss/oxide": "4.1.15", - "postcss": "^8.4.41", - "tailwindcss": "4.1.15" - } - }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -7459,7 +7185,7 @@ "version": "19.2.2", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.2.tgz", "integrity": "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "csstype": "^3.0.2" @@ -10054,7 +9780,7 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/data-view-buffer": { @@ -10712,20 +10438,6 @@ "node": ">= 0.8" } }, - "node_modules/enhanced-resolve": { - "version": "5.18.3", - "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.3.tgz", - "integrity": "sha512-d4lC8xfavMeBjzGr2vECC3fsGXziXZQyJxD868h2M/mBI3PwAuODxAkLkq5HYuvrPYcUtiLzsTo8U3PgX3Ocww==", - "dev": true, - "license": "MIT", - "dependencies": { - "graceful-fs": "^4.2.4", - "tapable": "^2.2.0" - }, - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/entities": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.3.tgz", @@ -13751,7 +13463,7 @@ "version": "16.8.1", "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" @@ -14955,6 +14667,8 @@ "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", "dev": true, "license": "MIT", + "optional": true, + "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -16442,25 +16156,6 @@ "url": "https://opencollective.com/napi-postinstall" } }, - "node_modules/nativewind": { - "version": "5.0.0-preview.2", - "resolved": "https://registry.npmjs.org/nativewind/-/nativewind-5.0.0-preview.2.tgz", - "integrity": "sha512-rTNrwFIwl/n2VH7KPvsZj/NdvKf+uGHF4NYtPamr5qG2eTYGT8B8VeyCPfYf/xUskpWOLJVqVEXaFO/vuIDEdw==", - "license": "MIT", - "workspaces": [ - "example" - ], - "dependencies": { - "tailwindcss-safe-area": "^1.1.0" - }, - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "react-native-css": "^3.0.1", - "tailwindcss": ">4.1.11" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -17819,6 +17514,17 @@ "@babel/types": "^7.26.0" } }, + "node_modules/react-native-edge-to-edge": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/react-native-edge-to-edge/-/react-native-edge-to-edge-1.7.0.tgz", + "integrity": "sha512-ERegbsq28yoMndn/Uq49i4h6aAhMvTEjOfkFh50yX9H/dMjjCr/Tix/es/9JcPRvC+q7VzCMWfxWDUb6Jrq1OQ==", + "license": "MIT", + "peer": true, + "peerDependencies": { + "react": "*", + "react-native": "*" + } + }, "node_modules/react-native-fit-image": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/react-native-fit-image/-/react-native-fit-image-1.5.5.tgz", @@ -17869,6 +17575,18 @@ "react-native": ">=0.50.4" } }, + "node_modules/react-native-nitro-modules": { + "version": "0.31.1", + "resolved": "https://registry.npmjs.org/react-native-nitro-modules/-/react-native-nitro-modules-0.31.1.tgz", + "integrity": "sha512-ecr3bTWvLj5/Ce3BO54XRHy4uNHfFR2iVFLf9g/zDRkaD4iOg6kF9hatkIXwUPZi6apIf9P91BS+bco0lSIcVA==", + "hasInstallScript": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "react": "*", + "react-native": "*" + } + }, "node_modules/react-native-permissions": { "version": "5.4.2", "resolved": "https://registry.npmjs.org/react-native-permissions/-/react-native-permissions-5.4.2.tgz", @@ -17953,6 +17671,33 @@ "react-native": "*" } }, + "node_modules/react-native-unistyles": { + "version": "3.0.15", + "resolved": "https://registry.npmjs.org/react-native-unistyles/-/react-native-unistyles-3.0.15.tgz", + "integrity": "sha512-mbALO/lTc1oCWrCww79Zzipr1uVoePE2VXXgJO7zOKApK/hjBas03zpe3C4H6w7gBnD/z6TJYwTwQZt/yAWBeQ==", + "license": "MIT", + "workspaces": [ + "example", + "docs", + "expo-example" + ], + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "@react-native/normalize-colors": "*", + "react": "*", + "react-native": ">=0.76.0", + "react-native-edge-to-edge": "*", + "react-native-nitro-modules": "*", + "react-native-reanimated": "*" + }, + "peerDependenciesMeta": { + "react-native-reanimated": { + "optional": true + } + } + }, "node_modules/react-native-web": { "version": "0.21.2", "resolved": "https://registry.npmjs.org/react-native-web/-/react-native-web-0.21.2.tgz", @@ -19720,42 +19465,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/tailwindcss": { - "version": "4.1.15", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.1.15.tgz", - "integrity": "sha512-k2WLnWkYFkdpRv+Oby3EBXIyQC8/s1HOFMBUViwtAh6Z5uAozeUSMQlIsn/c6Q2iJzqG6aJT3wdPaRNj70iYxQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/tailwindcss-safe-area": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/tailwindcss-safe-area/-/tailwindcss-safe-area-1.1.0.tgz", - "integrity": "sha512-wuPUeW5BhWNv9yr3OzaGgpqImQG9FBM4mQIQh2C6yjHmOOZsJ3gh5RfNHt+TM16TMtpQs/8k2TWx8yQTFG7Fcw==", - "license": "MIT", - "engines": { - "node": ">=22" - }, - "funding": { - "url": "https://github.com/sponsors/mvllow" - }, - "peerDependencies": { - "tailwindcss": "^4.0.0" - } - }, - "node_modules/tapable": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.0.tgz", - "integrity": "sha512-g9ljZiwki/LfxmQADO3dEY1CbpmXT5Hm2fJ+QaGKwSXUylMybePR7/67YW7jOrrvjEgL1Fmz5kzyAjWVWLlucg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/webpack" - } - }, "node_modules/tar": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", @@ -21467,6 +21176,8 @@ "@react-navigation/elements": "^2.6.3", "@react-navigation/native": "^7.1.8", "@siteed/expo-audio-studio": "^2.18.1", + "@speechmatics/expo-two-way-audio": "^0.1.2", + "buffer": "^6.0.3", "expo": "^54.0.18", "expo-audio": "~1.0.13", "expo-av": "^16.0.7", @@ -21487,7 +21198,6 @@ "expo-updates": "~29.0.12", "expo-web-browser": "~15.0.8", "lucide-react-native": "^0.546.0", - "nativewind": "^5.0.0-preview.2", "react": "19.1.0", "react-dom": "19.1.0", "react-native": "0.81.4", @@ -21499,21 +21209,43 @@ "react-native-safe-area-context": "~5.6.0", "react-native-screens": "~4.16.0", "react-native-svg": "^15.14.0", + "react-native-unistyles": "^3.0.15", "react-native-web": "~0.21.0", "react-native-webrtc": "^124.0.7", "react-native-worklets": "0.5.1" }, "devDependencies": { - "@tailwindcss/postcss": "^4.1.15", "@types/react": "~19.2.0", "eas-cli": "^16.24.1", "eslint": "^9.25.0", "eslint-config-expo": "~10.0.0", - "postcss": "^8.4.49", - "tailwindcss": "^4.1.15", "typescript": "~5.9.2" } }, + "packages/app/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "packages/server": { "name": "@voice-dev/server", "version": "0.1.0", diff --git a/packages/app/app/_layout.tsx b/packages/app/app/_layout.tsx index f898ddb70..5e85d062d 100644 --- a/packages/app/app/_layout.tsx +++ b/packages/app/app/_layout.tsx @@ -1,4 +1,4 @@ -import '../global.css'; +import '@/styles/unistyles'; import { Stack } from 'expo-router'; import { StatusBar } from 'expo-status-bar'; import { SafeAreaProvider } from 'react-native-safe-area-context'; diff --git a/packages/app/app/audio-test.tsx b/packages/app/app/audio-test.tsx index 18de2ae93..615c28600 100644 --- a/packages/app/app/audio-test.tsx +++ b/packages/app/app/audio-test.tsx @@ -1,8 +1,231 @@ import { View, Text, Pressable, ScrollView, Alert } from 'react-native'; import { useState, useEffect } from 'react'; +import { StyleSheet } from 'react-native-unistyles'; import { useAudioRecorder } from '../hooks/use-audio-recorder'; import { useAudioPlayer } from '../hooks/use-audio-player'; import { getRecordingPermissionsAsync, requestRecordingPermissionsAsync } from 'expo-audio'; +import type { Theme } from '../styles/theme'; + +const styles = StyleSheet.create((theme: Theme) => ({ + scrollView: { + flex: 1, + backgroundColor: theme.colors.white, + }, + scrollViewDark: { + backgroundColor: theme.colors.black, + }, + container: { + padding: theme.spacing[6], + }, + title: { + fontSize: theme.fontSize['3xl'], + fontWeight: theme.fontWeight.bold, + color: theme.colors.gray[900], + marginBottom: theme.spacing[2], + }, + titleDark: { + color: theme.colors.white, + }, + subtitle: { + fontSize: theme.fontSize.base, + color: theme.colors.gray[600], + marginBottom: theme.spacing[8], + }, + subtitleDark: { + color: theme.colors.gray[400], + }, + section: { + marginBottom: theme.spacing[8], + }, + sectionTitle: { + fontSize: theme.fontSize.lg, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.gray[900], + marginBottom: theme.spacing[2], + }, + sectionTitleDark: { + color: theme.colors.white, + }, + permissionCard: { + padding: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + }, + permissionGranted: { + backgroundColor: theme.colors.green[100], + }, + permissionGrantedDark: { + backgroundColor: theme.colors.green[900], + }, + permissionDenied: { + backgroundColor: theme.colors.red[100], + }, + permissionDeniedDark: { + backgroundColor: theme.colors.red[900], + }, + permissionText: { + fontSize: theme.fontSize.base, + }, + permissionTextGranted: { + color: theme.colors.green[800], + }, + permissionTextGrantedDark: { + color: theme.colors.green[200], + }, + permissionTextDenied: { + color: theme.colors.red[800], + }, + permissionTextDeniedDark: { + color: theme.colors.red[200], + }, + statusCard: { + marginBottom: theme.spacing[4], + padding: theme.spacing[4], + backgroundColor: theme.colors.gray[100], + borderRadius: theme.borderRadius.lg, + }, + statusCardDark: { + backgroundColor: theme.colors.gray[800], + }, + statusLabel: { + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.gray[700], + marginBottom: theme.spacing[1], + }, + statusLabelDark: { + color: theme.colors.gray[300], + }, + statusValue: { + fontSize: theme.fontSize.base, + color: theme.colors.gray[900], + }, + statusValueDark: { + color: theme.colors.white, + }, + controlsRow: { + flexDirection: 'row', + gap: theme.spacing[3], + marginBottom: theme.spacing[4], + }, + button: { + flex: 1, + padding: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + alignItems: 'center', + }, + buttonBlue: { + backgroundColor: theme.colors.blue[500], + }, + buttonBlueDark: { + backgroundColor: theme.colors.blue[600], + }, + buttonRed: { + backgroundColor: theme.colors.red[500], + }, + buttonRedDark: { + backgroundColor: theme.colors.red[600], + }, + buttonGreen: { + backgroundColor: theme.colors.green[500], + }, + buttonGreenDark: { + backgroundColor: theme.colors.green[600], + }, + buttonPurple: { + backgroundColor: theme.colors.purple[500], + }, + buttonPurpleDark: { + backgroundColor: theme.colors.purple[600], + }, + buttonOrange: { + backgroundColor: theme.colors.orange[500], + }, + buttonOrangeDark: { + backgroundColor: theme.colors.orange[600], + }, + buttonDisabled: { + backgroundColor: theme.colors.gray[300], + }, + buttonDisabledDark: { + backgroundColor: theme.colors.gray[700], + }, + buttonText: { + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.white, + }, + buttonTextDisabled: { + color: theme.colors.gray[500], + }, + buttonTextDisabledDark: { + color: theme.colors.gray[400], + }, + fullWidthButton: { + padding: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + alignItems: 'center', + }, + infoCard: { + padding: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + backgroundColor: theme.colors.blue[50], + }, + infoCardDark: { + backgroundColor: theme.colors.blue[950], + }, + infoCardLabel: { + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.blue[700], + marginBottom: theme.spacing[1], + }, + infoCardLabelDark: { + color: theme.colors.blue[300], + }, + infoCardValue: { + fontSize: theme.fontSize.base, + color: theme.colors.blue[900], + }, + infoCardValueDark: { + color: theme.colors.blue[100], + }, + infoCardSubtext: { + fontSize: theme.fontSize.sm, + color: theme.colors.blue[600], + marginTop: theme.spacing[1], + }, + infoCardSubtextDark: { + color: theme.colors.blue[400], + }, + configCard: { + padding: theme.spacing[4], + backgroundColor: theme.colors.gray[50], + borderRadius: theme.borderRadius.lg, + }, + configCardDark: { + backgroundColor: theme.colors.gray[900], + }, + configTitle: { + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + color: theme.colors.gray[700], + marginBottom: theme.spacing[2], + }, + configTitleDark: { + color: theme.colors.gray[300], + }, + configText: { + fontSize: theme.fontSize.sm, + color: theme.colors.gray[600], + marginBottom: theme.spacing[1], + }, + configTextDark: { + color: theme.colors.gray[400], + }, + gap3: { + gap: theme.spacing[3], + }, +})); export default function AudioTestScreen() { const [permissionGranted, setPermissionGranted] = useState(false); @@ -124,40 +347,46 @@ export default function AudioTestScreen() { } return ( - - + + {/* Header */} - + Audio Test - + Test audio recording and playback functionality {/* Permission Status */} - - + + Permission Status - - + + {permissionGranted ? '✓ Microphone permission granted' : '✗ Microphone permission denied'} {/* Recording Section */} - - + + Audio Recording {/* Recording Status */} - - + + Status - + {recordingStatus === 'idle' && 'Ready to record'} {recordingStatus === 'recording' && '🔴 Recording...'} {recordingStatus === 'processing' && 'Processing...'} @@ -165,21 +394,21 @@ export default function AudioTestScreen() { {/* Recording Controls */} - + - + Start Recording @@ -187,17 +416,17 @@ export default function AudioTestScreen() { - + Stop Recording @@ -205,14 +434,14 @@ export default function AudioTestScreen() { {/* Last Recording Info */} {lastRecordingSize !== null && ( - - + + Last Recording - + Size: {lastRecordingSize} bytes - + Type: audio/m4a @@ -220,55 +449,55 @@ export default function AudioTestScreen() { {/* Playback Section */} - - + + Audio Playback {/* Playback Status */} - - + + Status - + {playbackStatus} {/* Playback Controls */} - + - + Play Last Recording - + Play Test Audio - + Stop Playback @@ -276,20 +505,20 @@ export default function AudioTestScreen() { {/* Audio Settings Info */} - - + + Audio Configuration - + • Sample Rate: 16000 Hz - + • Channels: 1 (Mono) - + • Format: M4A/AAC - + • Optimized for voice/speech diff --git a/packages/app/app/index.tsx b/packages/app/app/index.tsx index 3ec720654..8b6b8434d 100644 --- a/packages/app/app/index.tsx +++ b/packages/app/app/index.tsx @@ -1,77 +1,111 @@ -import { useEffect, useState, useRef } from 'react'; -import { View, Pressable, Text, TextInput, Platform, KeyboardAvoidingView, ScrollView, Animated } from 'react-native'; -import { router } from 'expo-router'; -import { activateKeepAwakeAsync, deactivateKeepAwake } from 'expo-keep-awake'; -import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { useEffect, useState, useRef } from "react"; +import { + View, + Pressable, + Text, + TextInput, + Platform, + KeyboardAvoidingView, + ScrollView, + Animated, +} from "react-native"; +import { router } from "expo-router"; +import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { StyleSheet } from "react-native-unistyles"; +import type { Theme } from "@/styles/theme"; // Simple unique ID generator let messageIdCounter = 0; function generateMessageId(): string { return `msg_${Date.now()}_${messageIdCounter++}`; } -import { useWebSocket } from '@/hooks/use-websocket'; -import { useAudioRecorder } from '@/hooks/use-audio-recorder'; -import { useAudioPlayer } from '@/hooks/use-audio-player'; -import { useRealtimeAudio } from '@/hooks/use-realtime-audio'; -import { useSettings } from '@/hooks/use-settings'; -import { ConnectionStatus } from '@/components/connection-status'; -import { UserMessage, AssistantMessage, ActivityLog, ToolCall } from '@/components/message'; -import { ArtifactDrawer, type Artifact } from '@/components/artifact-drawer'; -import { ActiveProcesses } from '@/components/active-processes'; -import { AgentStreamView } from '@/components/agent-stream-view'; -import { ConversationSelector } from '@/components/conversation-selector'; -import { Settings, Mic, ArrowUp, Square, AudioLines } from 'lucide-react-native'; +import { useWebSocket } from "@/hooks/use-websocket"; +import { useAudioRecorder } from "@/hooks/use-audio-recorder"; +import { useAudioPlayer } from "@/hooks/use-audio-player"; +import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio"; +import { useSettings } from "@/hooks/use-settings"; +import { ConnectionStatus } from "@/components/connection-status"; +import { + UserMessage, + AssistantMessage, + ActivityLog, + ToolCall, +} from "@/components/message"; +import { ArtifactDrawer, type Artifact } from "@/components/artifact-drawer"; +import { ActiveProcesses } from "@/components/active-processes"; +import { AgentStreamView } from "@/components/agent-stream-view"; +import { ConversationSelector } from "@/components/conversation-selector"; +import { + Settings, + Mic, + ArrowUp, + Square, + AudioLines, +} from "lucide-react-native"; import type { ActivityLogPayload, SessionInboundMessage, WSInboundMessage, -} from '@server/server/messages'; -import type { AgentStatus } from '@server/server/acp/types'; -import type { SessionNotification } from '@agentclientprotocol/sdk'; +} from "@server/server/messages"; +import type { AgentStatus } from "@server/server/acp/types"; +import type { SessionNotification } from "@agentclientprotocol/sdk"; type MessageEntry = | { - type: 'user'; + type: "user"; id: string; timestamp: number; message: string; } | { - type: 'assistant'; + type: "assistant"; id: string; timestamp: number; message: string; } | { - type: 'activity'; + type: "activity"; id: string; timestamp: number; - activityType: 'system' | 'info' | 'success' | 'error'; + activityType: "system" | "info" | "success" | "error"; message: string; metadata?: Record; } | { - type: 'tool_call'; + type: "artifact"; + id: string; + timestamp: number; + artifactId: string; + artifactType: string; + title: string; + } + | { + type: "tool_call"; id: string; timestamp: number; toolName: string; args: any; result?: any; error?: any; - status: 'executing' | 'completed' | 'failed'; + status: "executing" | "completed" | "failed"; }; -type ViewMode = 'orchestrator' | 'agent'; +type ViewMode = "orchestrator" | "agent"; interface Agent { id: string; status: AgentStatus; createdAt: Date; - type: 'claude'; + type: "claude"; sessionId?: string; error?: string; currentModeId?: string; - availableModes?: Array<{ id: string; name: string; description?: string | null }>; + availableModes?: Array<{ + id: string; + name: string; + description?: string | null; + }>; } interface Command { @@ -98,81 +132,85 @@ export default function VoiceAssistantScreen() { const pulseAnim = useRef(new Animated.Value(1)).current; const audioRecorder = useAudioRecorder(); - const audioPlayer = useAudioPlayer({ useSpeaker: settings.useSpeaker }); + const audioPlayer = useAudioPlayer(); const insets = useSafeAreaInsets(); - // Realtime audio with WebRTC - const realtimeAudio = useRealtimeAudio({ + // Realtime audio with Speechmatics (echo cancellation) + const realtimeAudio = useSpeechmaticsAudio({ onSpeechStart: () => { - console.log('[App] Speech detected'); - // Pause audio playback if playing + console.log("[App] Speech detected"); + // Stop audio playback if playing if (isPlayingAudio) { - audioPlayer.pause(); + audioPlayer.stop(); } }, onSpeechEnd: () => { - console.log('[App] Speech ended'); + console.log("[App] Speech ended"); }, onAudioSegment: (base64Audio: string) => { - console.log('[App] Sending audio segment, length:', base64Audio.length); + console.log("[App] Sending audio segment, length:", base64Audio.length); // Send audio segment to server try { ws.send({ - type: 'session', + type: "session", message: { - type: 'audio_chunk', + type: "audio_chunk", audio: base64Audio, - format: 'audio/pcm', + format: "audio/wav", isLast: true, // Complete segment }, }); } catch (error) { - console.error('[App] Failed to send audio segment:', error); + console.error("[App] Failed to send audio segment:", error); } }, onError: (error) => { - console.error('[App] Realtime audio error:', error); + console.error("[App] Realtime audio error:", error); setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'error', + activityType: "error", message: `Realtime audio error: ${error.message}`, }, ]); }, - volumeThreshold: 0.02, + volumeThreshold: 0.3, silenceDuration: 1000, + speechConfirmationDuration: 300, + detectionGracePeriod: 200, }); const [messages, setMessages] = useState([]); - const [currentAssistantMessage, setCurrentAssistantMessage] = useState(''); + const [currentAssistantMessage, setCurrentAssistantMessage] = useState(""); const [isProcessingAudio, setIsProcessingAudio] = useState(false); const [isPlayingAudio, setIsPlayingAudio] = useState(false); const [isRecording, setIsRecording] = useState(false); - const [userInput, setUserInput] = useState(''); + const [userInput, setUserInput] = useState(""); // Artifact state const [artifacts, setArtifacts] = useState>(new Map()); const [currentArtifact, setCurrentArtifact] = useState(null); // Multi-view navigation state - const [viewMode, setViewMode] = useState('orchestrator'); + const [viewMode, setViewMode] = useState("orchestrator"); const [activeAgentId, setActiveAgentId] = useState(null); // Agent and command state const [agents, setAgents] = useState>(new Map()); const [commands, setCommands] = useState>(new Map()); - const [agentUpdates, setAgentUpdates] = useState>(new Map()); + const [agentUpdates, setAgentUpdates] = useState>( + new Map() + ); const scrollViewRef = useRef(null); // Pulse animation for speech indicator useEffect(() => { - if (realtimeAudio.isSpeaking) { + if (realtimeAudio.isSpeaking || realtimeAudio.isDetecting) { Animated.loop( Animated.sequence([ Animated.timing(pulseAnim, { @@ -191,16 +229,16 @@ export default function VoiceAssistantScreen() { pulseAnim.stopAnimation(); pulseAnim.setValue(1); } - }, [realtimeAudio.isSpeaking, pulseAnim]); + }, [realtimeAudio.isSpeaking, realtimeAudio.isDetecting, pulseAnim]); // Keep screen awake if setting is enabled (mobile only) useEffect(() => { - if (Platform.OS === 'web') return; + if (Platform.OS === "web") return; if (settings.keepScreenOn) { - activateKeepAwakeAsync('voice-assistant'); + activateKeepAwakeAsync("voice-assistant"); } else { - deactivateKeepAwake('voice-assistant'); + deactivateKeepAwake("voice-assistant"); } }, [settings.keepScreenOn]); @@ -212,11 +250,17 @@ export default function VoiceAssistantScreen() { // WebSocket message handlers useEffect(() => { // Session state handler - initial agents/commands - const unsubSessionState = ws.on('session_state', (message) => { - if (message.type !== 'session_state') return; + const unsubSessionState = ws.on("session_state", (message) => { + if (message.type !== "session_state") return; const { agents: agentsList, commands: commandsList } = message.payload; - console.log('[App] Session state received:', agentsList.length, 'agents,', commandsList.length, 'commands'); + console.log( + "[App] Session state received:", + agentsList.length, + "agents,", + commandsList.length, + "commands" + ); // Update agents setAgents(new Map(agentsList.map((a) => [a.id, a as Agent]))); @@ -226,11 +270,12 @@ export default function VoiceAssistantScreen() { }); // Agent created handler - const unsubAgentCreated = ws.on('agent_created', (message) => { - if (message.type !== 'agent_created') return; - const { agentId, status, type, currentModeId, availableModes } = message.payload; + const unsubAgentCreated = ws.on("agent_created", (message) => { + if (message.type !== "agent_created") return; + const { agentId, status, type, currentModeId, availableModes } = + message.payload; - console.log('[App] Agent created:', agentId); + console.log("[App] Agent created:", agentId); const agent: Agent = { id: agentId, @@ -246,11 +291,11 @@ export default function VoiceAssistantScreen() { }); // Agent update handler - const unsubAgentUpdate = ws.on('agent_update', (message) => { - if (message.type !== 'agent_update') return; + const unsubAgentUpdate = ws.on("agent_update", (message) => { + if (message.type !== "agent_update") return; const { agentId, timestamp, notification } = message.payload; - console.log('[App] Agent update:', agentId); + console.log("[App] Agent update:", agentId); setAgentUpdates((prev) => { const updated = new Map(prev); @@ -267,11 +312,11 @@ export default function VoiceAssistantScreen() { }); // Agent status handler - const unsubAgentStatus = ws.on('agent_status', (message) => { - if (message.type !== 'agent_status') return; + const unsubAgentStatus = ws.on("agent_status", (message) => { + if (message.type !== "agent_status") return; const { agentId, status, info } = message.payload; - console.log('[App] Agent status changed:', agentId, status); + console.log("[App] Agent status changed:", agentId, status); setAgents((prev) => { const updated = new Map(prev); @@ -289,12 +334,12 @@ export default function VoiceAssistantScreen() { }); // Activity log handler - const unsubActivity = ws.on('activity_log', (message) => { - if (message.type !== 'activity_log') return; + const unsubActivity = ws.on("activity_log", (message) => { + if (message.type !== "activity_log") return; const data = message.payload; // Handle tool calls - if (data.type === 'tool_call' && data.metadata) { + if (data.type === "tool_call" && data.metadata) { const { toolCallId, toolName, @@ -308,19 +353,19 @@ export default function VoiceAssistantScreen() { setMessages((prev) => [ ...prev, { - type: 'tool_call', + type: "tool_call", id: toolCallId, timestamp: Date.now(), toolName, args, - status: 'executing', + status: "executing", }, ]); return; } // Handle tool results - if (data.type === 'tool_result' && data.metadata) { + if (data.type === "tool_result" && data.metadata) { const { toolCallId, result } = data.metadata as { toolCallId: string; result: unknown; @@ -328,8 +373,8 @@ export default function VoiceAssistantScreen() { setMessages((prev) => prev.map((msg) => - msg.type === 'tool_call' && msg.id === toolCallId - ? { ...msg, result, status: 'completed' as const } + msg.type === "tool_call" && msg.id === toolCallId + ? { ...msg, result, status: "completed" as const } : msg ) ); @@ -338,9 +383,9 @@ export default function VoiceAssistantScreen() { // Handle tool errors if ( - data.type === 'error' && + data.type === "error" && data.metadata && - 'toolCallId' in data.metadata + "toolCallId" in data.metadata ) { const { toolCallId, error } = data.metadata as { toolCallId: string; @@ -349,23 +394,23 @@ export default function VoiceAssistantScreen() { setMessages((prev) => prev.map((msg) => - msg.type === 'tool_call' && msg.id === toolCallId - ? { ...msg, error, status: 'failed' as const } + msg.type === "tool_call" && msg.id === toolCallId + ? { ...msg, error, status: "failed" as const } : msg ) ); } // Map activity types to message types - let activityType: 'system' | 'info' | 'success' | 'error' = 'info'; - if (data.type === 'error') activityType = 'error'; + let activityType: "system" | "info" | "success" | "error" = "info"; + if (data.type === "error") activityType = "error"; // Add user transcripts as user messages - if (data.type === 'transcript') { + if (data.type === "transcript") { setMessages((prev) => [ ...prev, { - type: 'user', + type: "user", id: generateMessageId(), timestamp: Date.now(), message: data.content, @@ -375,17 +420,17 @@ export default function VoiceAssistantScreen() { } // Add assistant messages - if (data.type === 'assistant') { + if (data.type === "assistant") { setMessages((prev) => [ ...prev, { - type: 'assistant', + type: "assistant", id: generateMessageId(), timestamp: Date.now(), message: data.content, }, ]); - setCurrentAssistantMessage(''); + setCurrentAssistantMessage(""); return; } @@ -393,7 +438,7 @@ export default function VoiceAssistantScreen() { setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), activityType, @@ -404,35 +449,34 @@ export default function VoiceAssistantScreen() { }); // Assistant chunk handler (streaming) - const unsubChunk = ws.on('assistant_chunk', (message) => { - if (message.type !== 'assistant_chunk') return; + const unsubChunk = ws.on("assistant_chunk", (message) => { + if (message.type !== "assistant_chunk") return; setCurrentAssistantMessage((prev) => prev + message.payload.chunk); }); // Transcription result handler - const unsubTranscription = ws.on('transcription_result', (message) => { - if (message.type !== 'transcription_result') return; + const unsubTranscription = ws.on("transcription_result", (message) => { + if (message.type !== "transcription_result") return; setIsProcessingAudio(false); const transcriptText = message.payload.text.trim(); if (!transcriptText) { - // Empty transcription - false positive, resume playback - console.log('[App] Empty transcription (false positive) - resuming playback'); - audioPlayer.resume(); + // Empty transcription - false positive, let playback continue + console.log("[App] Empty transcription (false positive) - ignoring"); } else { // Has content - real speech detected, stop playback - console.log('[App] Transcription received - stopping playback'); + console.log("[App] Transcription received - stopping playback"); audioPlayer.stop(); setIsPlayingAudio(false); - setCurrentAssistantMessage(''); + setCurrentAssistantMessage(""); } }); // Audio output handler (TTS) - const unsubAudioOutput = ws.on('audio_output', async (message) => { - if (message.type !== 'audio_output') return; + const unsubAudioOutput = ws.on("audio_output", async (message) => { + if (message.type !== "audio_output") return; const data = message.payload; try { @@ -440,13 +484,13 @@ export default function VoiceAssistantScreen() { // Create blob-like object with correct mime type (React Native compatible) const mimeType = - data.format === 'mp3' ? 'audio/mpeg' : `audio/${data.format}`; + data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`; const base64Audio = data.audio; // Create a Blob-like object that works in React Native const audioBlob = { type: mimeType, - size: Math.ceil(base64Audio.length * 3 / 4), // Approximate size from base64 + size: Math.ceil((base64Audio.length * 3) / 4), // Approximate size from base64 arrayBuffer: async () => { // Convert base64 to ArrayBuffer const binaryString = atob(base64Audio); @@ -463,9 +507,9 @@ export default function VoiceAssistantScreen() { // Send confirmation back to server (properly typed) const confirmMessage: WSInboundMessage = { - type: 'session', + type: "session", message: { - type: 'audio_played', + type: "audio_played", id: data.id, }, }; @@ -473,14 +517,14 @@ export default function VoiceAssistantScreen() { setIsPlayingAudio(false); } catch (error: any) { - console.error('[App] Audio playback error:', error); + console.error("[App] Audio playback error:", error); setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'error', + activityType: "error", message: `Audio playback failed: ${error.message}`, }, ]); @@ -489,48 +533,53 @@ export default function VoiceAssistantScreen() { }); // Status handler - const unsubStatus = ws.on('status', (message) => { - if (message.type !== 'status') return; + const unsubStatus = ws.on("status", (message) => { + if (message.type !== "status") return; const msg = - 'message' in message.payload + "message" in message.payload ? String(message.payload.message) : `Status: ${message.payload.status}`; setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'info', + activityType: "info", message: msg, }, ]); }); // Conversation loaded handler - const unsubConversationLoaded = ws.on('conversation_loaded', (message) => { - if (message.type !== 'conversation_loaded') return; + const unsubConversationLoaded = ws.on("conversation_loaded", (message) => { + if (message.type !== "conversation_loaded") return; const { conversationId, messageCount } = message.payload; setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'success', + activityType: "success", message: `Loaded conversation with ${messageCount} messages`, }, ]); }); // Artifact handler - const unsubArtifact = ws.on('artifact', (message) => { - if (message.type !== 'artifact') return; + const unsubArtifact = ws.on("artifact", (message) => { + if (message.type !== "artifact") return; const artifactData = message.payload; - console.log('[App] Received artifact:', artifactData.id, artifactData.type, artifactData.title); + console.log( + "[App] Received artifact:", + artifactData.id, + artifactData.type, + artifactData.title + ); // Store artifact setArtifacts((prev) => { @@ -539,6 +588,19 @@ export default function VoiceAssistantScreen() { return updated; }); + // Add artifact entry to chat history + setMessages((prev) => [ + ...prev, + { + type: "artifact", + id: generateMessageId(), + timestamp: Date.now(), + artifactId: artifactData.id, + artifactType: artifactData.type, + title: artifactData.title, + }, + ]); + // Show drawer immediately setCurrentArtifact(artifactData); }); @@ -564,22 +626,22 @@ export default function VoiceAssistantScreen() { setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'success', - message: 'WebSocket connected', + activityType: "success", + message: "WebSocket connected", }, ]); } else if (messages.length > 0) { setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'error', - message: 'WebSocket disconnected', + activityType: "error", + message: "WebSocket disconnected", }, ]); } @@ -592,11 +654,11 @@ export default function VoiceAssistantScreen() { // If recording, stop and send if (isRecording) { try { - console.log('[App] Stopping recording...'); + console.log("[App] Stopping recording..."); const audioBlob = await audioRecorder.stop(); setIsRecording(false); - const format = audioBlob.type || 'audio/m4a'; + const format = audioBlob.type || "audio/m4a"; console.log( `[App] Recording complete: ${audioBlob.size} bytes, format: ${format}` ); @@ -605,11 +667,11 @@ export default function VoiceAssistantScreen() { setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'info', - message: 'Sending audio to server...', + activityType: "info", + message: "Sending audio to server...", }, ]); @@ -618,15 +680,15 @@ export default function VoiceAssistantScreen() { const base64Audio = btoa( new Uint8Array(arrayBuffer).reduce( (data, byte) => data + String.fromCharCode(byte), - '' + "" ) ); // Send to server (properly typed) const audioMessage: WSInboundMessage = { - type: 'session', + type: "session", message: { - type: 'audio_chunk', + type: "audio_chunk", audio: base64Audio, format: format, isLast: true, @@ -638,14 +700,14 @@ export default function VoiceAssistantScreen() { `[App] Sent audio: ${audioBlob.size} bytes, format: ${format}` ); } catch (error: any) { - console.error('[App] Recording error:', error); + console.error("[App] Recording error:", error); setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'error', + activityType: "error", message: `Failed to record audio: ${error.message}`, }, ]); @@ -654,7 +716,7 @@ export default function VoiceAssistantScreen() { } else { // Start recording try { - console.log('[App] Starting recording...'); + console.log("[App] Starting recording..."); // Stop any currently playing audio audioPlayer.stop(); @@ -663,14 +725,14 @@ export default function VoiceAssistantScreen() { await audioRecorder.start(); setIsRecording(true); } catch (error: any) { - console.error('[App] Failed to start recording:', error); + console.error("[App] Failed to start recording:", error); setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'error', + activityType: "error", message: `Failed to start recording: ${error.message}`, }, ]); @@ -682,10 +744,10 @@ export default function VoiceAssistantScreen() { function handleArtifactClick(artifactId: string) { const artifact = artifacts.get(artifactId); if (artifact) { - console.log('[App] Opening artifact:', artifactId); + console.log("[App] Opening artifact:", artifactId); setCurrentArtifact(artifact); } else { - console.warn('[App] Artifact not found:', artifactId); + console.warn("[App] Artifact not found:", artifactId); } } @@ -697,23 +759,23 @@ export default function VoiceAssistantScreen() { // Handle agent selection function handleSelectAgent(agentId: string) { setActiveAgentId(agentId); - setViewMode('agent'); + setViewMode("agent"); } // Handle back to orchestrator function handleBackToOrchestrator() { setActiveAgentId(null); - setViewMode('orchestrator'); + setViewMode("orchestrator"); } // Agent control handlers function handleKillAgent(agentId: string) { - console.log('[App] Kill agent:', agentId); + console.log("[App] Kill agent:", agentId); // TODO: Implement kill agent API call } function handleCancelAgent(agentId: string) { - console.log('[App] Cancel agent:', agentId); + console.log("[App] Cancel agent:", agentId); // TODO: Implement cancel agent API call } @@ -729,28 +791,28 @@ export default function VoiceAssistantScreen() { ws.sendUserMessage(userInput); // Clear input and reset streaming state - setUserInput(''); - setCurrentAssistantMessage(''); + setUserInput(""); + setCurrentAssistantMessage(""); } function handleCancel() { - console.log('[App] Cancelling operations...'); + console.log("[App] Cancelling operations..."); // Stop audio playback audioPlayer.stop(); setIsPlayingAudio(false); // Clear streaming state - setCurrentAssistantMessage(''); + setCurrentAssistantMessage(""); // Reset processing state setIsProcessingAudio(false); // Send abort request to server (properly typed) const abortMessage: WSInboundMessage = { - type: 'session', + type: "session", message: { - type: 'abort_request', + type: "abort_request", }, }; ws.send(abortMessage); @@ -758,11 +820,11 @@ export default function VoiceAssistantScreen() { setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'info', - message: 'Operations cancelled', + activityType: "info", + message: "Operations cancelled", }, ]); } @@ -796,27 +858,27 @@ export default function VoiceAssistantScreen() { try { await realtimeAudio.start(); setIsRealtimeMode(true); - console.log('[App] Realtime mode enabled'); + console.log("[App] Realtime mode enabled"); setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'success', - message: 'Realtime mode started - speak anytime!', + activityType: "success", + message: "Realtime mode started - speak anytime!", }, ]); } catch (error: any) { - console.error('[App] Failed to start realtime mode:', error); + console.error("[App] Failed to start realtime mode:", error); setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'error', + activityType: "error", message: `Failed to start realtime mode: ${error.message}`, }, ]); @@ -826,20 +888,20 @@ export default function VoiceAssistantScreen() { try { await realtimeAudio.stop(); setIsRealtimeMode(false); - console.log('[App] Realtime mode disabled'); + console.log("[App] Realtime mode disabled"); setMessages((prev) => [ ...prev, { - type: 'activity', + type: "activity", id: generateMessageId(), timestamp: Date.now(), - activityType: 'info', - message: 'Realtime mode stopped', + activityType: "info", + message: "Realtime mode stopped", }, ]); } catch (error: any) { - console.error('[App] Failed to stop realtime mode:', error); + console.error("[App] Failed to stop realtime mode:", error); } } } @@ -848,14 +910,14 @@ export default function VoiceAssistantScreen() { function handleSelectConversation(newConversationId: string | null) { // Clear all state setMessages([]); - setCurrentAssistantMessage(''); - setUserInput(''); + setCurrentAssistantMessage(""); + setUserInput(""); setArtifacts(new Map()); setCurrentArtifact(null); setAgents(new Map()); setCommands(new Map()); setAgentUpdates(new Map()); - setViewMode('orchestrator'); + setViewMode("orchestrator"); setActiveAgentId(null); // Stop any ongoing operations @@ -872,14 +934,14 @@ export default function VoiceAssistantScreen() { } // Render agent stream view - if (viewMode === 'agent' && activeAgentId) { + if (viewMode === "agent" && activeAgentId) { const agent = agents.get(activeAgentId); const updates = agentUpdates.get(activeAgentId) || []; if (!agent) { return ( - - Agent not found + + Agent not found ); } @@ -900,24 +962,24 @@ export default function VoiceAssistantScreen() { return ( {/* Connection status header with buttons */} - - + + - + router.push('/settings')} - className="bg-zinc-800 p-3 rounded-lg" + onPress={() => router.push("/settings")} + style={styles.settingsButton} > @@ -930,7 +992,7 @@ export default function VoiceAssistantScreen() { agents={Array.from(agents.values())} commands={Array.from(commands.values())} activeProcessId={activeAgentId} - activeProcessType={activeAgentId ? 'agent' : null} + activeProcessType={activeAgentId ? "agent" : null} onSelectAgent={handleSelectAgent} onBackToOrchestrator={handleBackToOrchestrator} /> @@ -938,131 +1000,146 @@ export default function VoiceAssistantScreen() { {/* Messages */} - {messages.map((msg) => { - if (msg.type === 'user') { - return ( - - ); - } + {messages.map((msg) => { + if (msg.type === "user") { + return ( + + ); + } - if (msg.type === 'assistant') { - return ( - - ); - } + if (msg.type === "assistant") { + return ( + + ); + } - if (msg.type === 'activity') { - return ( - - ); - } + if (msg.type === "activity") { + return ( + + ); + } - if (msg.type === 'tool_call') { - return ( - - ); - } + if (msg.type === "artifact") { + return ( + + ); + } - return null; - })} + if (msg.type === "tool_call") { + return ( + + ); + } - {/* Streaming assistant message */} - {currentAssistantMessage && ( - - )} + return null; + })} + + {/* Streaming assistant message */} + {currentAssistantMessage && ( + + )} {/* Input area */} - {/* Text input */} - + {/* Text input */} + - {/* Buttons */} - - {/* Realtime mode button */} - - - - - + {/* Buttons */} + + {/* Realtime mode button */} + + + + + - {/* Main action button */} - - {isInProgress ? ( - - ) : isRecording ? ( - - ) : userInput.trim() ? ( - - ) : ( - - )} - - + {/* Main action button */} + + {isInProgress ? ( + + ) : isRecording ? ( + + ) : userInput.trim() ? ( + + ) : ( + + )} + + {/* Artifact drawer */} @@ -1073,3 +1150,98 @@ export default function VoiceAssistantScreen() { ); } + +const styles = StyleSheet.create((theme: Theme) => ({ + container: { + flex: 1, + backgroundColor: theme.colors.black, + }, + agentNotFoundContainer: { + flex: 1, + backgroundColor: theme.colors.black, + alignItems: "center", + justifyContent: "center", + }, + agentNotFoundText: { + color: theme.colors.white, + }, + headerRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + paddingHorizontal: theme.spacing[6], + paddingBottom: theme.spacing[4], + }, + headerLeft: { + flex: 1, + }, + headerRight: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + settingsButton: { + backgroundColor: theme.colors.zinc[800], + padding: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + }, + scrollView: { + flex: 1, + }, + scrollContent: { + paddingBottom: theme.spacing[4], + }, + inputArea: { + paddingTop: theme.spacing[4], + paddingHorizontal: theme.spacing[6], + borderTopWidth: theme.borderWidth[1], + borderTopColor: theme.colors.zinc[800], + backgroundColor: theme.colors.black, + }, + textInput: { + backgroundColor: theme.colors.zinc[800], + color: theme.colors.white, + borderRadius: theme.borderRadius["2xl"], + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[3], + marginBottom: theme.spacing[4], + maxHeight: 128, + }, + buttonRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "flex-end", + gap: theme.spacing[3], + }, + realtimeButton: { + width: 48, + height: 48, + borderRadius: theme.borderRadius.full, + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.zinc[800], + }, + realtimeButtonActive: { + backgroundColor: theme.colors.blue[600], + }, + mainButton: { + width: 48, + height: 48, + borderRadius: theme.borderRadius.full, + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.zinc[700], + }, + mainButtonRecording: { + backgroundColor: theme.colors.red[500], + }, + mainButtonInProgress: { + backgroundColor: theme.colors.red[600], + }, + mainButtonWithText: { + backgroundColor: theme.colors.blue[600], + }, + buttonDisabled: { + opacity: theme.opacity[50], + }, +})); diff --git a/packages/app/app/settings.tsx b/packages/app/app/settings.tsx index e47c61ba1..c7b1a5f7c 100644 --- a/packages/app/app/settings.tsx +++ b/packages/app/app/settings.tsx @@ -1,8 +1,235 @@ -import { useState, useEffect } from 'react'; -import { View, Text, ScrollView, TextInput, Switch, Pressable, Alert, ActivityIndicator } from 'react-native'; -import { router } from 'expo-router'; -import { useSafeAreaInsets } from 'react-native-safe-area-context'; -import { useSettings } from '@/hooks/use-settings'; +import { useState, useEffect } from "react"; +import { + View, + Text, + ScrollView, + TextInput, + Switch, + Pressable, + Alert, + ActivityIndicator, +} from "react-native"; +import { router } from "expo-router"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { StyleSheet } from "react-native-unistyles"; +import { useSettings } from "@/hooks/use-settings"; +import type { Theme } from "@/styles/theme"; + +const styles = StyleSheet.create((theme: Theme) => ({ + loadingContainer: { + flex: 1, + backgroundColor: theme.colors.black, + alignItems: "center", + justifyContent: "center", + }, + loadingText: { + color: theme.colors.white, + fontSize: theme.fontSize.lg, + }, + container: { + flex: 1, + backgroundColor: theme.colors.black, + }, + header: { + paddingHorizontal: theme.spacing[6], + paddingBottom: theme.spacing[4], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.zinc[800], + }, + headerRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + }, + headerTitle: { + color: theme.colors.white, + fontSize: theme.fontSize["3xl"], + fontWeight: theme.fontWeight.bold, + }, + cancelButton: { + color: theme.colors.blue[500], + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + }, + scrollView: { + flex: 1, + }, + content: { + padding: theme.spacing[6], + }, + section: { + marginBottom: theme.spacing[8], + }, + sectionTitle: { + color: theme.colors.white, + fontSize: theme.fontSize.lg, + fontWeight: theme.fontWeight.semibold, + marginBottom: theme.spacing[4], + }, + label: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.sm, + marginBottom: theme.spacing[2], + }, + input: { + backgroundColor: theme.colors.zinc[900], + color: theme.colors.white, + padding: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + marginBottom: theme.spacing[2], + }, + helperText: { + color: theme.colors.zinc[500], + fontSize: theme.fontSize.xs, + marginBottom: theme.spacing[3], + }, + testButton: { + padding: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + marginBottom: theme.spacing[3], + flexDirection: "row", + alignItems: "center", + justifyContent: "center", + backgroundColor: theme.colors.blue[600], + }, + testButtonDisabled: { + backgroundColor: theme.colors.zinc[800], + }, + testButtonText: { + color: theme.colors.white, + fontWeight: theme.fontWeight.semibold, + marginLeft: theme.spacing[2], + }, + testResultSuccess: { + padding: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + backgroundColor: "#14532d", + borderWidth: theme.borderWidth[1], + borderColor: "#15803d", + }, + testResultError: { + padding: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + backgroundColor: "#7f1d1d", + borderWidth: theme.borderWidth[1], + borderColor: "#b91c1c", + }, + testResultTextSuccess: { + color: "#4ade80", + }, + testResultTextError: { + color: "#f87171", + }, + settingCard: { + backgroundColor: theme.colors.zinc[900], + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[4], + marginBottom: theme.spacing[3], + }, + settingRow: { + flexDirection: "row", + justifyContent: "space-between", + alignItems: "center", + }, + settingContent: { + flex: 1, + }, + settingTitle: { + color: theme.colors.white, + fontSize: theme.fontSize.base, + marginBottom: theme.spacing[1], + }, + settingDescription: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.sm, + }, + themeCardDisabled: { + backgroundColor: theme.colors.zinc[900], + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[4], + opacity: theme.opacity[50], + }, + themeHelpText: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.sm, + marginBottom: theme.spacing[3], + }, + themeOption: { + flexDirection: "row", + alignItems: "center", + paddingVertical: theme.spacing[2], + }, + radioOuter: { + width: 20, + height: 20, + borderRadius: theme.borderRadius.full, + borderWidth: theme.borderWidth[2], + marginRight: theme.spacing[3], + alignItems: "center", + justifyContent: "center", + }, + radioOuterSelected: { + borderColor: theme.colors.blue[500], + }, + radioOuterUnselected: { + borderColor: theme.colors.zinc[600], + }, + radioInner: { + width: 12, + height: 12, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.blue[500], + }, + themeOptionText: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.base, + textTransform: "capitalize", + }, + saveButton: { + padding: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + marginBottom: theme.spacing[3], + backgroundColor: theme.colors.blue[500], + }, + saveButtonDisabled: { + backgroundColor: "#1e3a8a", + opacity: theme.opacity[50], + }, + saveButtonText: { + color: theme.colors.white, + textAlign: "center", + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + }, + resetButton: { + padding: theme.spacing[4], + borderRadius: theme.borderRadius.lg, + borderWidth: theme.borderWidth[1], + borderColor: "#7f1d1d", + }, + resetButtonText: { + color: theme.colors.red[500], + textAlign: "center", + fontSize: theme.fontSize.base, + fontWeight: theme.fontWeight.semibold, + }, + footer: { + borderTopWidth: theme.borderWidth[1], + borderTopColor: theme.colors.zinc[800], + paddingTop: theme.spacing[6], + }, + footerText: { + color: theme.colors.zinc[500], + fontSize: theme.fontSize.sm, + textAlign: "center", + }, + footerVersion: { + color: theme.colors.zinc[600], + fontSize: theme.fontSize.xs, + textAlign: "center", + marginTop: theme.spacing[1], + }, +})); export default function SettingsScreen() { const { settings, isLoading, updateSettings, resetSettings } = useSettings(); @@ -11,10 +238,13 @@ export default function SettingsScreen() { const [serverUrl, setServerUrl] = useState(settings.serverUrl); const [useSpeaker, setUseSpeaker] = useState(settings.useSpeaker); const [keepScreenOn, setKeepScreenOn] = useState(settings.keepScreenOn); - const [theme, setTheme] = useState<'dark' | 'light' | 'auto'>(settings.theme); + const [theme, setTheme] = useState<"dark" | "light" | "auto">(settings.theme); const [hasChanges, setHasChanges] = useState(false); const [isTesting, setIsTesting] = useState(false); - const [testResult, setTestResult] = useState<{ success: boolean; message: string } | null>(null); + const [testResult, setTestResult] = useState<{ + success: boolean; + message: string; + } | null>(null); // Update local state when settings load useEffect(() => { @@ -37,7 +267,7 @@ export default function SettingsScreen() { function validateServerUrl(url: string): boolean { try { const urlObj = new URL(url); - return urlObj.protocol === 'ws:' || urlObj.protocol === 'wss:'; + return urlObj.protocol === "ws:" || urlObj.protocol === "wss:"; } catch { return false; } @@ -47,9 +277,9 @@ export default function SettingsScreen() { // Validate server URL if (!validateServerUrl(serverUrl)) { Alert.alert( - 'Invalid URL', - 'Server URL must be a valid WebSocket URL (ws:// or wss://)', - [{ text: 'OK' }] + "Invalid URL", + "Server URL must be a valid WebSocket URL (ws:// or wss://)", + [{ text: "OK" }] ); return; } @@ -63,39 +293,43 @@ export default function SettingsScreen() { }); Alert.alert( - 'Settings Saved', - 'Your settings have been saved successfully.', + "Settings Saved", + "Your settings have been saved successfully.", [ { - text: 'OK', + text: "OK", onPress: () => router.back(), }, ] ); } catch (error) { - Alert.alert( - 'Error', - 'Failed to save settings. Please try again.', - [{ text: 'OK' }] - ); + Alert.alert("Error", "Failed to save settings. Please try again.", [ + { text: "OK" }, + ]); } } async function handleReset() { Alert.alert( - 'Reset Settings', - 'Are you sure you want to reset all settings to defaults?', + "Reset Settings", + "Are you sure you want to reset all settings to defaults?", [ - { text: 'Cancel', style: 'cancel' }, + { text: "Cancel", style: "cancel" }, { - text: 'Reset', - style: 'destructive', + text: "Reset", + style: "destructive", onPress: async () => { try { await resetSettings(); - Alert.alert('Settings Reset', 'All settings have been reset to defaults.'); + Alert.alert( + "Settings Reset", + "All settings have been reset to defaults." + ); } catch (error) { - Alert.alert('Error', 'Failed to reset settings. Please try again.'); + Alert.alert( + "Error", + "Failed to reset settings. Please try again." + ); } }, }, @@ -106,13 +340,13 @@ export default function SettingsScreen() { function handleCancel() { if (hasChanges) { Alert.alert( - 'Discard Changes', - 'You have unsaved changes. Are you sure you want to go back?', + "Discard Changes", + "You have unsaved changes. Are you sure you want to go back?", [ - { text: 'Stay', style: 'cancel' }, + { text: "Stay", style: "cancel" }, { - text: 'Discard', - style: 'destructive', + text: "Discard", + style: "destructive", onPress: () => router.back(), }, ] @@ -125,9 +359,9 @@ export default function SettingsScreen() { async function handleTestConnection() { if (!validateServerUrl(serverUrl)) { Alert.alert( - 'Invalid URL', - 'Server URL must be a valid WebSocket URL (ws:// or wss://)', - [{ text: 'OK' }] + "Invalid URL", + "Server URL must be a valid WebSocket URL (ws:// or wss://)", + [{ text: "OK" }] ); return; } @@ -142,7 +376,7 @@ export default function SettingsScreen() { ws.close(); setTestResult({ success: false, - message: 'Connection timeout - server did not respond', + message: "Connection timeout - server did not respond", }); setIsTesting(false); }, 5000); @@ -152,7 +386,7 @@ export default function SettingsScreen() { ws.close(); setTestResult({ success: true, - message: 'Connection successful', + message: "Connection successful", }); setIsTesting(false); }; @@ -161,14 +395,14 @@ export default function SettingsScreen() { clearTimeout(timeout); setTestResult({ success: false, - message: 'Connection failed - check URL and network', + message: "Connection failed - check URL and network", }); setIsTesting(false); }; } catch (error) { setTestResult({ success: false, - message: 'Failed to create connection', + message: "Failed to create connection", }); setIsTesting(false); } @@ -176,35 +410,33 @@ export default function SettingsScreen() { if (isLoading) { return ( - - Loading settings... + + Loading settings... ); } return ( - + {/* Header */} - - - Settings + + + Settings - Cancel + Cancel - - + + {/* Server Configuration */} - - - Server Configuration - + + Server Configuration - WebSocket URL + WebSocket URL - + Must be a valid WebSocket URL (ws:// or wss://) @@ -224,30 +456,38 @@ export default function SettingsScreen() { {isTesting ? ( <> - Testing... + Testing... ) : ( - Test Connection + Test Connection )} {/* Test Result */} {testResult && ( - + {testResult.message} @@ -255,110 +495,99 @@ export default function SettingsScreen() { {/* Audio Settings */} - - - Audio - + + Audio - - - - Use Speaker - + + + + Use Speaker + Play audio through speaker instead of earpiece - - - - Keep Screen On - + + + + Keep Screen On + Prevent screen from sleeping during voice sessions {/* Theme Settings */} - - - Theme - + + Theme - - + + Theme selection (coming soon) - {(['dark', 'light', 'auto'] as const).map((themeOption) => ( + {(["dark", "light", "auto"] as const).map((themeOption) => ( - + {theme === themeOption && ( - + )} - - {themeOption} - + {themeOption} ))} {/* Action Buttons */} - + - - Save Settings - + Save Settings - - - Reset to Defaults - + + Reset to Defaults {/* App Info */} - - - Voice Assistant Mobile - - - Version 1.0.0 - + + Voice Assistant Mobile + Version 1.0.0 diff --git a/packages/app/components/active-processes.tsx b/packages/app/components/active-processes.tsx index de1ade738..10fd79d33 100644 --- a/packages/app/components/active-processes.tsx +++ b/packages/app/components/active-processes.tsx @@ -1,4 +1,5 @@ import { View, Text, Pressable, ScrollView } from 'react-native'; +import { StyleSheet } from 'react-native-unistyles'; import type { AgentStatus } from '@server/server/acp/types'; export interface ActiveProcessesProps { @@ -59,6 +60,81 @@ function getModeColor(modeId?: string): string { return '#9ca3af'; // gray - unknown } +const styles = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + container: { + backgroundColor: theme.colors.zinc[900], + borderBottomWidth: 1, + borderBottomColor: theme.colors.zinc[800], + }, + header: { + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[3], + borderBottomWidth: 1, + borderBottomColor: theme.colors.zinc[800], + }, + backButton: { + backgroundColor: theme.colors.zinc[800], + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[2], + borderRadius: theme.borderRadius.lg, + }, + backButtonActive: { + backgroundColor: theme.colors.zinc[700], + }, + backButtonText: { + color: '#fff', + fontSize: theme.fontSize.sm, + fontWeight: '600', + textAlign: 'center', + }, + scrollView: { + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[3], + }, + processItem: { + flexDirection: 'row', + alignItems: 'center', + gap: theme.spacing[2], + paddingHorizontal: theme.spacing[3], + paddingVertical: theme.spacing[2], + borderRadius: theme.borderRadius.lg, + }, + processItemActive: { + backgroundColor: theme.colors.blue[600], + }, + processItemInactive: { + backgroundColor: theme.colors.zinc[800], + }, + agentIcon: { + width: 12, + height: 12, + borderRadius: theme.borderRadius.full, + backgroundColor: theme.colors.blue[500], + }, + commandIcon: { + width: 12, + height: 12, + borderRadius: theme.borderRadius.sm, + backgroundColor: theme.colors.purple[500], + }, + processText: { + color: '#fff', + fontSize: theme.fontSize.xs, + fontWeight: '500', + }, + statusDot: { + width: 8, + height: 8, + borderRadius: theme.borderRadius.full, + }, + modeIndicator: { + width: 6, + height: 6, + borderRadius: theme.borderRadius.full, + opacity: 0.3, + }, +})); + export function ActiveProcesses({ agents, commands, @@ -74,29 +150,24 @@ export function ActiveProcesses({ } return ( - - {/* Header with Back button */} + {hasActiveProcess && ( - + [styles.backButton, pressed && styles.backButtonActive]} > - - Back to Chat - + Back to Chat )} - {/* Scrollable process list */} - {/* Agents */} {agents.map((agent) => { const modeName = getModeName(agent.currentModeId, agent.availableModes); const isActive = activeProcessType === 'agent' && activeProcessId === agent.id; @@ -105,36 +176,25 @@ export function ActiveProcesses({ onSelectAgent(agent.id)} - className={`flex-row items-center gap-2 px-3 py-2 rounded-lg ${ - isActive ? 'bg-blue-600' : 'bg-zinc-800' - } active:opacity-70`} + style={({ pressed }) => [ + styles.processItem, + isActive ? styles.processItemActive : styles.processItemInactive, + pressed && { opacity: 0.7 }, + ]} > - {/* Agent icon */} - + - {/* Agent ID (shortened) */} - - {agent.id.substring(0, 8)} - + {agent.id.substring(0, 8)} - {/* Status indicator */} - + - {/* Mode indicator */} {agent.currentModeId && ( - + )} ); })} - {/* Commands */} {commands.map((command) => { const statusColor = command.isDead ? command.exitCode === 0 @@ -143,23 +203,14 @@ export function ActiveProcesses({ : '#3b82f6'; return ( - - {/* Command icon */} - + + - {/* Command name */} - + {command.name || command.currentCommand.substring(0, 20)} - {/* Status indicator */} - + ); })} diff --git a/packages/app/components/agent-stream-view.tsx b/packages/app/components/agent-stream-view.tsx index 430c6bfb5..1cda6ea32 100644 --- a/packages/app/components/agent-stream-view.tsx +++ b/packages/app/components/agent-stream-view.tsx @@ -1,6 +1,7 @@ import { useState, useEffect, useRef } from 'react'; import { View, Text, ScrollView, Pressable } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { StyleSheet } from 'react-native-unistyles'; import type { AgentStatus } from '@server/server/acp/types'; import type { SessionNotification } from '@agentclientprotocol/sdk'; @@ -80,11 +81,11 @@ function AgentUpdate({ // Handle message chunks if (sessionUpdate.kind === 'agent_message_chunk') { return ( - - + + {formatTimestamp(update.timestamp)} - {sessionUpdate.chunk || ''} + {sessionUpdate.chunk || ''} ); } @@ -92,21 +93,21 @@ function AgentUpdate({ // Handle tool calls if (sessionUpdate.kind === 'tool_call') { return ( - + setIsExpanded(!isExpanded)} - className="px-3 py-2 active:bg-zinc-700" + style={stylesheet.collapsibleHeader} > - + {formatTimestamp(update.timestamp)} - + Tool Call: {sessionUpdate.toolName || 'unknown'} {isExpanded && sessionUpdate.arguments && ( - - + + {JSON.stringify(sessionUpdate.arguments, null, 2)} @@ -118,11 +119,11 @@ function AgentUpdate({ // Handle tool call updates if (sessionUpdate.kind === 'tool_call_update') { return ( - - + + {formatTimestamp(update.timestamp)} - + {sessionUpdate.status || 'updating'} @@ -133,22 +134,22 @@ function AgentUpdate({ if (sessionUpdate.kind === 'available_commands_update') { const commands = sessionUpdate.commands || []; return ( - + setIsExpanded(!isExpanded)} - className="px-3 py-2 active:bg-zinc-700" + style={stylesheet.collapsibleHeader} > - + {formatTimestamp(update.timestamp)} - + Available Commands ({commands.length}) {isExpanded && ( - + {commands.map((cmd: any, idx: number) => ( - + • {cmd.name || cmd} ))} @@ -160,11 +161,11 @@ function AgentUpdate({ // Generic session update return ( - - + + {formatTimestamp(update.timestamp)} - + {JSON.stringify(sessionUpdate, null, 2)} @@ -173,11 +174,11 @@ function AgentUpdate({ // Fallback for unknown notification types return ( - - + + {formatTimestamp(update.timestamp)} - + {JSON.stringify(notification, null, 2)} @@ -204,49 +205,51 @@ export function AgentStreamView({ const canKill = agent.status !== 'killed' && agent.status !== 'completed'; return ( - + {/* Header */} - + {/* Back button */} - + ← Back to Chat {/* Agent info */} - - - Agent: - + + + Agent: + {agentId.substring(0, 8)} - {getStatusIcon(agent.status)} - {agent.status} + {getStatusIcon(agent.status)} + {agent.status} - + Created: {formatTimestamp(agent.createdAt)} {/* Control buttons */} {(canCancel || canKill) && ( - + {canCancel && ( onCancelAgent(agentId)} - className="flex-1 bg-orange-600 px-4 py-2 rounded-lg active:bg-orange-700" + style={stylesheet.cancelButton} > - + Cancel @@ -254,9 +257,9 @@ export function AgentStreamView({ {canKill && ( onKillAgent(agentId)} - className="flex-1 bg-red-600 px-4 py-2 rounded-lg active:bg-red-700" + style={stylesheet.killButton} > - + Kill @@ -268,12 +271,12 @@ export function AgentStreamView({ {/* Updates list */} {updates.length === 0 ? ( - - + + No updates yet.{'\n'}Waiting for agent activity... @@ -284,3 +287,167 @@ export function AgentStreamView({ ); } + +const stylesheet = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + container: { + flex: 1, + backgroundColor: theme.colors.black, + }, + header: { + backgroundColor: theme.colors.zinc[900], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.zinc[800], + paddingHorizontal: theme.spacing[4], + paddingBottom: theme.spacing[4], + }, + backButton: { + backgroundColor: theme.colors.zinc[800], + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[2], + borderRadius: theme.borderRadius.lg, + marginBottom: theme.spacing[4], + }, + backButtonText: { + color: theme.colors.white, + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + textAlign: "center", + }, + agentInfoRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + marginBottom: theme.spacing[3], + }, + agentIdRow: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + agentLabel: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.sm, + }, + agentId: { + color: theme.colors.white, + fontSize: theme.fontSize.sm, + fontFamily: "monospace", + }, + statusBadge: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + paddingHorizontal: theme.spacing[3], + paddingVertical: theme.spacing[1], + borderRadius: theme.borderRadius.full, + }, + statusIcon: { + color: theme.colors.white, + fontSize: theme.fontSize.xs, + }, + statusText: { + color: theme.colors.white, + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + }, + createdText: { + color: theme.colors.zinc[500], + fontSize: theme.fontSize.xs, + }, + controlButtons: { + flexDirection: "row", + gap: theme.spacing[2], + marginTop: theme.spacing[4], + }, + cancelButton: { + flex: 1, + backgroundColor: theme.colors.orange[600], + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[2], + borderRadius: theme.borderRadius.lg, + }, + killButton: { + flex: 1, + backgroundColor: theme.colors.red[600], + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[2], + borderRadius: theme.borderRadius.lg, + }, + controlButtonText: { + color: theme.colors.white, + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + textAlign: "center", + }, + scrollView: { + flex: 1, + paddingHorizontal: theme.spacing[4], + }, + emptyState: { + flex: 1, + alignItems: "center", + justifyContent: "center", + paddingVertical: theme.spacing[12], + }, + emptyStateText: { + color: theme.colors.zinc[500], + fontSize: theme.fontSize.sm, + textAlign: "center", + }, + updateCard: { + backgroundColor: theme.colors.zinc[800], + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[3], + marginBottom: theme.spacing[2], + }, + collapsibleCard: { + backgroundColor: theme.colors.zinc[800], + borderRadius: theme.borderRadius.lg, + marginBottom: theme.spacing[2], + overflow: "hidden", + }, + collapsibleHeader: { + paddingHorizontal: theme.spacing[3], + paddingVertical: theme.spacing[2], + }, + collapsibleContent: { + paddingHorizontal: theme.spacing[3], + paddingBottom: theme.spacing[3], + borderTopWidth: theme.borderWidth[1], + borderTopColor: theme.colors.zinc[700], + }, + timestampText: { + color: theme.colors.zinc[500], + fontSize: theme.fontSize.xs, + marginBottom: theme.spacing[1], + }, + messageText: { + color: theme.colors.white, + fontSize: theme.fontSize.sm, + }, + toolCallText: { + color: "#60a5fa", + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + }, + updateStatusText: { + color: "#fbbf24", + fontSize: theme.fontSize.sm, + }, + commandsText: { + color: "#4ade80", + fontSize: theme.fontSize.sm, + fontWeight: theme.fontWeight.semibold, + }, + commandItem: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.xs, + marginTop: theme.spacing[1], + }, + codeText: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.xs, + fontFamily: "monospace", + marginTop: theme.spacing[2], + }, +})); diff --git a/packages/app/components/artifact-drawer.tsx b/packages/app/components/artifact-drawer.tsx index 2328167f8..53b5481a7 100644 --- a/packages/app/components/artifact-drawer.tsx +++ b/packages/app/components/artifact-drawer.tsx @@ -1,10 +1,17 @@ -import { View, Text, ScrollView, Pressable, Modal } from 'react-native'; -import { SafeAreaView } from 'react-native-safe-area-context'; -import { useEffect } from 'react'; +import { + View, + Text, + ScrollView, + Pressable, + Modal, +} from "react-native"; +import { SafeAreaView } from "react-native-safe-area-context"; +import { useEffect } from "react"; +import { StyleSheet } from "react-native-unistyles"; export interface Artifact { id: string; - type: 'markdown' | 'diff' | 'image' | 'code'; + type: "markdown" | "diff" | "image" | "code"; title: string; content: string; isBase64: boolean; @@ -15,10 +22,145 @@ interface ArtifactDrawerProps { onClose: () => void; } +const styles = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + container: { + flex: 1, + backgroundColor: theme.colors.black, + flexDirection: "column", + }, + header: { + paddingBottom: theme.spacing[4], + paddingHorizontal: theme.spacing[4], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.zinc[800], + }, + headerRow: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + }, + titleContainer: { + flex: 1, + marginRight: theme.spacing[4], + }, + title: { + color: theme.colors.white, + fontSize: theme.fontSize["2xl"], + fontWeight: theme.fontWeight.bold, + }, + headerActions: { + flexDirection: "row", + alignItems: "center", + gap: theme.spacing[2], + }, + badge: { + paddingHorizontal: theme.spacing[3], + paddingVertical: theme.spacing[1], + borderRadius: theme.borderRadius.full, + }, + badgeMarkdown: { + backgroundColor: theme.colors.blue[600], + }, + badgeDiff: { + backgroundColor: theme.colors.purple[600], + }, + badgeImage: { + backgroundColor: theme.colors.green[600], + }, + badgeCode: { + backgroundColor: theme.colors.orange[600], + }, + badgeText: { + color: theme.colors.white, + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + }, + closeButton: { + backgroundColor: theme.colors.zinc[800], + width: 40, + height: 40, + borderRadius: 20, + alignItems: "center", + justifyContent: "center", + }, + closeButtonText: { + color: theme.colors.white, + fontSize: theme.fontSize["2xl"], + fontWeight: theme.fontWeight.bold, + }, + contentScroll: { + flex: 1, + backgroundColor: theme.colors.black, + }, + contentScrollContainer: { + padding: theme.spacing[4], + flexGrow: 1, + }, + imagePlaceholder: { + flex: 1, + alignItems: "center", + justifyContent: "center", + }, + imagePlaceholderText: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.sm, + }, + imagePlaceholderSubtext: { + color: theme.colors.zinc[600], + fontSize: theme.fontSize.xs, + marginTop: theme.spacing[2], + }, + codeContainer: { + backgroundColor: theme.colors.zinc[900], + borderRadius: theme.borderRadius.lg, + padding: theme.spacing[4], + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.zinc[800], + }, + codeText: { + color: theme.colors.zinc[300], + fontSize: theme.fontSize.sm, + fontFamily: "monospace", + }, + metadataContainer: { + backgroundColor: theme.colors.zinc[900], + padding: theme.spacing[3], + borderTopWidth: theme.borderWidth[1], + borderTopColor: theme.colors.zinc[800], + marginTop: theme.spacing[2], + }, + metadataTitle: { + color: theme.colors.zinc[500], + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + marginBottom: theme.spacing[2], + }, + metadataRow: { + flexDirection: "row", + marginBottom: theme.spacing[1], + }, + metadataLabel: { + color: theme.colors.zinc[500], + fontSize: theme.fontSize.xs, + width: 80, + }, + metadataValue: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.xs, + flex: 1, + fontFamily: "monospace", + }, +})); + export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) { useEffect(() => { if (!artifact) return; - console.log('[ArtifactDrawer] Showing artifact:', artifact.id, artifact.type, artifact.title); + console.log( + "[ArtifactDrawer] Showing artifact:", + artifact.id, + artifact.type, + artifact.title + ); }, [artifact]); if (!artifact) { @@ -28,12 +170,12 @@ export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) { // Decode content if base64 const content = artifact.isBase64 ? atob(artifact.content) : artifact.content; - // Type badge colors - const typeColors = { - markdown: 'bg-blue-600', - diff: 'bg-purple-600', - image: 'bg-green-600', - code: 'bg-orange-600', + // Type badge style mapping + const typeBadgeStyles = { + markdown: styles.badgeMarkdown, + diff: styles.badgeDiff, + image: styles.badgeImage, + code: styles.badgeCode, }; return ( @@ -43,84 +185,81 @@ export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) { presentationStyle="pageSheet" onRequestClose={onClose} > - + {/* Header */} - - - - + + + + {artifact.title} - - - - {artifact.type} + + + + {artifact.type.toUpperCase()} - - × + + × - {/* Content */} - - {artifact.type === 'image' ? ( - - + + {artifact.type === "image" ? ( + + Image viewing not yet implemented - + Base64 image data received ) : ( - + - - {content} - + {content} )} + - {/* Metadata */} - - METADATA - - - ID: - - {artifact.id} - - - - Type: - {artifact.type} - - - Encoding: - - {artifact.isBase64 ? 'Base64' : 'Plain text'} - - - - Size: - - {content.length.toLocaleString()} characters - - + {/* Metadata - Fixed at bottom */} + + METADATA + + + ID: + {artifact.id} + + + Type: + {artifact.type} + + + Encoding: + + {artifact.isBase64 ? "Base64" : "Plain text"} + + + + Size: + + {content.length.toLocaleString()} characters + - + ); diff --git a/packages/app/components/connection-status.tsx b/packages/app/components/connection-status.tsx index dad139af7..6c88d3c68 100644 --- a/packages/app/components/connection-status.tsx +++ b/packages/app/components/connection-status.tsx @@ -1,15 +1,50 @@ import { View, Text } from 'react-native'; +import { StyleSheet } from 'react-native-unistyles'; interface ConnectionStatusProps { isConnected: boolean; } +const styles = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + container: { + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[3], + borderBottomWidth: 1, + borderBottomColor: theme.colors.zinc[800], + }, + row: { + flexDirection: 'row', + alignItems: 'center', + }, + dot: { + width: 8, + height: 8, + borderRadius: theme.borderRadius.full, + marginRight: theme.spacing[2], + }, + dotConnected: { + backgroundColor: theme.colors.green[500], + }, + dotDisconnected: { + backgroundColor: theme.colors.red[500], + }, + text: { + fontSize: theme.fontSize.sm, + }, + textConnected: { + color: theme.colors.green[500], + }, + textDisconnected: { + color: theme.colors.red[500], + }, +})); + export function ConnectionStatus({ isConnected }: ConnectionStatusProps) { return ( - - - - + + + + {isConnected ? 'Connected' : 'Disconnected'} diff --git a/packages/app/components/conversation-selector.tsx b/packages/app/components/conversation-selector.tsx index f4fdcc589..bee704dd0 100644 --- a/packages/app/components/conversation-selector.tsx +++ b/packages/app/components/conversation-selector.tsx @@ -3,6 +3,7 @@ import { Modal, View, Text, Pressable, ScrollView, Alert, KeyboardAvoidingView, import { SafeAreaView } from 'react-native-safe-area-context'; import { MessageSquare, X, Plus, Trash2 } from 'lucide-react-native'; import AsyncStorage from '@react-native-async-storage/async-storage'; +import { StyleSheet } from 'react-native-unistyles'; import type { UseWebSocketReturn } from '../hooks/use-websocket'; const STORAGE_KEY = '@voice-dev:conversation-id'; @@ -177,7 +178,7 @@ export function ConversationSelector({ setIsOpen(true)} - className="bg-zinc-800 px-4 py-2 rounded-lg" + style={styles.triggerButton} > @@ -188,99 +189,94 @@ export function ConversationSelector({ transparent={true} onRequestClose={() => setIsOpen(false)} > - + setIsOpen(false)} /> - - + + {/* Header */} - - - Conversations {conversations.length > 0 && `(${conversations.length})`} - - setIsOpen(false)}> - - - + + + Conversations {conversations.length > 0 && `(${conversations.length})`} + + setIsOpen(false)}> + + + - {/* New Conversation Button */} - - - - New Conversation - - - - {/* Conversations List */} - - {isLoading && ( - - Loading... - - )} - - {!isLoading && conversations.length === 0 && ( - - No saved conversations - - )} - - {!isLoading && conversations.map((conversation) => ( - + - handleSelectConversation(conversation.id)} - > - - {conversation.messageCount} messages - - - {formatDate(conversation.lastUpdated)} - - - handleDeleteConversation(conversation.id)} - style={{ padding: 8 }} - hitSlop={{ top: 10, bottom: 10, left: 10, right: 10 }} - > - - - - ))} + + New Conversation + + - {!isLoading && conversations.length > 0 && ( - - + {isLoading && ( + + Loading... + + )} + + {!isLoading && conversations.length === 0 && ( + + No saved conversations + + )} + + {!isLoading && conversations.map((conversation) => ( + - - Clear All - - - )} - + handleSelectConversation(conversation.id)} + > + + {conversation.messageCount} messages + + + {formatDate(conversation.lastUpdated)} + + + handleDeleteConversation(conversation.id)} + style={styles.deleteButton} + hitSlop={{ top: 10, bottom: 10, left: 10, right: 10 }} + > + + + + ))} + + {!isLoading && conversations.length > 0 && ( + + + + Clear All + + + )} + @@ -288,3 +284,125 @@ export function ConversationSelector({ ); } + +const styles = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + triggerButton: { + backgroundColor: theme.colors.zinc[800], + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[2], + borderRadius: theme.borderRadius.lg, + }, + modalOverlay: { + flex: 1, + backgroundColor: 'rgba(0,0,0,0.5)', + }, + modalBackdrop: { + flex: 1, + }, + modalContent: { + backgroundColor: theme.colors.zinc[900], + borderTopLeftRadius: theme.spacing[6], + borderTopRightRadius: theme.spacing[6], + height: '80%', + }, + modalInner: { + flex: 1, + }, + header: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + padding: theme.spacing[6], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.zinc[800], + }, + headerTitle: { + color: theme.colors.white, + fontSize: theme.fontSize.lg, + fontWeight: theme.fontWeight.semibold, + }, + newButtonContainer: { + padding: theme.spacing[4], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.zinc[800], + }, + newButton: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'center', + gap: theme.spacing[2], + backgroundColor: theme.colors.zinc[800], + paddingVertical: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + }, + newButtonText: { + color: theme.colors.white, + fontWeight: theme.fontWeight.semibold, + }, + scrollView: { + flex: 1, + }, + scrollContent: { + paddingBottom: theme.spacing[4] * 5, + }, + loadingContainer: { + padding: theme.spacing[8], + alignItems: 'center', + }, + loadingText: { + color: theme.colors.zinc[400], + }, + emptyContainer: { + padding: theme.spacing[8], + alignItems: 'center', + }, + emptyText: { + color: theme.colors.zinc[400], + }, + conversationItem: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + padding: theme.spacing[4], + borderBottomWidth: theme.borderWidth[1], + borderBottomColor: theme.colors.zinc[800], + backgroundColor: 'transparent', + }, + conversationItemActive: { + backgroundColor: theme.colors.zinc[800], + }, + conversationContent: { + flex: 1, + }, + conversationTitle: { + color: theme.colors.white, + fontWeight: theme.fontWeight.semibold, + fontSize: theme.fontSize.base, + }, + conversationDate: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.sm, + marginTop: theme.spacing[1], + }, + deleteButton: { + padding: theme.spacing[2], + }, + clearAllContainer: { + padding: theme.spacing[4], + }, + clearAllButton: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'center', + gap: theme.spacing[2], + backgroundColor: 'rgba(127, 29, 29, 0.2)', + paddingVertical: theme.spacing[3], + borderRadius: theme.borderRadius.lg, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.red[900], + }, + clearAllText: { + color: theme.colors.red[500], + fontWeight: theme.fontWeight.semibold, + }, +})); diff --git a/packages/app/components/message.tsx b/packages/app/components/message.tsx index e4345223a..38a2f6626 100644 --- a/packages/app/components/message.tsx +++ b/packages/app/components/message.tsx @@ -2,17 +2,40 @@ import { View, Text, Pressable, Animated } from 'react-native'; import { useState, useEffect, useRef } from 'react'; import Markdown from 'react-native-markdown-display'; import { Circle, Info, CheckCircle, XCircle, FileText, ChevronRight, ChevronDown, RefreshCw } from 'lucide-react-native'; +import { StyleSheet } from 'react-native-unistyles'; interface UserMessageProps { message: string; timestamp: number; } +const userMessageStylesheet = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + container: { + flexDirection: 'row', + justifyContent: 'flex-end', + marginBottom: theme.spacing[3], + paddingHorizontal: theme.spacing[4], + }, + bubble: { + backgroundColor: theme.colors.blue[600], + borderRadius: theme.borderRadius['2xl'], + borderTopRightRadius: theme.borderRadius.sm, + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[3], + maxWidth: '80%', + }, + text: { + color: theme.colors.white, + fontSize: theme.fontSize.lg, + lineHeight: 24, + }, +})); + export function UserMessage({ message, timestamp }: UserMessageProps) { return ( - - - {message} + + + {message} ); @@ -24,6 +47,22 @@ interface AssistantMessageProps { isStreaming?: boolean; } +const assistantMessageStylesheet = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + container: { + marginBottom: theme.spacing[3], + paddingHorizontal: theme.spacing[4], + paddingVertical: theme.spacing[3], + }, + streamingIndicator: { + marginTop: theme.spacing[1], + }, + streamingText: { + color: theme.colors.teal[200], + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.bold, + }, +})); + export function AssistantMessage({ message, timestamp, isStreaming = false }: AssistantMessageProps) { const fadeAnim = useRef(new Animated.Value(0.3)).current; @@ -60,10 +99,10 @@ export function AssistantMessage({ message, timestamp, isStreaming = false }: As marginBottom: 8, }, strong: { - fontWeight: '700', + fontWeight: '700' as const, }, em: { - fontStyle: 'italic', + fontStyle: 'italic' as const, }, code_inline: { backgroundColor: '#134e4a', @@ -91,7 +130,7 @@ export function AssistantMessage({ message, timestamp, isStreaming = false }: As }, link: { color: '#5eead4', - textDecorationLine: 'underline', + textDecorationLine: 'underline' as const, }, bullet_list: { marginBottom: 8, @@ -105,11 +144,11 @@ export function AssistantMessage({ message, timestamp, isStreaming = false }: As }; return ( - + {message} {isStreaming && ( - - ... + + ... )} @@ -127,6 +166,76 @@ interface ActivityLogProps { onArtifactClick?: (artifactId: string) => void; } +const activityLogStylesheet = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + pressable: { + marginHorizontal: theme.spacing[2], + marginBottom: theme.spacing[1], + borderRadius: theme.borderRadius.md, + overflow: 'hidden', + }, + pressableActive: { + opacity: 0.7, + }, + systemBg: { + backgroundColor: 'rgba(39, 39, 42, 0.5)', // zinc-800/50 + }, + infoBg: { + backgroundColor: 'rgba(30, 58, 138, 0.3)', // blue-900/30 + }, + successBg: { + backgroundColor: 'rgba(20, 83, 45, 0.3)', // green-900/30 + }, + errorBg: { + backgroundColor: 'rgba(127, 29, 29, 0.3)', // red-900/30 + }, + artifactBg: { + backgroundColor: 'rgba(30, 58, 138, 0.4)', // blue-900/40 + }, + content: { + paddingHorizontal: theme.spacing[3], + paddingVertical: 10, + }, + row: { + flexDirection: 'row', + alignItems: 'flex-start', + gap: theme.spacing[2], + }, + iconContainer: { + flexShrink: 0, + }, + textContainer: { + flex: 1, + }, + messageText: { + fontSize: theme.fontSize.sm, + lineHeight: 20, + }, + detailsRow: { + flexDirection: 'row', + alignItems: 'center', + marginTop: theme.spacing[1], + }, + detailsText: { + color: theme.colors.zinc[500], + fontSize: theme.fontSize.xs, + marginRight: theme.spacing[1], + }, + metadataContainer: { + marginTop: theme.spacing[2], + backgroundColor: 'rgba(0, 0, 0, 0.5)', + borderRadius: theme.borderRadius.base, + padding: theme.spacing[2], + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.zinc[700], + }, + metadataText: { + color: theme.colors.zinc[300], + fontSize: theme.fontSize.xs, + fontFamily: 'monospace', + lineHeight: 16, + }, +})); + export function ActivityLog({ type, message, @@ -140,11 +249,11 @@ export function ActivityLog({ const [isExpanded, setIsExpanded] = useState(false); const typeConfig = { - system: { bg: 'bg-zinc-800/50', color: '#a1a1aa', Icon: Circle }, - info: { bg: 'bg-blue-900/30', color: '#60a5fa', Icon: Info }, - success: { bg: 'bg-green-900/30', color: '#4ade80', Icon: CheckCircle }, - error: { bg: 'bg-red-900/30', color: '#f87171', Icon: XCircle }, - artifact: { bg: 'bg-blue-900/40', color: '#93c5fd', Icon: FileText }, + system: { bg: activityLogStylesheet.systemBg, color: '#a1a1aa', Icon: Circle }, + info: { bg: activityLogStylesheet.infoBg, color: '#60a5fa', Icon: Info }, + success: { bg: activityLogStylesheet.successBg, color: '#4ade80', Icon: CheckCircle }, + error: { bg: activityLogStylesheet.errorBg, color: '#f87171', Icon: XCircle }, + artifact: { bg: activityLogStylesheet.artifactBg, color: '#93c5fd', Icon: FileText }, }; const config = typeConfig[type]; @@ -162,24 +271,30 @@ export function ActivityLog({ ? `${artifactType}: ${title}` : message; + const isInteractive = type === 'artifact' || metadata; + return ( - - - - - + + + + + + + {displayMessage} {metadata && ( - - Details + + Details {isExpanded ? ( ) : ( @@ -190,8 +305,8 @@ export function ActivityLog({ {isExpanded && metadata && ( - - + + {JSON.stringify(metadata, null, 2)} @@ -209,6 +324,102 @@ interface ToolCallProps { status: 'executing' | 'completed' | 'failed'; } +const toolCallStylesheet = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + pressable: { + marginHorizontal: theme.spacing[2], + marginBottom: theme.spacing[2], + backgroundColor: theme.colors.zinc[900], + borderRadius: theme.borderRadius.lg, + borderWidth: theme.borderWidth[1], + overflow: 'hidden', + }, + pressableActive: { + opacity: 0.8, + }, + executingBorder: { + borderColor: theme.colors.amber[500], + }, + completedBorder: { + borderColor: theme.colors.green[500], + }, + failedBorder: { + borderColor: theme.colors.red[500], + }, + content: { + padding: theme.spacing[3], + }, + headerRow: { + flexDirection: 'row', + alignItems: 'center', + }, + chevronContainer: { + marginRight: theme.spacing[2], + }, + toolName: { + color: theme.colors.slate[200], + fontFamily: 'monospace', + fontWeight: theme.fontWeight.semibold, + fontSize: theme.fontSize.sm, + flex: 1, + }, + statusBadge: { + flexDirection: 'row', + alignItems: 'center', + gap: 6, + paddingHorizontal: theme.spacing[2], + paddingVertical: theme.spacing[1], + borderRadius: theme.borderRadius.base, + }, + executingBadgeBg: { + backgroundColor: 'rgba(245, 158, 11, 0.2)', // amber-500/20 + }, + completedBadgeBg: { + backgroundColor: 'rgba(34, 197, 94, 0.2)', // green-500/20 + }, + failedBadgeBg: { + backgroundColor: 'rgba(239, 68, 68, 0.2)', // red-500/20 + }, + statusText: { + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + textTransform: 'uppercase', + }, + expandedContent: { + marginTop: theme.spacing[3], + gap: theme.spacing[2], + }, + section: { + // empty - just for grouping + }, + sectionTitle: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.xs, + fontWeight: theme.fontWeight.semibold, + textTransform: 'uppercase', + letterSpacing: 0.5, + marginBottom: 6, + }, + errorSectionTitle: { + color: '#fca5a5', // red-300 + }, + sectionContent: { + backgroundColor: theme.colors.black, + borderRadius: theme.borderRadius.base, + borderWidth: theme.borderWidth[1], + borderColor: theme.colors.zinc[700], + padding: theme.spacing[2], + }, + errorSectionContent: { + borderColor: '#991b1b', // red-800 + }, + sectionText: { + color: theme.colors.slate[200], + fontSize: theme.fontSize.xs, + fontFamily: 'monospace', + lineHeight: 16, + }, +})); + export function ToolCall({ toolName, args, result, error, status }: ToolCallProps) { const [isExpanded, setIsExpanded] = useState(false); const spinAnim = useRef(new Animated.Value(0)).current; @@ -235,20 +446,20 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp const statusConfig = { executing: { - border: 'border-amber-500', - bg: 'bg-amber-500/20', + border: toolCallStylesheet.executingBorder, + badgeBg: toolCallStylesheet.executingBadgeBg, color: '#fcd34d', label: 'executing', }, completed: { - border: 'border-green-500', - bg: 'bg-green-500/20', + border: toolCallStylesheet.completedBorder, + badgeBg: toolCallStylesheet.completedBadgeBg, color: '#86efac', label: 'completed', }, failed: { - border: 'border-red-500', - bg: 'bg-red-500/20', + border: toolCallStylesheet.failedBorder, + badgeBg: toolCallStylesheet.failedBadgeBg, color: '#fca5a5', label: 'failed', }, @@ -259,19 +470,21 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp return ( setIsExpanded(!isExpanded)} - className={`mx-2 mb-2 bg-zinc-900 rounded-lg border ${config.border} overflow-hidden active:opacity-80`} + style={[toolCallStylesheet.pressable, toolCallStylesheet.pressableActive, config.border]} > - - - {isExpanded ? ( - - ) : ( - - )} - + + + + {isExpanded ? ( + + ) : ( + + )} + + {toolName} - + {status === 'executing' ? ( @@ -281,32 +494,32 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp ) : ( )} - + {config.label} {isExpanded && ( - - - + + + Arguments - - + + {JSON.stringify(args, null, 2)} {result !== undefined && ( - - + + Result - - + + {JSON.stringify(result, null, 2)} @@ -314,12 +527,12 @@ export function ToolCall({ toolName, args, result, error, status }: ToolCallProp )} {error !== undefined && ( - - + + Error - - + + {JSON.stringify(error, null, 2)} diff --git a/packages/app/components/voice-button.tsx b/packages/app/components/voice-button.tsx index 18e8423a7..0d070f651 100644 --- a/packages/app/components/voice-button.tsx +++ b/packages/app/components/voice-button.tsx @@ -1,5 +1,6 @@ import { Pressable, View, Text, Animated } from 'react-native'; import { useEffect, useRef } from 'react'; +import { StyleSheet } from 'react-native-unistyles'; interface VoiceButtonProps { state: 'idle' | 'recording' | 'processing' | 'playing'; @@ -7,6 +8,125 @@ interface VoiceButtonProps { disabled?: boolean; } +const styles = StyleSheet.create((theme: import('../styles/theme').Theme) => ({ + container: { + alignItems: 'center', + gap: theme.spacing[4], + }, + pressable: { + opacity: 1, + }, + pressableDisabled: { + opacity: 0.5, + }, + button: { + width: 80, + height: 80, + borderRadius: theme.borderRadius.full, + alignItems: 'center', + justifyContent: 'center', + shadowColor: '#000', + shadowOffset: { width: 0, height: 4 }, + shadowOpacity: 0.3, + shadowRadius: 8, + elevation: 8, + }, + buttonIdle: { + backgroundColor: theme.colors.zinc[700], + }, + buttonRecording: { + backgroundColor: theme.colors.red[500], + }, + buttonProcessing: { + backgroundColor: theme.colors.blue[500], + }, + buttonPlaying: { + backgroundColor: theme.colors.green[500], + }, + label: { + color: theme.colors.zinc[400], + fontSize: theme.fontSize.sm, + }, + // Recording icon + recordingIcon: { + width: 24, + height: 24, + backgroundColor: '#fff', + borderRadius: theme.borderRadius.md, + }, + // Processing icon + processingIconContainer: { + width: 24, + height: 24, + }, + processingDot: { + width: 6, + height: 6, + backgroundColor: '#fff', + borderRadius: theme.borderRadius.full, + position: 'absolute', + }, + processingDotTop: { + top: 0, + left: 12, + }, + processingDotRight: { + top: 12, + right: 0, + }, + processingDotBottom: { + bottom: 0, + left: 12, + }, + // Playing icon + playingIconContainer: { + flexDirection: 'row', + alignItems: 'center', + gap: theme.spacing[1], + }, + playingBar: { + backgroundColor: '#fff', + borderRadius: theme.borderRadius.sm, + }, + playingBar1: { + width: 4, + height: 16, + }, + playingBar2: { + width: 4, + height: 24, + }, + playingBar3: { + width: 4, + height: 12, + }, + // Idle icon (microphone) + micContainer: { + width: 24, + height: 32, + position: 'relative', + }, + micCapsule: { + position: 'absolute', + bottom: 0, + left: 4, + width: 16, + height: 24, + backgroundColor: '#fff', + borderTopLeftRadius: 999, + borderTopRightRadius: 999, + }, + micBase: { + position: 'absolute', + bottom: 0, + left: 0, + width: 24, + height: 6, + backgroundColor: '#fff', + borderRadius: theme.borderRadius.full, + }, +})); + export function VoiceButton({ state, onPress, disabled = false }: VoiceButtonProps) { const pulseAnim = useRef(new Animated.Value(1)).current; @@ -34,43 +154,41 @@ export function VoiceButton({ state, onPress, disabled = false }: VoiceButtonPro const getButtonStyle = () => { switch (state) { case 'recording': - return 'bg-red-500'; + return styles.buttonRecording; case 'processing': - return 'bg-blue-500'; + return styles.buttonProcessing; case 'playing': - return 'bg-green-500'; + return styles.buttonPlaying; default: - return 'bg-zinc-700'; + return styles.buttonIdle; } }; const getIcon = () => { switch (state) { case 'recording': - return ( - - ); + return ; case 'processing': return ( - - - - + + + + ); case 'playing': return ( - - - - + + + + ); default: return ( - - - + + + ); } @@ -90,20 +208,17 @@ export function VoiceButton({ state, onPress, disabled = false }: VoiceButtonPro }; return ( - + - + {getIcon()} - {getLabel()} + {getLabel()} ); } diff --git a/packages/app/global.css b/packages/app/global.css deleted file mode 100644 index e28a7fd46..000000000 --- a/packages/app/global.css +++ /dev/null @@ -1,4 +0,0 @@ -@import "tailwindcss/theme.css" layer(theme); -@import "tailwindcss/preflight.css" layer(base); -@import "tailwindcss/utilities.css"; -@import "nativewind/theme"; diff --git a/packages/app/hooks/use-audio-player.ts b/packages/app/hooks/use-audio-player.ts index bb6201f8c..19b47990a 100644 --- a/packages/app/hooks/use-audio-player.ts +++ b/packages/app/hooks/use-audio-player.ts @@ -1,7 +1,5 @@ -import { useAudioPlayer as useExpoAudioPlayer, setAudioModeAsync } from 'expo-audio'; -import { Paths, File } from 'expo-file-system'; import { useState, useRef, useEffect } from 'react'; -import { Platform } from 'react-native'; +import { initialize, playPCMData } from '@speechmatics/expo-two-way-audio'; interface QueuedAudio { audioData: Blob; @@ -9,75 +7,57 @@ interface QueuedAudio { reject: (error: Error) => void; } -interface AudioPlayerOptions { - useSpeaker?: boolean; +/** + * Resample PCM16 audio from 24kHz to 16kHz + * OpenAI returns 24kHz, Speechmatics expects 16kHz + */ +function resamplePcm24kTo16k(pcm24k: Uint8Array): Uint8Array { + // PCM16 = 2 bytes per sample + const samples24k = pcm24k.length / 2; + const samples16k = Math.floor(samples24k * 16000 / 24000); + + const pcm16k = new Uint8Array(samples16k * 2); + const ratio = 24000 / 16000; // 1.5 + + for (let i = 0; i < samples16k; i++) { + const srcIndex = Math.floor(i * ratio) * 2; + if (srcIndex + 1 < pcm24k.length) { + pcm16k[i * 2] = pcm24k[srcIndex]; + pcm16k[i * 2 + 1] = pcm24k[srcIndex + 1]; + } + } + + console.log('[AudioPlayer] Resampled PCM:', { + input24k: pcm24k.length, + output16k: pcm16k.length, + durationMs: (samples16k / 16), + }); + + return pcm16k; } /** - * Convert Blob to file URI for expo-audio playback + * Hook for audio playback using Speechmatics two-way audio with echo cancellation */ -async function blobToFile(blob: Blob): Promise { - // Read blob as array buffer - const arrayBuffer = await blob.arrayBuffer(); - - // Create temporary file in cache directory - const fileName = `audio_${Date.now()}.mp3`; - const file = new File(Paths.cache, fileName); - - // Write array buffer to file - const uint8Array = new Uint8Array(arrayBuffer); - file.create(); - const stream = file.writableStream(); - const writer = stream.getWriter(); - await writer.write(uint8Array); - await writer.close(); - - return file; -} - -/** - * Hook for audio playback with queue system matching web version functionality - */ -export function useAudioPlayer(options?: AudioPlayerOptions) { - const player = useExpoAudioPlayer(); +export function useAudioPlayer() { const [isPlaying, setIsPlaying] = useState(false); - const [isPaused, setIsPaused] = useState(false); + const [audioInitialized, setAudioInitialized] = useState(false); const queueRef = useRef([]); const isProcessingQueueRef = useRef(false); - const currentRejectRef = useRef<((error: Error) => void) | null>(null); - const currentFileRef = useRef(null); - // Configure audio mode based on speaker/earpiece preference + // Initialize Speechmatics audio on mount useEffect(() => { - async function configureAudioMode() { + const initializeAudio = async () => { try { - if (Platform.OS === 'android') { - // On Android, use shouldRouteThroughEarpiece - await setAudioModeAsync({ - shouldRouteThroughEarpiece: !options?.useSpeaker, - }); - } else if (Platform.OS === 'ios') { - // On iOS, the audio routing is controlled by the AVAudioSession category - // The default category should work, but we can configure it if needed - // For now, we rely on the iOS default behavior which routes to the receiver - // unless the user has changed the route (e.g., via speaker button in Control Center) - } + await initialize(); + setAudioInitialized(true); + console.log('[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)'); } catch (error) { - console.warn('[AudioPlayer] Failed to configure audio mode:', error); + console.error('[AudioPlayer] Failed to initialize audio:', error); } - } - - configureAudioMode(); - }, [options?.useSpeaker]); - - // Monitor playback status - useEffect(() => { - if (!player.playing && isPlaying && !isPaused) { - // Playback finished - setIsPlaying(false); - processNextInQueue(); - } - }, [player.playing, isPlaying, isPaused]); + }; + initializeAudio(); + }, []); async function play(audioData: Blob): Promise { return new Promise((resolve, reject) => { @@ -127,92 +107,55 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { async function playAudio(audioData: Blob): Promise { return new Promise(async (resolve, reject) => { - currentRejectRef.current = reject; - try { console.log( `[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})` ); - // Convert blob to file - const file = await blobToFile(audioData); - currentFileRef.current = file; + if (!audioInitialized) { + throw new Error('Audio not initialized'); + } + + // Get PCM data from blob (server now sends PCM format) + const arrayBuffer = await audioData.arrayBuffer(); + let pcm24k = new Uint8Array(arrayBuffer); + + // Resample from 24kHz (OpenAI) to 16kHz (Speechmatics) + const pcm16k = resamplePcm24kTo16k(pcm24k); + + // Calculate total duration + const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample + const durationSec = samples / 16000; // 16kHz sample rate + + const audioSizeKb = (pcm16k.length / 1024).toFixed(2); + console.log('[AudioPlayer] 🔊 Playing audio:', audioSizeKb, 'KB, duration:', durationSec.toFixed(2), 's'); setIsPlaying(true); - setIsPaused(false); - // Load and play audio - await player.replace({ uri: file.uri }); - player.play(); + // Play entire PCM data at once through Speechmatics + playPCMData(pcm16k); - // Wait for playback to finish - const checkInterval = setInterval(() => { - if (!player.playing && player.duration > 0) { - clearInterval(checkInterval); - - const duration = player.duration; - console.log(`[AudioPlayer] Playback finished (duration: ${duration}s)`); - - setIsPlaying(false); - currentRejectRef.current = null; - - // Clean up temporary file - if (currentFileRef.current) { - try { - currentFileRef.current.delete(); - } catch (err: any) { - console.warn('[AudioPlayer] Failed to delete temp file:', err); - } - currentFileRef.current = null; - } - - resolve(duration); - } - }, 100); + // Wait for playback to finish (estimate based on duration) + setTimeout(() => { + console.log('[AudioPlayer] ✅ Playback finished'); + setIsPlaying(false); + resolve(durationSec); + }, durationSec * 1000); } catch (error) { console.error('[AudioPlayer] Error playing audio:', error); setIsPlaying(false); - currentRejectRef.current = null; - - // Clean up temporary file - if (currentFileRef.current) { - try { - currentFileRef.current.delete(); - } catch (err: any) { - console.warn('[AudioPlayer] Failed to delete temp file:', err); - } - currentFileRef.current = null; - } - reject(error); } }); } function stop(): void { - // Stop currently playing audio - if (player) { - player.pause(); + if (isPlaying) { + console.log('[AudioPlayer] 🛑 Stopping playback (interrupted)'); } + setIsPlaying(false); - setIsPaused(false); - - // Clean up temporary file - if (currentFileRef.current) { - try { - currentFileRef.current.delete(); - } catch (err: any) { - console.warn('[AudioPlayer] Failed to delete temp file:', err); - } - currentFileRef.current = null; - } - - // Reject the current playing promise if it exists - if (currentRejectRef.current) { - currentRejectRef.current(new Error('Playback stopped')); - currentRejectRef.current = null; - } // Reject all pending promises in the queue while (queueRef.current.length > 0) { @@ -223,22 +166,6 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { isProcessingQueueRef.current = false; } - function pause(): void { - if (player && isPlaying && !isPaused) { - player.pause(); - setIsPaused(true); - console.log('[AudioPlayer] Playback paused'); - } - } - - function resume(): void { - if (player && isPaused) { - player.play(); - setIsPaused(false); - console.log('[AudioPlayer] Playback resumed'); - } - } - function clearQueue(): void { // Reject all pending promises in the queue while (queueRef.current.length > 0) { @@ -250,10 +177,7 @@ export function useAudioPlayer(options?: AudioPlayerOptions) { return { play, stop, - pause, - resume, isPlaying: () => isPlaying, - isPaused: () => isPaused, clearQueue, }; } diff --git a/packages/app/hooks/use-audio-recorder.ts b/packages/app/hooks/use-audio-recorder.ts index f3bebf5a3..8f33eb35a 100644 --- a/packages/app/hooks/use-audio-recorder.ts +++ b/packages/app/hooks/use-audio-recorder.ts @@ -1,4 +1,4 @@ -import { useAudioRecorder as useExpoAudioRecorder, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio'; +import { useAudioRecorder as useExpoAudioRecorder, useAudioRecorderState, RecordingOptions, RecordingPresets, requestRecordingPermissionsAsync, setAudioModeAsync } from 'expo-audio'; import { Paths, File, Directory, FileInfo } from 'expo-file-system'; import { useState, useEffect } from 'react'; @@ -159,24 +159,25 @@ export function useAudioRecorder(config?: AudioCaptureConfig) { }; const audioRecorder = useExpoAudioRecorder(recordingOptions); + const recorderState = useAudioRecorderState(audioRecorder, 100); // Monitor audio levels if metering is enabled useEffect(() => { if (!config?.onAudioLevel || !isRecording) return; const interval = setInterval(() => { - const metering = audioRecorder.state?.metering; + const metering = recorderState.metering; if (metering !== undefined && metering !== null) { // Normalize metering value (typically ranges from -160 to 0 dB) // Convert to 0-1 range where 0 is silence and 1 is loud // We'll use -40 dB as the threshold for "loud" const normalized = Math.max(0, Math.min(1, (metering + 40) / 40)); - config.onAudioLevel(normalized); + config.onAudioLevel?.(normalized); } }, 100); // Check every 100ms return () => clearInterval(interval); - }, [isRecording, config, audioRecorder.state?.metering]); + }, [isRecording, config, recorderState.metering]); async function start(): Promise { if (isRecording) { diff --git a/packages/app/hooks/use-realtime-audio.ts b/packages/app/hooks/use-realtime-audio.ts index 3f747612a..56d39a592 100644 --- a/packages/app/hooks/use-realtime-audio.ts +++ b/packages/app/hooks/use-realtime-audio.ts @@ -10,6 +10,7 @@ export interface RealtimeAudioConfig { onError?: (error: Error) => void; volumeThreshold?: number; // RMS threshold for speech detection (0-1) silenceDuration?: number; // ms of silence before ending segment + speechStartDuration?: number; // ms of sustained volume before starting segment } export interface RealtimeAudio { @@ -19,6 +20,115 @@ export interface RealtimeAudio { isSpeaking: boolean; } +/** + * Convert raw PCM data to WAV format by adding WAV headers + * @param pcmBase64 Base64-encoded PCM data + * @param sampleRate Sample rate (default: 16000) + * @param channels Number of channels (default: 1 for mono) + * @param bitsPerSample Bits per sample (default: 16) + * @returns Base64-encoded WAV data + */ +function pcmToWav( + pcmBase64: string, + sampleRate: number = 16000, + channels: number = 1, + bitsPerSample: number = 16 +): string { + // Decode base64 PCM data + const pcmBinary = atob(pcmBase64); + const pcmLength = pcmBinary.length; + + // Calculate WAV parameters + const byteRate = sampleRate * channels * (bitsPerSample / 8); + const blockAlign = channels * (bitsPerSample / 8); + const dataSize = pcmLength; + const fileSize = 44 + dataSize; // WAV header is 44 bytes + + // Create WAV buffer + const wavBuffer = new Uint8Array(fileSize); + const view = new DataView(wavBuffer.buffer); + + // Write WAV header + let offset = 0; + + // RIFF chunk descriptor + writeString(view, offset, 'RIFF'); offset += 4; + view.setUint32(offset, fileSize - 8, true); offset += 4; // File size - 8 + writeString(view, offset, 'WAVE'); offset += 4; + + // fmt sub-chunk + writeString(view, offset, 'fmt '); offset += 4; + view.setUint32(offset, 16, true); offset += 4; // Subchunk size (16 for PCM) + view.setUint16(offset, 1, true); offset += 2; // Audio format (1 = PCM) + view.setUint16(offset, channels, true); offset += 2; + view.setUint32(offset, sampleRate, true); offset += 4; + view.setUint32(offset, byteRate, true); offset += 4; + view.setUint16(offset, blockAlign, true); offset += 2; + view.setUint16(offset, bitsPerSample, true); offset += 2; + + // data sub-chunk + writeString(view, offset, 'data'); offset += 4; + view.setUint32(offset, dataSize, true); offset += 4; + + // Copy PCM data + for (let i = 0; i < pcmLength; i++) { + wavBuffer[offset + i] = pcmBinary.charCodeAt(i); + } + + // Convert to base64 + let binary = ''; + for (let i = 0; i < wavBuffer.length; i++) { + binary += String.fromCharCode(wavBuffer[i]); + } + return btoa(binary); +} + +/** + * Helper to write string to DataView + */ +function writeString(view: DataView, offset: number, str: string): void { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } +} + +/** + * Decode base64 string to Uint8Array + */ +function decodeBase64ToUint8Array(base64: string): Uint8Array { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + +/** + * Encode Uint8Array to base64 string + */ +function uint8ArrayToBase64(bytes: Uint8Array): string { + let binary = ''; + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +/** + * Concatenate multiple Uint8Arrays into one + */ +function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array { + const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const arr of arrays) { + result.set(arr, offset); + offset += arr.length; + } + return result; +} + /** * Hook for realtime audio streaming with volume-based VAD * Uses expo-audio-studio for recording with echo cancellation and noise suppression @@ -28,13 +138,17 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio { const [isSpeaking, setIsSpeaking] = useState(false); const audioRecorder = useAudioRecorder(); - const audioBufferRef = useRef([]); + const audioBufferRef = useRef([]); const silenceStartRef = useRef(null); const isSpeakingRef = useRef(false); + const speechStartRef = useRef(null); // Track when volume first exceeded threshold + const speechConfirmedRef = useRef(false); // Track if speech has been confirmed const lastVolumeCheckRef = useRef(Date.now()); + const lastVolumeLogRef = useRef(Date.now()); - const VOLUME_THRESHOLD = config.volumeThreshold ?? 0.02; + const VOLUME_THRESHOLD = config.volumeThreshold ?? 0.2; const SILENCE_DURATION_MS = config.silenceDuration ?? 1000; + const SPEECH_START_DURATION_MS = config.speechStartDuration ?? 1000; const VOLUME_CHECK_INTERVAL = 200; // Check volume every 200ms async function requestMicrophonePermission(): Promise { @@ -84,7 +198,6 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio { } const rms = Math.sqrt(sum / sampleCount); - console.log('[RealtimeAudio] Volume:', rms.toFixed(6)); return rms; } catch (error) { console.error('[RealtimeAudio] Volume calculation error:', error); @@ -126,38 +239,84 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio { const volume = calculateVolume(audioData); const speechDetected = volume > VOLUME_THRESHOLD; - if (speechDetected && !isSpeakingRef.current) { - // Speech START - console.log('[RealtimeAudio] Speech started (volume:', volume.toFixed(4), ')'); - isSpeakingRef.current = true; + // Decode base64 chunk to binary for buffering + const decodedChunk = decodeBase64ToUint8Array(audioData); + + // Check if we're in detection phase (volume detected but not yet confirmed) + const inDetectionPhase = speechStartRef.current !== null && !isSpeakingRef.current; + + if (inDetectionPhase) { + // We're in detection phase - check timer regardless of current volume + const sustainedDuration = Date.now() - speechStartRef.current!; + + if (sustainedDuration >= SPEECH_START_DURATION_MS) { + // Timer elapsed - confirm speech + console.log('[RealtimeAudio] ✅ Speech confirmed after', sustainedDuration, 'ms (volume:', volume.toFixed(4), 'threshold:', VOLUME_THRESHOLD, ')'); + isSpeakingRef.current = true; + speechConfirmedRef.current = true; + setIsSpeaking(true); + config.onSpeechStart?.(); + audioBufferRef.current.push(decodedChunk); + silenceStartRef.current = null; + } else if (speechDetected) { + // Still in detection phase, volume still above threshold, keep buffering + audioBufferRef.current.push(decodedChunk); + + // Log progress every 500ms during detection + const logNow = Date.now(); + if (logNow - lastVolumeLogRef.current >= 500) { + console.log('[RealtimeAudio] 🔍 Detecting...', sustainedDuration, '/', SPEECH_START_DURATION_MS, 'ms (volume:', volume.toFixed(4), ')'); + lastVolumeLogRef.current = logNow; + } + } else { + // Volume dropped before timer elapsed - false start + console.log('[RealtimeAudio] ❌ False start (volume dropped after', sustainedDuration, 'ms, needed', SPEECH_START_DURATION_MS, 'ms)'); + speechStartRef.current = null; + speechConfirmedRef.current = false; + audioBufferRef.current = []; + } + } else if (speechDetected && !isSpeakingRef.current) { + // First volume detection - start timer and buffering + console.log('[RealtimeAudio] 🎤 Volume spike detected! Starting detection... (volume:', volume.toFixed(4), 'threshold:', VOLUME_THRESHOLD, ')'); + speechStartRef.current = Date.now(); + speechConfirmedRef.current = false; + audioBufferRef.current = [decodedChunk]; silenceStartRef.current = null; - audioBufferRef.current = [audioData]; - setIsSpeaking(true); - config.onSpeechStart?.(); } else if (speechDetected && isSpeakingRef.current) { - // Continuing speech - audioBufferRef.current.push(audioData); + // Continuing confirmed speech - just buffer, no logs + audioBufferRef.current.push(decodedChunk); silenceStartRef.current = null; } else if (!speechDetected && isSpeakingRef.current) { - // Potential speech END - audioBufferRef.current.push(audioData); + // Potential speech END (after confirmed speech) + audioBufferRef.current.push(decodedChunk); if (silenceStartRef.current === null) { silenceStartRef.current = Date.now(); + console.log('[RealtimeAudio] 🔇 Silence detected, waiting', SILENCE_DURATION_MS, 'ms to confirm end...'); } else { const silenceDuration = Date.now() - silenceStartRef.current; if (silenceDuration >= SILENCE_DURATION_MS) { // Speech END confirmed - console.log('[RealtimeAudio] Speech ended after', silenceDuration, 'ms silence'); + const totalChunks = audioBufferRef.current.length; + console.log('[RealtimeAudio] 🛑 Speech ended after', silenceDuration, 'ms silence (captured', totalChunks, 'chunks)'); isSpeakingRef.current = false; + speechStartRef.current = null; + speechConfirmedRef.current = false; silenceStartRef.current = null; setIsSpeaking(false); config.onSpeechEnd?.(); // Send buffered audio segment if (audioBufferRef.current.length > 0) { - const combinedAudio = audioBufferRef.current.join(''); - config.onAudioSegment?.(combinedAudio); + // Concatenate all binary chunks + const combinedBinary = concatenateUint8Arrays(audioBufferRef.current); + const audioSizeKb = (combinedBinary.length / 1024).toFixed(2); + console.log('[RealtimeAudio] 📤 Sending audio segment:', audioSizeKb, 'KB'); + // Convert to base64 + const combinedPcmBase64 = uint8ArrayToBase64(combinedBinary); + // Convert PCM to WAV format + const wavAudio = pcmToWav(combinedPcmBase64, 16000, 1, 16); + config.onAudioSegment?.(wavAudio); audioBufferRef.current = []; } } @@ -165,13 +324,14 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio { } } else if (isSpeakingRef.current) { // Just buffer if we're speaking but not checking volume yet - audioBufferRef.current.push(audioData); + const decodedChunk = decodeBase64ToUint8Array(audioData); + audioBufferRef.current.push(decodedChunk); } }, }); setIsActive(true); - console.log('[RealtimeAudio] Realtime audio started successfully'); + console.log('[RealtimeAudio] 🎙️ Audio capture started (threshold:', VOLUME_THRESHOLD, 'duration:', SPEECH_START_DURATION_MS, 'ms)'); } catch (error) { console.error('[RealtimeAudio] Start error:', error); const err = error instanceof Error ? error : new Error(String(error)); @@ -198,6 +358,8 @@ export function useRealtimeAudio(config: RealtimeAudioConfig): RealtimeAudio { // Reset state audioBufferRef.current = []; isSpeakingRef.current = false; + speechStartRef.current = null; + speechConfirmedRef.current = false; silenceStartRef.current = null; setIsActive(false); setIsSpeaking(false); diff --git a/packages/app/hooks/use-speechmatics-audio.ts b/packages/app/hooks/use-speechmatics-audio.ts new file mode 100644 index 000000000..4372c9514 --- /dev/null +++ b/packages/app/hooks/use-speechmatics-audio.ts @@ -0,0 +1,352 @@ +import { useState, useEffect, useRef, useCallback } from "react"; +import { + initialize, + toggleRecording, + tearDown, + useExpoTwoWayAudioEventListener, + type MicrophoneDataCallback, + type VolumeLevelCallback, +} from "@speechmatics/expo-two-way-audio"; + +export interface SpeechmaticsAudioConfig { + onAudioSegment?: (audioData: string) => void; + onSpeechStart?: () => void; + onSpeechEnd?: () => void; + onError?: (error: Error) => void; + volumeThreshold: number; // Volume threshold for speech detection (0-1) + silenceDuration: number; // ms of silence before ending segment + speechConfirmationDuration: number; // ms of sustained speech before confirming + detectionGracePeriod: number; // ms grace period for volume dips during detection +} + +export interface SpeechmaticsAudio { + start: () => Promise; + stop: () => Promise; + isActive: boolean; + isSpeaking: boolean; + isDetecting: boolean; +} + +/** + * Convert raw PCM data to WAV format by adding WAV headers + */ +function pcmToWav( + pcmBase64: string, + sampleRate: number = 16000, + channels: number = 1, + bitsPerSample: number = 16 +): string { + const pcmBinary = atob(pcmBase64); + const pcmLength = pcmBinary.length; + + const byteRate = sampleRate * channels * (bitsPerSample / 8); + const blockAlign = channels * (bitsPerSample / 8); + const dataSize = pcmLength; + const fileSize = 44 + dataSize; + + const wavBuffer = new Uint8Array(fileSize); + const view = new DataView(wavBuffer.buffer); + + let offset = 0; + + // RIFF chunk descriptor + writeString(view, offset, "RIFF"); + offset += 4; + view.setUint32(offset, fileSize - 8, true); + offset += 4; + writeString(view, offset, "WAVE"); + offset += 4; + + // fmt sub-chunk + writeString(view, offset, "fmt "); + offset += 4; + view.setUint32(offset, 16, true); + offset += 4; + view.setUint16(offset, 1, true); + offset += 2; + view.setUint16(offset, channels, true); + offset += 2; + view.setUint32(offset, sampleRate, true); + offset += 4; + view.setUint32(offset, byteRate, true); + offset += 4; + view.setUint16(offset, blockAlign, true); + offset += 2; + view.setUint16(offset, bitsPerSample, true); + offset += 2; + + // data sub-chunk + writeString(view, offset, "data"); + offset += 4; + view.setUint32(offset, dataSize, true); + offset += 4; + + // Copy PCM data + for (let i = 0; i < pcmLength; i++) { + wavBuffer[offset + i] = pcmBinary.charCodeAt(i); + } + + // Convert to base64 + let binary = ""; + for (let i = 0; i < wavBuffer.length; i++) { + binary += String.fromCharCode(wavBuffer[i]); + } + return btoa(binary); +} + +function writeString(view: DataView, offset: number, str: string): void { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } +} + +function uint8ArrayToBase64(bytes: Uint8Array): string { + let binary = ""; + for (let i = 0; i < bytes.length; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +function concatenateUint8Arrays(arrays: Uint8Array[]): Uint8Array { + const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const arr of arrays) { + result.set(arr, offset); + offset += arr.length; + } + return result; +} + +/** + * Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio + */ +export function useSpeechmaticsAudio( + config: SpeechmaticsAudioConfig +): SpeechmaticsAudio { + const [isActive, setIsActive] = useState(false); + const [isSpeaking, setIsSpeaking] = useState(false); + const [isDetecting, setIsDetecting] = useState(false); + const [audioInitialized, setAudioInitialized] = useState(false); + + const audioBufferRef = useRef([]); + const silenceStartRef = useRef(null); + const isSpeakingRef = useRef(false); + const speechDetectionStartRef = useRef(null); + const speechConfirmedRef = useRef(false); + const detectionSilenceStartRef = useRef(null); + + const VOLUME_THRESHOLD = config.volumeThreshold; + const SILENCE_DURATION_MS = config.silenceDuration; + const SPEECH_CONFIRMATION_MS = config.speechConfirmationDuration; + const DETECTION_GRACE_PERIOD_MS = config.detectionGracePeriod; + + // Initialize audio on mount + useEffect(() => { + const initializeAudio = async () => { + try { + await initialize(); + setAudioInitialized(true); + console.log("[SpeechmaticsAudio] Audio initialized"); + } catch (error) { + console.error("[SpeechmaticsAudio] Initialization error:", error); + const err = error instanceof Error ? error : new Error(String(error)); + config.onError?.(err); + } + }; + initializeAudio(); + + return () => { + tearDown(); + }; + }, []); + + // Listen to microphone data + useExpoTwoWayAudioEventListener( + "onMicrophoneData", + useCallback( + (event) => { + if (!isActive) return; + + const pcmData: Uint8Array = event.data; + + // Buffer the audio chunk if we're detecting or speaking + // Start buffering from first spike to capture beginning of speech + if (speechDetectionStartRef.current !== null || isSpeakingRef.current) { + audioBufferRef.current.push(pcmData); + } + }, + [isActive] + ) + ); + + // Listen to volume level for VAD + useExpoTwoWayAudioEventListener( + "onInputVolumeLevelData", + useCallback( + (event) => { + if (!isActive) return; + + const volume: number = event.data; + const speechDetected = volume > VOLUME_THRESHOLD; + + // console.log('[SpeechmaticsAudio] Volume:', volume.toFixed(6)); + + if (speechDetected && !isSpeakingRef.current && !speechConfirmedRef.current) { + // Initial speech detection - start tracking + if (speechDetectionStartRef.current === null) { + console.log( + "[SpeechmaticsAudio] Speech started (volume:", + volume.toFixed(4), + ")" + ); + speechDetectionStartRef.current = Date.now(); + detectionSilenceStartRef.current = null; + audioBufferRef.current = []; + setIsDetecting(true); + } else { + // Volume is back above threshold - reset grace period + detectionSilenceStartRef.current = null; + + // Check if speech has been sustained long enough + const speechDuration = Date.now() - speechDetectionStartRef.current; + if (speechDuration >= SPEECH_CONFIRMATION_MS) { + // Speech CONFIRMED + console.log( + "[SpeechmaticsAudio] Speech confirmed after", + speechDuration, + "ms" + ); + isSpeakingRef.current = true; + speechConfirmedRef.current = true; + silenceStartRef.current = null; + setIsDetecting(false); + setIsSpeaking(true); + config.onSpeechStart?.(); + } + } + } else if (speechDetected && isSpeakingRef.current && speechConfirmedRef.current) { + // Continuing confirmed speech + silenceStartRef.current = null; + } else if (!speechDetected && !speechConfirmedRef.current && speechDetectionStartRef.current !== null) { + // Volume dropped during detection phase - apply grace period + if (detectionSilenceStartRef.current === null) { + detectionSilenceStartRef.current = Date.now(); + } else { + const graceDuration = Date.now() - detectionSilenceStartRef.current; + if (graceDuration >= DETECTION_GRACE_PERIOD_MS) { + // Grace period expired - cancel detection + console.log( + "[SpeechmaticsAudio] Speech detection cancelled after", + graceDuration, + "ms grace period" + ); + speechDetectionStartRef.current = null; + detectionSilenceStartRef.current = null; + audioBufferRef.current = []; + setIsDetecting(false); + } + } + } else if (!speechDetected && isSpeakingRef.current && speechConfirmedRef.current) { + // Potential speech END + if (silenceStartRef.current === null) { + silenceStartRef.current = Date.now(); + } else { + const silenceDuration = Date.now() - silenceStartRef.current; + if (silenceDuration >= SILENCE_DURATION_MS) { + // Speech END confirmed + console.log( + "[SpeechmaticsAudio] Speech ended after", + silenceDuration, + "ms silence" + ); + isSpeakingRef.current = false; + speechConfirmedRef.current = false; + speechDetectionStartRef.current = null; + silenceStartRef.current = null; + setIsSpeaking(false); + config.onSpeechEnd?.(); + + // Send buffered audio segment + if (audioBufferRef.current.length > 0) { + const combinedBinary = concatenateUint8Arrays( + audioBufferRef.current + ); + const combinedPcmBase64 = uint8ArrayToBase64(combinedBinary); + const wavAudio = pcmToWav(combinedPcmBase64, 16000, 1, 16); + config.onAudioSegment?.(wavAudio); + audioBufferRef.current = []; + } + } + } + } + }, + [isActive, VOLUME_THRESHOLD, SILENCE_DURATION_MS, SPEECH_CONFIRMATION_MS, DETECTION_GRACE_PERIOD_MS, config] + ) + ); + + async function start(): Promise { + if (!audioInitialized) { + throw new Error("Audio not initialized"); + } + + if (isActive) { + console.log("[SpeechmaticsAudio] Already active"); + return; + } + + try { + console.log("[SpeechmaticsAudio] Starting audio capture..."); + + // Start recording + toggleRecording(true); + + setIsActive(true); + console.log("[SpeechmaticsAudio] Audio capture started successfully"); + } catch (error) { + console.error("[SpeechmaticsAudio] Start error:", error); + const err = error instanceof Error ? error : new Error(String(error)); + config.onError?.(err); + await stop(); + throw error; + } + } + + async function stop(): Promise { + console.log("[SpeechmaticsAudio] Stopping audio capture..."); + + // Stop recording + toggleRecording(false); + + // Reset state + audioBufferRef.current = []; + isSpeakingRef.current = false; + speechConfirmedRef.current = false; + speechDetectionStartRef.current = null; + detectionSilenceStartRef.current = null; + silenceStartRef.current = null; + setIsActive(false); + setIsSpeaking(false); + setIsDetecting(false); + + console.log("[SpeechmaticsAudio] Audio capture stopped"); + } + + // Cleanup on unmount + useEffect(() => { + return () => { + if (isActive) { + stop(); + } + }; + }, [isActive]); + + return { + start, + stop, + isActive, + isSpeaking, + isDetecting, + }; +} diff --git a/packages/app/hooks/use-websocket.ts b/packages/app/hooks/use-websocket.ts index 6baa10991..c3557492c 100644 --- a/packages/app/hooks/use-websocket.ts +++ b/packages/app/hooks/use-websocket.ts @@ -58,7 +58,7 @@ export function useWebSocket(url: string, conversationId?: string | null): UseWe // Only session messages trigger handlers if (wsMessage.type === 'session') { const sessionMessage = wsMessage.message; - console.log(`[WS] Received session message type: ${sessionMessage.type}`); + // console.log(`[WS] Received session message type: ${sessionMessage.type}`); // Track conversation ID when loaded if (sessionMessage.type === 'conversation_loaded') { diff --git a/packages/app/metro.config.js b/packages/app/metro.config.js index eaa4fb851..f3678d3b4 100644 --- a/packages/app/metro.config.js +++ b/packages/app/metro.config.js @@ -1,6 +1,5 @@ const { getDefaultConfig } = require("expo/metro-config"); -const { withNativewind } = require("nativewind/metro"); const config = getDefaultConfig(__dirname); -module.exports = withNativewind(config); +module.exports = config; diff --git a/packages/app/package.json b/packages/app/package.json index f91a26184..2fce63cf7 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -17,6 +17,8 @@ "@react-navigation/elements": "^2.6.3", "@react-navigation/native": "^7.1.8", "@siteed/expo-audio-studio": "^2.18.1", + "@speechmatics/expo-two-way-audio": "^0.1.2", + "buffer": "^6.0.3", "expo": "^54.0.18", "expo-audio": "~1.0.13", "expo-av": "^16.0.7", @@ -37,7 +39,6 @@ "expo-updates": "~29.0.12", "expo-web-browser": "~15.0.8", "lucide-react-native": "^0.546.0", - "nativewind": "^5.0.0-preview.2", "react": "19.1.0", "react-dom": "19.1.0", "react-native": "0.81.4", @@ -49,18 +50,16 @@ "react-native-safe-area-context": "~5.6.0", "react-native-screens": "~4.16.0", "react-native-svg": "^15.14.0", + "react-native-unistyles": "^3.0.15", "react-native-web": "~0.21.0", "react-native-webrtc": "^124.0.7", "react-native-worklets": "0.5.1" }, "devDependencies": { - "@tailwindcss/postcss": "^4.1.15", "@types/react": "~19.2.0", "eas-cli": "^16.24.1", "eslint": "^9.25.0", "eslint-config-expo": "~10.0.0", - "postcss": "^8.4.49", - "tailwindcss": "^4.1.15", "typescript": "~5.9.2" }, "overrides": { diff --git a/packages/app/postcss.config.mjs b/packages/app/postcss.config.mjs deleted file mode 100644 index c2ddf7482..000000000 --- a/packages/app/postcss.config.mjs +++ /dev/null @@ -1,5 +0,0 @@ -export default { - plugins: { - "@tailwindcss/postcss": {}, - }, -}; diff --git a/packages/app/styles/theme.ts b/packages/app/styles/theme.ts new file mode 100644 index 000000000..716246256 --- /dev/null +++ b/packages/app/styles/theme.ts @@ -0,0 +1,153 @@ +export const theme = { + colors: { + // Base colors + white: "#ffffff", + black: "#000000", + + // Zinc scale (primary gray palette) + zinc: { + 50: "#fafafa", + 100: "#f4f4f5", + 200: "#e4e4e7", + 300: "#d4d4d8", + 400: "#a1a1aa", + 500: "#71717a", + 600: "#52525b", + 700: "#3f3f46", + 800: "#27272a", + 900: "#18181b", + }, + + // Gray scale + gray: { + 50: "#f9fafb", + 100: "#f3f4f6", + 200: "#e5e7eb", + 300: "#d1d5db", + 400: "#9ca3af", + 500: "#6b7280", + 600: "#4b5563", + 700: "#374151", + 800: "#1f2937", + 900: "#111827", + }, + + // Slate scale + slate: { + 200: "#e2e8f0", + }, + + // Blue scale + blue: { + 50: "#eff6ff", + 100: "#dbeafe", + 200: "#bfdbfe", + 300: "#93c5fd", + 400: "#60a5fa", + 500: "#3b82f6", + 600: "#2563eb", + 700: "#1d4ed8", + 800: "#1e40af", + 900: "#1e3a8a", + 950: "#172554", + }, + + // Green scale + green: { + 100: "#dcfce7", + 200: "#bbf7d0", + 500: "#22c55e", + 600: "#16a34a", + 800: "#166534", + 900: "#14532d", + }, + + // Red scale + red: { + 100: "#fee2e2", + 200: "#fecaca", + 500: "#ef4444", + 600: "#dc2626", + 800: "#991b1b", + 900: "#7f1d1d", + }, + + // Teal scale + teal: { + 200: "#99f6e4", + }, + + // Amber scale + amber: { + 500: "#f59e0b", + }, + + // Purple scale + purple: { + 500: "#a855f7", + 600: "#9333ea", + }, + + // Orange scale + orange: { + 500: "#f97316", + 600: "#ea580c", + }, + }, + + spacing: { + 0: 0, + 1: 4, + 2: 8, + 3: 12, + 4: 16, + 6: 24, + 8: 32, + 12: 48, + 16: 64, + 20: 80, + 24: 96, + 32: 128, + }, + + fontSize: { + xs: 10, + sm: 12, + base: 14, + lg: 16, + xl: 18, + "2xl": 20, + "3xl": 24, + }, + + fontWeight: { + normal: "normal" as const, + semibold: "600" as const, + bold: "bold" as const, + }, + + borderRadius: { + none: 0, + sm: 2, + base: 4, + md: 6, + lg: 8, + xl: 12, + "2xl": 16, + full: 9999, + }, + + borderWidth: { + 0: 0, + 1: 1, + 2: 2, + }, + + opacity: { + 0: 0, + 50: 0.5, + 100: 1, + }, +} as const; + +export type Theme = typeof theme; diff --git a/packages/app/styles/unistyles.ts b/packages/app/styles/unistyles.ts new file mode 100644 index 000000000..af8b1befe --- /dev/null +++ b/packages/app/styles/unistyles.ts @@ -0,0 +1,21 @@ +import { StyleSheet } from "react-native-unistyles"; +import { theme } from "./theme"; + +// Configure Unistyles with our theme +StyleSheet.configure({ + themes: { + dark: theme, + }, + settings: { + initialTheme: "dark", + }, +}); + +// Type augmentation for TypeScript +type AppThemes = { + dark: typeof theme; +}; + +declare module "react-native-unistyles" { + export interface UnistylesThemes extends AppThemes {} +} diff --git a/packages/server/src/server/agent/tts-openai.ts b/packages/server/src/server/agent/tts-openai.ts index f97890465..57408cdb0 100644 --- a/packages/server/src/server/agent/tts-openai.ts +++ b/packages/server/src/server/agent/tts-openai.ts @@ -20,14 +20,14 @@ export function initializeTTS(ttsConfig: TTSConfig): void { config = { model: "tts-1", voice: "alloy", - responseFormat: "mp3", + responseFormat: "pcm", ...ttsConfig, }; openaiClient = new OpenAI({ apiKey: ttsConfig.apiKey, }); console.log( - `✓ TTS (OpenAI) initialized with voice: ${config.voice}, model: ${config.model}` + `✓ TTS (OpenAI) initialized with voice: ${config.voice}, model: ${config.model}, format: ${config.responseFormat}` ); } @@ -55,7 +55,7 @@ export async function synthesizeSpeech(text: string): Promise { voice: config.voice!, input: text, // speed: 1.2, - response_format: config.responseFormat as "mp3" | "opus" | "aac" | "flac", + response_format: config.responseFormat as "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm", }); // Response is a stream, convert to buffer diff --git a/packages/server/src/server/index.ts b/packages/server/src/server/index.ts index 75fad07a0..b92d15c23 100644 --- a/packages/server/src/server/index.ts +++ b/packages/server/src/server/index.ts @@ -13,7 +13,10 @@ import { function createServer() { const app = express(); - // Basic authentication + // Serve static files from public directory (no auth required for APK downloads) + app.use("/public", express.static("public")); + + // Basic authentication (skip for /public routes) app.use( basicAuth({ users: { mo: "bo" }, @@ -85,7 +88,7 @@ function main() { apiKey, voice: ttsVoice, model: ttsModel, - responseFormat: "mp3", + responseFormat: "pcm", }); } else { console.warn(