mirror of
https://github.com/getpaseo/paseo.git
synced 2026-07-29 12:01:31 +00:00
Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c45b414476 | ||
|
|
73bd0c1dba | ||
|
|
502dd2d2e2 | ||
|
|
80a2f8052c | ||
|
|
730f34753a | ||
|
|
013f573cff | ||
|
|
ee8a81d2e1 | ||
|
|
bed03be332 | ||
|
|
debe4b543b | ||
|
|
1c601e09b6 | ||
|
|
ba54aae206 | ||
|
|
36961f83bf | ||
|
|
6db99c03cb | ||
|
|
999d100464 | ||
|
|
9190d86148 | ||
|
|
4b5cb5a3e2 | ||
|
|
847e1d2c60 | ||
|
|
970df6478b | ||
|
|
3371f17606 | ||
|
|
0f65fe894c | ||
|
|
2ac27a3e3e | ||
|
|
defcc54af2 | ||
|
|
fe84c5c9a3 | ||
|
|
2ff8041e15 | ||
|
|
1d3e551e7f | ||
|
|
79795b6f49 | ||
|
|
16da3300e3 |
130
.github/workflows/desktop-release.yml
vendored
130
.github/workflows/desktop-release.yml
vendored
@@ -59,13 +59,11 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source_tag="${SOURCE_TAG}"
|
||||
release_tag="$source_tag"
|
||||
for prefix in desktop-windows-v desktop-linux-v desktop-macos-v desktop-v; do
|
||||
if [[ "$source_tag" == ${prefix}* ]]; then
|
||||
release_tag="v${source_tag#${prefix}}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$source_tag" =~ ^(desktop-(windows|linux|macos)-|desktop-)?v([0-9]+\.[0-9]+\.[0-9]+) ]]; then
|
||||
release_tag="v${BASH_REMATCH[3]}"
|
||||
else
|
||||
release_tag="$source_tag"
|
||||
fi
|
||||
echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
|
||||
|
||||
version="${release_tag#v}"
|
||||
@@ -253,13 +251,11 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source_tag="${SOURCE_TAG}"
|
||||
release_tag="$source_tag"
|
||||
for prefix in desktop-windows-v desktop-linux-v desktop-macos-v desktop-v; do
|
||||
if [[ "$source_tag" == ${prefix}* ]]; then
|
||||
release_tag="v${source_tag#${prefix}}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$source_tag" =~ ^(desktop-(windows|linux|macos)-|desktop-)?v([0-9]+\.[0-9]+\.[0-9]+) ]]; then
|
||||
release_tag="v${BASH_REMATCH[3]}"
|
||||
else
|
||||
release_tag="$source_tag"
|
||||
fi
|
||||
echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
|
||||
|
||||
version="${release_tag#v}"
|
||||
@@ -369,23 +365,87 @@ jobs:
|
||||
fi
|
||||
echo "RELEASE_DRAFT=$release_draft" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build and publish Linux Tauri release
|
||||
- name: Build Linux Tauri release
|
||||
if: env.IS_SMOKE_TAG != 'true'
|
||||
uses: tauri-apps/tauri-action@v0
|
||||
id: linux_tauri
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
||||
NO_STRIP: "true"
|
||||
NO_STRIP: "1"
|
||||
APPIMAGE_EXTRACT_AND_RUN: "1"
|
||||
with:
|
||||
projectPath: packages/desktop
|
||||
tagName: ${{ env.RELEASE_TAG }}
|
||||
releaseName: Paseo ${{ env.RELEASE_TAG }}
|
||||
releaseBody: See the assets to download and install this version.
|
||||
releaseDraft: ${{ env.RELEASE_DRAFT }}
|
||||
prerelease: false
|
||||
args: --bundles appimage
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
npm run tauri --workspace=@getpaseo/desktop build -- --bundles appimage
|
||||
|
||||
- name: Attempt manual Linux AppImage fallback
|
||||
if: env.IS_SMOKE_TAG != 'true' && steps.linux_tauri.outcome == 'failure'
|
||||
env:
|
||||
TAURI_SIGNING_PRIVATE_KEY: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY }}
|
||||
TAURI_SIGNING_PRIVATE_KEY_PASSWORD: ${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
appimage_dir="packages/desktop/src-tauri/target/release/bundle/appimage"
|
||||
appdir_path="$appimage_dir/Paseo.AppDir"
|
||||
canonical_appimage="$appimage_dir/Paseo_${DESKTOP_VERSION}_amd64.AppImage"
|
||||
existing_appimage="$(find "$appimage_dir" -maxdepth 1 -type f -name '*.AppImage' | head -n 1)"
|
||||
if [ -n "$existing_appimage" ]; then
|
||||
if [ "$existing_appimage" != "$canonical_appimage" ]; then
|
||||
mv "$existing_appimage" "$canonical_appimage"
|
||||
fi
|
||||
if [ ! -f "$canonical_appimage.sig" ]; then
|
||||
npx tauri signer sign "$canonical_appimage"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -d "$appdir_path" ]; then
|
||||
echo "::error::AppDir was not generated at $appdir_path"
|
||||
exit 1
|
||||
fi
|
||||
cp --remove-destination "$appdir_path/usr/share/applications/Paseo.desktop" "$appdir_path/Paseo.desktop"
|
||||
cp --remove-destination "$appdir_path/Paseo.png" "$appdir_path/.DirIcon"
|
||||
env | sort | grep -E '^(APPIMAGE|DESKTOP_VERSION|NO_STRIP|RELEASE_TAG|SOURCE_TAG|TAURI_)' || true
|
||||
tools_dir="$(mktemp -d)"
|
||||
curl -fsSL https://github.com/AppImage/appimagetool/releases/download/continuous/appimagetool-x86_64.AppImage -o "$tools_dir/appimagetool-x86_64.AppImage"
|
||||
chmod +x "$tools_dir/appimagetool-x86_64.AppImage"
|
||||
ARCH=x86_64 APPIMAGE_EXTRACT_AND_RUN=1 "$tools_dir/appimagetool-x86_64.AppImage" "$appdir_path" "$canonical_appimage"
|
||||
npx tauri signer sign "$canonical_appimage"
|
||||
|
||||
- name: Fail Linux release when AppImage bundling fails
|
||||
if: env.IS_SMOKE_TAG != 'true' && steps.linux_tauri.outcome == 'failure'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
shopt -s nullglob
|
||||
assets=(
|
||||
packages/desktop/src-tauri/target/release/bundle/appimage/*.AppImage
|
||||
packages/desktop/src-tauri/target/release/bundle/appimage/*.AppImage.sig
|
||||
)
|
||||
if [ "${#assets[@]}" -eq 0 ]; then
|
||||
echo "::error::Linux AppImage assets are still missing after the manual fallback."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload Linux release assets
|
||||
if: env.IS_SMOKE_TAG != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
shopt -s nullglob
|
||||
assets=(
|
||||
packages/desktop/src-tauri/target/release/bundle/appimage/*.AppImage
|
||||
packages/desktop/src-tauri/target/release/bundle/appimage/*.AppImage.sig
|
||||
)
|
||||
if [ "${#assets[@]}" -eq 0 ]; then
|
||||
echo "::error::No Linux AppImage assets were produced."
|
||||
exit 1
|
||||
fi
|
||||
printf 'Uploading Linux assets:\n%s\n' "${assets[@]}"
|
||||
gh release upload "$RELEASE_TAG" "${assets[@]}" --repo "${{ github.repository }}" --clobber
|
||||
|
||||
- name: Build Linux app (smoke only)
|
||||
if: env.IS_SMOKE_TAG == 'true'
|
||||
@@ -409,13 +469,11 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
source_tag="${SOURCE_TAG}"
|
||||
release_tag="$source_tag"
|
||||
for prefix in desktop-windows-v desktop-linux-v desktop-macos-v desktop-v; do
|
||||
if [[ "$source_tag" == ${prefix}* ]]; then
|
||||
release_tag="v${source_tag#${prefix}}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$source_tag" =~ ^(desktop-(windows|linux|macos)-|desktop-)?v([0-9]+\.[0-9]+\.[0-9]+) ]]; then
|
||||
release_tag="v${BASH_REMATCH[3]}"
|
||||
else
|
||||
release_tag="$source_tag"
|
||||
fi
|
||||
echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
|
||||
|
||||
version="${release_tag#v}"
|
||||
@@ -485,7 +543,11 @@ jobs:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build web app for Tauri
|
||||
run: npm run build:web --workspace=@getpaseo/app
|
||||
shell: pwsh
|
||||
run: |
|
||||
$patchPath = (Get-Item "$env:GITHUB_WORKSPACE/scripts/metro-config-windows-loader-patch.cjs").FullName
|
||||
$env:NODE_OPTIONS = "--require=$patchPath"
|
||||
npm run build:web --workspace=@getpaseo/app
|
||||
|
||||
- name: Build managed runtime
|
||||
run: npm run prepare:managed-runtime --workspace=@getpaseo/desktop
|
||||
|
||||
17
CHANGELOG.md
17
CHANGELOG.md
@@ -1,5 +1,22 @@
|
||||
# Changelog
|
||||
|
||||
## 0.1.27 - 2026-03-13
|
||||
|
||||
### Added
|
||||
- Added voice runtime with new audio engine architecture for voice interactions.
|
||||
- Added Grep tool support in Claude tool-call mapping.
|
||||
- Added ability to open workspace files directly from agent chat messages.
|
||||
- Added desktop notifications via a custom native bridge.
|
||||
|
||||
### Improved
|
||||
- Improved image picker, markdown rendering, and UI interactions.
|
||||
- Improved shell environment detection using shell-env.
|
||||
|
||||
### Fixed
|
||||
- Fixed platform-specific markdown link rendering.
|
||||
- Fixed Linux AppImage CLI resource paths.
|
||||
- Fixed Codex replacement stream being killed by stale turn notifications.
|
||||
|
||||
## 0.1.26 - 2026-03-12
|
||||
|
||||
### Added
|
||||
|
||||
438
package-lock.json
generated
438
package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "paseo",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "paseo",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"hasInstallScript": true,
|
||||
"license": "AGPL-3.0-or-later",
|
||||
"workspaces": [
|
||||
@@ -6402,128 +6402,6 @@
|
||||
"node": ">=20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex": {
|
||||
"version": "0.114.0",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0.tgz",
|
||||
"integrity": "sha512-HMo8LRR6CtfKkaa28xvFK6eOarmBFTDfsrS9GJtEoaspGGemFok494CpafDspiTZaHZZGHfSUe5SWTUxYq7OaA==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"codex": "bin/codex.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@openai/codex-darwin-arm64": "npm:@openai/codex@0.114.0-darwin-arm64",
|
||||
"@openai/codex-darwin-x64": "npm:@openai/codex@0.114.0-darwin-x64",
|
||||
"@openai/codex-linux-arm64": "npm:@openai/codex@0.114.0-linux-arm64",
|
||||
"@openai/codex-linux-x64": "npm:@openai/codex@0.114.0-linux-x64",
|
||||
"@openai/codex-win32-arm64": "npm:@openai/codex@0.114.0-win32-arm64",
|
||||
"@openai/codex-win32-x64": "npm:@openai/codex@0.114.0-win32-x64"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-darwin-arm64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.114.0-darwin-arm64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-darwin-arm64.tgz",
|
||||
"integrity": "sha512-c9dlgo9O+66alr3s3G36fKE3KaO2+xrLZ/QcU3oujDruV86pqgVSEK2njHi+WIyyOa6m2AyWxdaHLEbKxPCNRg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-darwin-x64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.114.0-darwin-x64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-darwin-x64.tgz",
|
||||
"integrity": "sha512-oPAPeZSaJZ1AQneFPSJu44uqbZr63Bxut9FOTWZwuSqYx4YEees7i0HJmJcqQc0XnCbYtHJdqo/i07Uv374NLw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-linux-arm64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.114.0-linux-arm64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-linux-arm64.tgz",
|
||||
"integrity": "sha512-oUFZGZgMiEqmo85C+dfhckpVApH25alLWfwBgfKr2IRgdx/tovy8vN101f6kj01E6nDDe4LfJdNSZGtZht4fRA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-linux-x64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.114.0-linux-x64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-linux-x64.tgz",
|
||||
"integrity": "sha512-MF6RhxfBoccccd5CYII2C7TL2TvYzombBQhanDZfQAajr6n7IuoazCmoHDlrFHS4AcSw9bYA4MRlrwEjbEjgsw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-win32-arm64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.114.0-win32-arm64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-win32-arm64.tgz",
|
||||
"integrity": "sha512-CnRMHopj3en9aqQ2UaDW7EgpEGkxHdZVLLRq2cOy5D0HyuzF6Qb6595ADoFVJHEmPeN5Iz/KUbiGs5GLDjUwOA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/codex-win32-x64": {
|
||||
"name": "@openai/codex",
|
||||
"version": "0.114.0-win32-x64",
|
||||
"resolved": "https://registry.npmjs.org/@openai/codex/-/codex-0.114.0-win32-x64.tgz",
|
||||
"integrity": "sha512-ztRsH5Z+gPVZ5ZInx6HzXsTFFkuEdjk3Ay0UGVOhRRCWvevXQi/SL4eU8hwysCYrs8K/WRq3mo4c95w9zQ2wLw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "Apache-2.0",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
}
|
||||
},
|
||||
"node_modules/@opencode-ai/sdk": {
|
||||
"version": "1.2.6",
|
||||
"resolved": "https://registry.npmjs.org/@opencode-ai/sdk/-/sdk-1.2.6.tgz",
|
||||
@@ -12835,6 +12713,18 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/default-shell": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/default-shell/-/default-shell-2.2.0.tgz",
|
||||
"integrity": "sha512-sPpMZcVhRQ0nEMDtuMJ+RtCxt7iHPAMBU+I4tAlo5dU1sjRpNax0crj6nR3qKpvVnckaQ9U38enXcwW9nZJeCw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/defaults": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz",
|
||||
@@ -14462,6 +14352,35 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/execa": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
|
||||
"integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cross-spawn": "^7.0.3",
|
||||
"get-stream": "^6.0.0",
|
||||
"human-signals": "^2.1.0",
|
||||
"is-stream": "^2.0.0",
|
||||
"merge-stream": "^2.0.0",
|
||||
"npm-run-path": "^4.0.1",
|
||||
"onetime": "^5.1.2",
|
||||
"signal-exit": "^3.0.3",
|
||||
"strip-final-newline": "^2.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sindresorhus/execa?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/execa/node_modules/signal-exit": {
|
||||
"version": "3.0.7",
|
||||
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
|
||||
"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/expect-type": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz",
|
||||
@@ -17102,6 +17021,18 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/get-stream": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
|
||||
"integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/get-symbol-description": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/get-symbol-description/-/get-symbol-description-1.1.0.tgz",
|
||||
@@ -17686,6 +17617,15 @@
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/human-signals": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
|
||||
"integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=10.17.0"
|
||||
}
|
||||
},
|
||||
"node_modules/humanize-ms": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
|
||||
@@ -18346,7 +18286,6 @@
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
|
||||
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
@@ -20787,7 +20726,6 @@
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
|
||||
"integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
@@ -21368,6 +21306,18 @@
|
||||
"integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/npm-run-path": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
|
||||
"integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"path-key": "^3.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/nth-check": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
|
||||
@@ -21586,7 +21536,6 @@
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
|
||||
"integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"mimic-fn": "^2.1.0"
|
||||
@@ -21682,161 +21631,6 @@
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/opencode-ai": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-ai/-/opencode-ai-1.2.24.tgz",
|
||||
"integrity": "sha512-LaSoATkVEF6jyXNnAPrkqYpmHsZfuf2uPLDSEGJkL4hT6HYFq8LkoknHDjCGIiE5Be3MmId4+l17eSIKDfmddw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"opencode": "bin/opencode"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"opencode-darwin-arm64": "1.2.24",
|
||||
"opencode-darwin-x64": "1.2.24",
|
||||
"opencode-darwin-x64-baseline": "1.2.24",
|
||||
"opencode-linux-arm64": "1.2.24",
|
||||
"opencode-linux-arm64-musl": "1.2.24",
|
||||
"opencode-linux-x64": "1.2.24",
|
||||
"opencode-linux-x64-baseline": "1.2.24",
|
||||
"opencode-linux-x64-baseline-musl": "1.2.24",
|
||||
"opencode-linux-x64-musl": "1.2.24",
|
||||
"opencode-windows-x64": "1.2.24",
|
||||
"opencode-windows-x64-baseline": "1.2.24"
|
||||
}
|
||||
},
|
||||
"node_modules/opencode-darwin-arm64": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-darwin-arm64/-/opencode-darwin-arm64-1.2.24.tgz",
|
||||
"integrity": "sha512-mk5AkNbmATLwjnczd09Fu9dLTPcW4GSlRlIGWJBbLuGSPe7DWfbuhA5tj00VuaGeC09wDKVnW06BxWp/gRzgMQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-darwin-x64": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-darwin-x64/-/opencode-darwin-x64-1.2.24.tgz",
|
||||
"integrity": "sha512-SuG7ItUejx42z9Bz46t0R3LLFIM34vZUcwJAkqqpS6MRd1KV9G4OQIJl9BR+Cwwvk70iqCnoDxj5NgFZjeeNOw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-darwin-x64-baseline": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-darwin-x64-baseline/-/opencode-darwin-x64-baseline-1.2.24.tgz",
|
||||
"integrity": "sha512-+MT/ZkgXxeID/kOcT04SBn8Vg+4tk8CytKldaNDLgUnsi/78vTxDXWs0Cc21Ohc894r45fEtOOb5npSNIMOVfQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-linux-arm64": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-linux-arm64/-/opencode-linux-arm64-1.2.24.tgz",
|
||||
"integrity": "sha512-zSpbVCwVfUQ/Bey1MEGkJg3A52EiIPgnpS44HwmSANkiLPVXRBVuwbFYN2qJ7eJVAH939K9+QPmDOW4X8DWKeQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-linux-arm64-musl": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-linux-arm64-musl/-/opencode-linux-arm64-musl-1.2.24.tgz",
|
||||
"integrity": "sha512-LY28mspJOJ2s32E0AbrfbdWH5Bw+5YQkuB3ACJiVxW1MafjhRgPOsXjnw38WMiFjZ+NbhSPBOKnL0guKseVd4Q==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-linux-x64": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-linux-x64/-/opencode-linux-x64-1.2.24.tgz",
|
||||
"integrity": "sha512-OM3umRJ2ZpDe4YtDOGp32cQCqvMPuFJBKKngfvWL4zz4EWDzLPttb0sNL7xdC1xR/XidtADRYcvtzc2rr7/4Rw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-linux-x64-baseline": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-linux-x64-baseline/-/opencode-linux-x64-baseline-1.2.24.tgz",
|
||||
"integrity": "sha512-yakykM+GxCMZ5WK2G0CQk5cGJkoKt09A/8Y0bhKy7l+Jc/QiTV2ZIlBwT5FO15dnuGlNH6zvukLyOQvOZinM+Q==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-linux-x64-baseline-musl": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-linux-x64-baseline-musl/-/opencode-linux-x64-baseline-musl-1.2.24.tgz",
|
||||
"integrity": "sha512-XHPuf9Rj9nWkVX7JJZ5NtYx9UWT0hPiem/kAhZzYITseRGmQMMfFAhIOBYYvvlzJzgdhcrLi198kJHaKCdXjag==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-linux-x64-musl": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-linux-x64-musl/-/opencode-linux-x64-musl-1.2.24.tgz",
|
||||
"integrity": "sha512-nFT4J4m4wKiZh4RsfgQJlW6W2w/VtMNnXnMiQk93FnncSf0nUXEB5Nam3P+vCjdR3i4cs355JiOvbshnMiJGSA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-windows-x64": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-windows-x64/-/opencode-windows-x64-1.2.24.tgz",
|
||||
"integrity": "sha512-7nkwFgS8Jn35kgw+6JIWRU6PmbggSuTDiQ+RnFGoySFgYW/Sz/R30Z4KmnbfJLElzEef0kX+nZQasfVl0pSf6A==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/opencode-windows-x64-baseline": {
|
||||
"version": "1.2.24",
|
||||
"resolved": "https://registry.npmjs.org/opencode-windows-x64-baseline/-/opencode-windows-x64-baseline-1.2.24.tgz",
|
||||
"integrity": "sha512-hXp1nPnkrHgM6cid1AogRJvjz931UbZ/G165B6AAR1gOb5EqtIh2gZcT0IRUyjooEpwH7+oGFRhT+3Kql/M0WA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/optionator": {
|
||||
"version": "0.9.4",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
|
||||
@@ -24688,6 +24482,50 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/shell-env": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/shell-env/-/shell-env-4.0.3.tgz",
|
||||
"integrity": "sha512-Ioe5h+hCDZ7pKL5+JGzbtPvZ5ESMHePZ8nLxohlDL+twmlcmutttMhRkrQOed8DeLT8mkYBgbwZfohe8pqaA3g==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"default-shell": "^2.0.0",
|
||||
"execa": "^5.1.1",
|
||||
"strip-ansi": "^7.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/shell-env/node_modules/ansi-regex": {
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz",
|
||||
"integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/ansi-regex?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/shell-env/node_modules/strip-ansi": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz",
|
||||
"integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ansi-regex": "^6.2.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/strip-ansi?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/shell-quote": {
|
||||
"version": "1.8.3",
|
||||
"resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz",
|
||||
@@ -25427,6 +25265,15 @@
|
||||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/strip-final-newline": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
|
||||
"integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/strip-indent": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-4.1.1.tgz",
|
||||
@@ -27847,7 +27694,7 @@
|
||||
},
|
||||
"packages/app": {
|
||||
"name": "@getpaseo/app",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"dependencies": {
|
||||
"@boudra/expo-two-way-audio": "^0.1.3",
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
@@ -27855,7 +27702,7 @@
|
||||
"@dnd-kit/utilities": "^3.2.2",
|
||||
"@expo/vector-icons": "^15.0.2",
|
||||
"@floating-ui/react-native": "^0.10.7",
|
||||
"@getpaseo/server": "0.1.26",
|
||||
"@getpaseo/server": "0.1.27",
|
||||
"@gorhom/bottom-sheet": "^5.2.6",
|
||||
"@gorhom/portal": "^1.0.14",
|
||||
"@lezer/common": "^1.5.0",
|
||||
@@ -27994,11 +27841,11 @@
|
||||
},
|
||||
"packages/cli": {
|
||||
"name": "@getpaseo/cli",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^1.0.0",
|
||||
"@getpaseo/relay": "0.1.26",
|
||||
"@getpaseo/server": "0.1.26",
|
||||
"@getpaseo/relay": "0.1.27",
|
||||
"@getpaseo/server": "0.1.27",
|
||||
"chalk": "^5.3.0",
|
||||
"commander": "^12.0.0",
|
||||
"mime-types": "^2.1.35",
|
||||
@@ -28035,14 +27882,14 @@
|
||||
},
|
||||
"packages/desktop": {
|
||||
"name": "@getpaseo/desktop",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"devDependencies": {
|
||||
"@tauri-apps/cli": "^2.9.6"
|
||||
}
|
||||
},
|
||||
"packages/relay": {
|
||||
"name": "@getpaseo/relay",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"dependencies": {
|
||||
"base64-js": "^1.5.1",
|
||||
"tweetnacl": "^1.0.3",
|
||||
@@ -28058,12 +27905,12 @@
|
||||
},
|
||||
"packages/server": {
|
||||
"name": "@getpaseo/server",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "2.0.52",
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.2.11",
|
||||
"@deepgram/sdk": "^3.4.0",
|
||||
"@getpaseo/relay": "0.1.26",
|
||||
"@getpaseo/relay": "0.1.27",
|
||||
"@lezer/common": "^1.5.0",
|
||||
"@lezer/css": "^1.3.0",
|
||||
"@lezer/highlight": "^1.2.3",
|
||||
@@ -28073,7 +27920,6 @@
|
||||
"@lezer/markdown": "^1.6.2",
|
||||
"@lezer/python": "^1.1.18",
|
||||
"@modelcontextprotocol/sdk": "^1.20.1",
|
||||
"@openai/codex": "^0.114.0",
|
||||
"@opencode-ai/sdk": "1.2.6",
|
||||
"@sctg/sentencepiece-js": "^1.1.0",
|
||||
"@xterm/headless": "^6.0.0",
|
||||
@@ -28088,13 +27934,13 @@
|
||||
"node-pty": "1.2.0-beta.11",
|
||||
"onnxruntime-node": "^1.23.0",
|
||||
"openai": "^4.20.0",
|
||||
"opencode-ai": "^1.2.24",
|
||||
"pino": "^10.2.0",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"qrcode": "^1.5.4",
|
||||
"rotating-file-stream": "^3.2.9",
|
||||
"sherpa-onnx": "^1.12.23",
|
||||
"sherpa-onnx-node": "^1.12.23",
|
||||
"shell-env": "^4.0.3",
|
||||
"sherpa-onnx": "1.12.28",
|
||||
"sherpa-onnx-node": "1.12.28",
|
||||
"strip-ansi": "^7.1.2",
|
||||
"tiny-invariant": "^1.3.3",
|
||||
"uuid": "^9.0.1",
|
||||
@@ -28421,7 +28267,7 @@
|
||||
},
|
||||
"packages/website": {
|
||||
"name": "@getpaseo/website",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"dependencies": {
|
||||
"@cloudflare/vite-plugin": "^1.20.3",
|
||||
"@cloudflare/workers-types": "^4.20260114.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "paseo",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"private": true,
|
||||
"workspaces": [
|
||||
"packages/server",
|
||||
|
||||
BIN
packages/app/assets/audio/thinking-tone.wav
Normal file
BIN
packages/app/assets/audio/thinking-tone.wav
Normal file
Binary file not shown.
@@ -1,6 +1,4 @@
|
||||
const { getDefaultConfig } = require("expo/metro-config");
|
||||
const exclusionList =
|
||||
require("@expo/metro/metro-config/defaults/exclusionList").default;
|
||||
const { resolve } = require("metro-resolver");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
@@ -18,8 +16,10 @@ const customWebPlatform = (process.env.PASEO_WEB_PLATFORM ?? "")
|
||||
const config = getDefaultConfig(projectRoot);
|
||||
const defaultResolveRequest = config.resolver.resolveRequest ?? resolve;
|
||||
const escapedAppSrcRoot = appSrcRoot
|
||||
.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&")
|
||||
.replace(/\//g, "[\\\\/]");
|
||||
.split(path.sep)
|
||||
.map((segment) => segment.replace(/[|\\{}()[\]^$+*?.]/g, "\\$&"))
|
||||
.join("[\\\\/]");
|
||||
const pathSeparatorPattern = "[\\\\/]";
|
||||
|
||||
config.resolver.extraNodeModules = {
|
||||
...(config.resolver.extraNodeModules ?? {}),
|
||||
@@ -28,9 +28,9 @@ config.resolver.extraNodeModules = {
|
||||
"react/jsx-runtime": path.join(appNodeModulesRoot, "react/jsx-runtime"),
|
||||
"react/jsx-dev-runtime": path.join(appNodeModulesRoot, "react/jsx-dev-runtime"),
|
||||
};
|
||||
config.resolver.blockList = exclusionList([
|
||||
new RegExp(`^${escapedAppSrcRoot}[\\\\/].*\\.(test|spec)\\.(ts|tsx)$`),
|
||||
]);
|
||||
config.resolver.blockList = new RegExp(
|
||||
`(^${escapedAppSrcRoot}${pathSeparatorPattern}.*\\.(test|spec)\\.(ts|tsx)$|${pathSeparatorPattern}__tests__${pathSeparatorPattern}.*)$`,
|
||||
);
|
||||
|
||||
function isLocalModuleImport(moduleName) {
|
||||
return (
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@getpaseo/app",
|
||||
"main": "index.ts",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"start": "expo start",
|
||||
@@ -33,7 +33,7 @@
|
||||
"@dnd-kit/utilities": "^3.2.2",
|
||||
"@expo/vector-icons": "^15.0.2",
|
||||
"@floating-ui/react-native": "^0.10.7",
|
||||
"@getpaseo/server": "0.1.26",
|
||||
"@getpaseo/server": "0.1.27",
|
||||
"@gorhom/bottom-sheet": "^5.2.6",
|
||||
"@gorhom/portal": "^1.0.14",
|
||||
"@lezer/common": "^1.5.0",
|
||||
|
||||
@@ -50,6 +50,7 @@ import { getIsTauri } from "@/constants/layout";
|
||||
import { CommandCenter } from "@/components/command-center";
|
||||
import { ProjectPickerModal } from "@/components/project-picker-modal";
|
||||
import { KeyboardShortcutsDialog } from "@/components/keyboard-shortcuts-dialog";
|
||||
import { listenToDesktopNotificationClicks } from "@/desktop/notifications/desktop-notifications";
|
||||
import { useKeyboardShortcuts } from "@/hooks/use-keyboard-shortcuts";
|
||||
import { queryClient } from "@/query/query-client";
|
||||
import {
|
||||
@@ -70,18 +71,6 @@ polyfillCrypto();
|
||||
attachConsole();
|
||||
const HostRuntimeBootstrapContext = createContext(false);
|
||||
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
|
||||
|
||||
function logLeftSidebarOpenGesture(
|
||||
event: string,
|
||||
details: Record<string, unknown>
|
||||
): void {
|
||||
if (!IS_DEV) {
|
||||
return;
|
||||
}
|
||||
console.log(`[LeftSidebarOpenGesture] ${event}`, details);
|
||||
}
|
||||
|
||||
function PushNotificationRouter() {
|
||||
const router = useRouter();
|
||||
const lastHandledIdRef = useRef<string | null>(null);
|
||||
@@ -89,20 +78,39 @@ function PushNotificationRouter() {
|
||||
useEffect(() => {
|
||||
if (Platform.OS === "web") {
|
||||
if (getTauri()) {
|
||||
void ensureOsNotificationPermission().then((granted) => {
|
||||
console.log(
|
||||
"[OSNotifications][Tauri] Startup permission preflight result:",
|
||||
granted ? "granted" : "not-granted"
|
||||
);
|
||||
});
|
||||
void ensureOsNotificationPermission();
|
||||
|
||||
let disposed = false;
|
||||
let unlisten: (() => void) | null = null;
|
||||
|
||||
void listenToDesktopNotificationClicks((payload) => {
|
||||
router.push(buildNotificationRoute(payload.data) as any);
|
||||
})
|
||||
.then((cleanup) => {
|
||||
if (disposed) {
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
unlisten = cleanup;
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error(
|
||||
"[OSNotifications][Desktop] Failed to register notification click listener",
|
||||
error
|
||||
);
|
||||
});
|
||||
|
||||
return () => {
|
||||
disposed = true;
|
||||
unlisten?.();
|
||||
};
|
||||
}
|
||||
|
||||
const target = globalThis as unknown as EventTarget;
|
||||
const openFromWebClick = (event: Event) => {
|
||||
const customEvent = event as CustomEvent<WebNotificationClickDetail>;
|
||||
const route = buildNotificationRoute(customEvent.detail?.data);
|
||||
event.preventDefault();
|
||||
router.push(route as any);
|
||||
router.push(buildNotificationRoute(customEvent.detail?.data) as any);
|
||||
};
|
||||
|
||||
target.addEventListener(
|
||||
@@ -319,10 +327,6 @@ function AppContainer({
|
||||
})
|
||||
.onStart(() => {
|
||||
isGesturing.value = true;
|
||||
runOnJS(logLeftSidebarOpenGesture)("start", {
|
||||
mobileView,
|
||||
openGestureEnabled,
|
||||
});
|
||||
})
|
||||
.onUpdate((event) => {
|
||||
// Start from closed position (-windowWidth) and move towards 0
|
||||
@@ -339,13 +343,6 @@ function AppContainer({
|
||||
isGesturing.value = false;
|
||||
// Open if dragged more than 1/3 of sidebar or fast swipe
|
||||
const shouldOpen = event.translationX > windowWidth / 3 || event.velocityX > 500;
|
||||
runOnJS(logLeftSidebarOpenGesture)("end", {
|
||||
translationX: event.translationX,
|
||||
velocityX: event.velocityX,
|
||||
shouldOpen,
|
||||
mobileView,
|
||||
openGestureEnabled,
|
||||
});
|
||||
if (shouldOpen) {
|
||||
animateToOpen();
|
||||
runOnJS(openAgentList)();
|
||||
@@ -422,6 +419,7 @@ function ProvidersWrapper({ children }: { children: ReactNode }) {
|
||||
return (
|
||||
<VoiceProvider>
|
||||
<OfferLinkListener upsertDaemonFromOfferUrl={upsertConnectionFromOfferUrl} />
|
||||
<HostSessionManager />
|
||||
{children}
|
||||
</VoiceProvider>
|
||||
);
|
||||
@@ -562,7 +560,6 @@ export default function RootLayout() {
|
||||
<QueryProvider>
|
||||
<HostRuntimeBootstrapProvider>
|
||||
<PushNotificationRouter />
|
||||
<HostSessionManager />
|
||||
<ProvidersWrapper>
|
||||
<SidebarAnimationProvider>
|
||||
<HorizontalScrollProvider>
|
||||
|
||||
@@ -31,6 +31,7 @@ import { useAgentAutocomplete } from '@/hooks/use-agent-autocomplete'
|
||||
import { useHostRuntimeSession } from '@/runtime/host-runtime'
|
||||
import {
|
||||
deleteAttachments,
|
||||
persistAttachmentFromBlob,
|
||||
persistAttachmentFromFileUri,
|
||||
} from '@/attachments/service'
|
||||
import { resolveStatusControlMode } from '@/components/agent-input-area.status-controls'
|
||||
@@ -390,16 +391,24 @@ export function AgentInputArea({
|
||||
|
||||
async function handlePickImage() {
|
||||
const result = await pickImages()
|
||||
if (!result?.assets?.length) {
|
||||
if (!result?.length) {
|
||||
return
|
||||
}
|
||||
|
||||
const newImages = await Promise.all(
|
||||
result.assets.map(async (asset) => {
|
||||
result.map(async (pickedImage) => {
|
||||
if (pickedImage.source.kind === 'blob') {
|
||||
return await persistAttachmentFromBlob({
|
||||
blob: pickedImage.source.blob,
|
||||
mimeType: pickedImage.mimeType || 'image/jpeg',
|
||||
fileName: pickedImage.fileName ?? null,
|
||||
})
|
||||
}
|
||||
|
||||
return await persistAttachmentFromFileUri({
|
||||
uri: asset.uri,
|
||||
mimeType: asset.mimeType || 'image/jpeg',
|
||||
fileName: asset.fileName ?? null,
|
||||
uri: pickedImage.source.uri,
|
||||
mimeType: pickedImage.mimeType || 'image/jpeg',
|
||||
fileName: pickedImage.fileName ?? null,
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
@@ -86,6 +86,7 @@ export interface AgentStreamViewProps {
|
||||
pendingPermissions: Map<string, PendingPermission>;
|
||||
routeBottomAnchorRequest?: BottomAnchorRouteRequest | null;
|
||||
isAuthoritativeHistoryReady?: boolean;
|
||||
onOpenWorkspaceFile?: (input: { filePath: string }) => void;
|
||||
}
|
||||
|
||||
export const AgentStreamView = forwardRef<AgentStreamViewHandle, AgentStreamViewProps>(function AgentStreamView({
|
||||
@@ -96,6 +97,7 @@ export const AgentStreamView = forwardRef<AgentStreamViewHandle, AgentStreamView
|
||||
pendingPermissions,
|
||||
routeBottomAnchorRequest = null,
|
||||
isAuthoritativeHistoryReady = true,
|
||||
onOpenWorkspaceFile,
|
||||
}, ref) {
|
||||
const viewportRef = useRef<StreamViewportHandle | null>(null);
|
||||
const { theme } = useUnistyles();
|
||||
@@ -159,6 +161,11 @@ export const AgentStreamView = forwardRef<AgentStreamViewHandle, AgentStreamView
|
||||
}
|
||||
|
||||
if (normalized.file) {
|
||||
if (onOpenWorkspaceFile) {
|
||||
onOpenWorkspaceFile({ filePath: normalized.file });
|
||||
return;
|
||||
}
|
||||
|
||||
const route = buildHostWorkspaceFileRoute(
|
||||
resolvedServerId,
|
||||
workspaceId,
|
||||
@@ -188,6 +195,7 @@ export const AgentStreamView = forwardRef<AgentStreamViewHandle, AgentStreamView
|
||||
resolvedServerId,
|
||||
router,
|
||||
setExplorerTabForCheckout,
|
||||
onOpenWorkspaceFile,
|
||||
workspaceId,
|
||||
]
|
||||
);
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
Modal,
|
||||
} from "react-native";
|
||||
import { SafeAreaView } from "react-native-safe-area-context";
|
||||
import { useEffect } from "react";
|
||||
import { StyleSheet } from "react-native-unistyles";
|
||||
import { Fonts } from "@/constants/theme";
|
||||
|
||||
@@ -154,16 +153,6 @@ const styles = StyleSheet.create((theme) => ({
|
||||
}));
|
||||
|
||||
export function ArtifactDrawer({ artifact, onClose }: ArtifactDrawerProps) {
|
||||
useEffect(() => {
|
||||
if (!artifact) return;
|
||||
console.log(
|
||||
"[ArtifactDrawer] Showing artifact:",
|
||||
artifact.id,
|
||||
artifact.type,
|
||||
artifact.title
|
||||
);
|
||||
}, [artifact]);
|
||||
|
||||
if (!artifact) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -23,13 +23,7 @@ import { FileExplorerPane } from "./file-explorer-pane";
|
||||
import { useKeyboardShiftStyle } from "@/hooks/use-keyboard-shift-style";
|
||||
|
||||
const MIN_CHAT_WIDTH = 400;
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
|
||||
|
||||
function logExplorerSidebar(event: string, details: Record<string, unknown>): void {
|
||||
if (!IS_DEV) {
|
||||
return;
|
||||
}
|
||||
console.log(`[ExplorerSidebar] ${event}`, details);
|
||||
function logExplorerSidebar(_event: string, _details: Record<string, unknown>): void {
|
||||
}
|
||||
|
||||
interface ExplorerSidebarProps {
|
||||
|
||||
@@ -838,7 +838,7 @@ function getErrorRecoveryPath(state: AgentFileExplorerState | undefined): string
|
||||
const styles = StyleSheet.create((theme) => ({
|
||||
container: {
|
||||
flex: 1,
|
||||
backgroundColor: theme.colors.surface0,
|
||||
backgroundColor: theme.colors.surfaceSidebar,
|
||||
},
|
||||
desktopSplit: {
|
||||
flex: 1,
|
||||
@@ -879,7 +879,7 @@ const styles = StyleSheet.create((theme) => ({
|
||||
paddingHorizontal: theme.spacing[3],
|
||||
borderBottomWidth: 1,
|
||||
borderBottomColor: theme.colors.border,
|
||||
backgroundColor: theme.colors.surface0,
|
||||
backgroundColor: theme.colors.surfaceSidebar,
|
||||
},
|
||||
paneHeaderLeft: {
|
||||
flex: 1,
|
||||
|
||||
@@ -15,6 +15,7 @@ import { router, usePathname } from 'expo-router'
|
||||
import { usePanelStore } from '@/stores/panel-store'
|
||||
import { SidebarWorkspaceList } from './sidebar-workspace-list'
|
||||
import { SidebarAgentListSkeleton } from './sidebar-agent-list-skeleton'
|
||||
import { useSidebarShortcutModel } from '@/hooks/use-sidebar-shortcut-model'
|
||||
import { useSidebarWorkspacesList } from '@/hooks/use-sidebar-workspaces-list'
|
||||
import { useSidebarAnimation } from '@/contexts/sidebar-animation-context'
|
||||
import { useTauriDragHandlers, useTrafficLightPadding } from '@/utils/tauri-window'
|
||||
@@ -31,14 +32,6 @@ import {
|
||||
import { useKeyboardShortcutsStore } from '@/stores/keyboard-shortcuts-store'
|
||||
|
||||
const DESKTOP_SIDEBAR_WIDTH = 320
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__)
|
||||
|
||||
function logLeftSidebarCloseGesture(event: string, details: Record<string, unknown>): void {
|
||||
if (!IS_DEV) {
|
||||
return
|
||||
}
|
||||
console.log(`[LeftSidebarCloseGesture] ${event}`, details)
|
||||
}
|
||||
|
||||
interface LeftSidebarProps {
|
||||
selectedAgentId?: string
|
||||
@@ -124,6 +117,8 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
|
||||
serverId: activeServerId,
|
||||
enabled: isOpen,
|
||||
})
|
||||
const { collapsedProjectKeys, shortcutIndexByWorkspaceKey, toggleProjectCollapsed } =
|
||||
useSidebarShortcutModel(projects)
|
||||
const {
|
||||
translateX,
|
||||
backdropOpacity,
|
||||
@@ -253,7 +248,6 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
|
||||
})
|
||||
.onStart(() => {
|
||||
isGesturing.value = true
|
||||
runOnJS(logLeftSidebarCloseGesture)('start', { isOpen, isMobile })
|
||||
})
|
||||
.onUpdate((event) => {
|
||||
if (!isMobile) return
|
||||
@@ -271,11 +265,6 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
|
||||
isGesturing.value = false
|
||||
if (!isMobile) return
|
||||
const shouldClose = event.translationX < -windowWidth / 3 || event.velocityX < -500
|
||||
runOnJS(logLeftSidebarCloseGesture)('end', {
|
||||
translationX: event.translationX,
|
||||
velocityX: event.velocityX,
|
||||
shouldClose,
|
||||
})
|
||||
if (shouldClose) {
|
||||
animateToClose()
|
||||
runOnJS(handleClose)()
|
||||
@@ -351,8 +340,10 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
|
||||
<SidebarAgentListSkeleton />
|
||||
) : (
|
||||
<SidebarWorkspaceList
|
||||
isOpen={isOpen}
|
||||
serverId={activeServerId}
|
||||
collapsedProjectKeys={collapsedProjectKeys}
|
||||
onToggleProjectCollapsed={toggleProjectCollapsed}
|
||||
shortcutIndexByWorkspaceKey={shortcutIndexByWorkspaceKey}
|
||||
projects={projects}
|
||||
isRefreshing={isManualRefresh && isRevalidating}
|
||||
onRefresh={handleRefresh}
|
||||
@@ -475,8 +466,10 @@ export function LeftSidebar({ selectedAgentId: _selectedAgentId }: LeftSidebarPr
|
||||
<SidebarAgentListSkeleton />
|
||||
) : (
|
||||
<SidebarWorkspaceList
|
||||
isOpen={isOpen}
|
||||
serverId={activeServerId}
|
||||
collapsedProjectKeys={collapsedProjectKeys}
|
||||
onToggleProjectCollapsed={toggleProjectCollapsed}
|
||||
shortcutIndexByWorkspaceKey={shortcutIndexByWorkspaceKey}
|
||||
projects={projects}
|
||||
isRefreshing={isManualRefresh && isRevalidating}
|
||||
onRefresh={handleRefresh}
|
||||
|
||||
@@ -101,7 +101,6 @@ export interface MessageInputRef {
|
||||
const MIN_INPUT_HEIGHT = 30
|
||||
const MAX_INPUT_HEIGHT = 160
|
||||
const IS_WEB = Platform.OS === 'web'
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__)
|
||||
|
||||
type WebTextInputKeyPressEvent = NativeSyntheticEvent<
|
||||
TextInputKeyPressEventData & {
|
||||
@@ -125,13 +124,10 @@ type TextAreaHandle = {
|
||||
}
|
||||
|
||||
function logWebStickyBottom(
|
||||
event: string,
|
||||
details: Record<string, unknown>
|
||||
_event: string,
|
||||
_details: Record<string, unknown>
|
||||
): void {
|
||||
if (!IS_DEV || !IS_WEB) {
|
||||
return
|
||||
}
|
||||
console.log('[WebStickyBottom]', event, details)
|
||||
// Intentionally disabled: this path is too noisy during voice debugging.
|
||||
}
|
||||
|
||||
function getDebugNow(): number | null {
|
||||
@@ -407,6 +403,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
|
||||
const showRealtimeOverlay = isRealtimeVoiceForCurrentAgent
|
||||
const showOverlay = showDictationOverlay || showRealtimeOverlay
|
||||
|
||||
|
||||
useEffect(() => {
|
||||
if (isDictating || isDictationProcessing) {
|
||||
return
|
||||
@@ -1070,10 +1067,7 @@ export const MessageInput = forwardRef<MessageInputRef, MessageInputProps>(funct
|
||||
/>
|
||||
) : showRealtimeOverlay && voice ? (
|
||||
<RealtimeVoiceOverlay
|
||||
volume={voice.volume}
|
||||
isMuted={voice.isMuted}
|
||||
isDetecting={voice.isDetecting}
|
||||
isSpeaking={voice.isSpeaking}
|
||||
isSwitching={voice.isVoiceSwitching}
|
||||
onToggleMute={voice.toggleMute}
|
||||
onStop={() => {
|
||||
|
||||
@@ -18,6 +18,9 @@ import {
|
||||
useCallback,
|
||||
createContext,
|
||||
useContext,
|
||||
isValidElement,
|
||||
Children,
|
||||
cloneElement,
|
||||
} from "react";
|
||||
import type { ReactNode, ComponentType } from "react";
|
||||
import Markdown, { MarkdownIt } from "react-native-markdown-display";
|
||||
@@ -68,7 +71,11 @@ import {
|
||||
buildToolCallDisplayModel,
|
||||
} from "@/utils/tool-call-display";
|
||||
import { resolveToolCallIcon } from "@/utils/tool-call-icon";
|
||||
import { parseInlinePathToken, type InlinePathTarget } from "@/utils/inline-path";
|
||||
import {
|
||||
parseFileProtocolUrl,
|
||||
parseInlinePathToken,
|
||||
type InlinePathTarget,
|
||||
} from "@/utils/inline-path";
|
||||
import { getMarkdownListMarker } from "@/utils/markdown-list";
|
||||
import { openExternalUrl } from "@/utils/open-external-url";
|
||||
import { markScrollInvestigationEvent } from "@/utils/scroll-jank-investigation";
|
||||
@@ -433,20 +440,6 @@ export const assistantMessageStylesheet = StyleSheet.create((theme) => ({
|
||||
containerSpacing: {
|
||||
marginBottom: theme.spacing[4],
|
||||
},
|
||||
// Used in custom markdownRules for inline code styling
|
||||
markdownCodeInline: {
|
||||
backgroundColor: theme.colors.surface2,
|
||||
color: theme.colors.foreground,
|
||||
paddingHorizontal: theme.spacing[2],
|
||||
paddingVertical: 2,
|
||||
borderRadius: theme.borderRadius.sm,
|
||||
fontFamily: Fonts.mono,
|
||||
fontSize: 13,
|
||||
},
|
||||
markdownCodeInlineLink: {
|
||||
color: theme.colors.primary,
|
||||
textDecorationLine: "underline",
|
||||
},
|
||||
// Used in custom markdownRules for path chip styling
|
||||
pathChip: {
|
||||
backgroundColor: theme.colors.surface2,
|
||||
@@ -463,6 +456,44 @@ export const assistantMessageStylesheet = StyleSheet.create((theme) => ({
|
||||
},
|
||||
}));
|
||||
|
||||
function MarkdownLink({
|
||||
href,
|
||||
style,
|
||||
onPress,
|
||||
children,
|
||||
}: {
|
||||
href: string;
|
||||
style: any;
|
||||
onPress: (url: string) => void;
|
||||
children: ReactNode;
|
||||
}) {
|
||||
const [hovered, setHovered] = useState(false);
|
||||
if (Platform.OS !== "web") {
|
||||
return (
|
||||
<Text
|
||||
accessibilityRole="link"
|
||||
onPress={() => onPress(href)}
|
||||
style={style}
|
||||
>
|
||||
{children}
|
||||
</Text>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Pressable
|
||||
accessibilityRole="link"
|
||||
onPress={() => onPress(href)}
|
||||
onHoverIn={() => setHovered(true)}
|
||||
onHoverOut={() => setHovered(false)}
|
||||
>
|
||||
<Text style={[style, hovered && { textDecorationLine: "underline" }]}>
|
||||
{children}
|
||||
</Text>
|
||||
</Pressable>
|
||||
);
|
||||
}
|
||||
|
||||
function getInlineCodeAutoLinkUrl(
|
||||
markdownParser: ReturnType<typeof MarkdownIt>,
|
||||
content: string
|
||||
@@ -487,6 +518,19 @@ function getInlineCodeAutoLinkUrl(
|
||||
return match.url;
|
||||
}
|
||||
|
||||
function nodeHasParentType(parent: unknown, type: string): boolean {
|
||||
if (Array.isArray(parent)) {
|
||||
return parent.some((entry) => entry?.type === type);
|
||||
}
|
||||
|
||||
return (
|
||||
typeof parent === "object" &&
|
||||
parent !== null &&
|
||||
"type" in parent &&
|
||||
(parent as { type?: string }).type === type
|
||||
);
|
||||
}
|
||||
|
||||
const turnCopyButtonStylesheet = StyleSheet.create((theme) => ({
|
||||
container: {
|
||||
alignSelf: "flex-start",
|
||||
@@ -709,16 +753,35 @@ export const AssistantMessage = memo(function AssistantMessage({
|
||||
const markdownStyles = useMemo(() => createMarkdownStyles(theme), [theme]);
|
||||
|
||||
const markdownParser = useMemo(
|
||||
() => MarkdownIt({ typographer: true, linkify: true }),
|
||||
() => {
|
||||
const parser = MarkdownIt({ typographer: true, linkify: true });
|
||||
const defaultValidateLink = parser.validateLink.bind(parser);
|
||||
parser.validateLink = (url: string) => {
|
||||
if (url.trim().toLowerCase().startsWith("file://")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return defaultValidateLink(url);
|
||||
};
|
||||
return parser;
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
const handleLinkPress = useCallback((url: string) => {
|
||||
const fileTarget = onInlinePathPress
|
||||
? parseFileProtocolUrl(url)
|
||||
: null;
|
||||
if (fileTarget) {
|
||||
onInlinePathPress?.(fileTarget);
|
||||
return false;
|
||||
}
|
||||
|
||||
void openExternalUrl(url);
|
||||
// react-native-markdown-display opens the link itself when this returns true.
|
||||
// We already handled it above, so return false to avoid duplicate opens.
|
||||
return false;
|
||||
}, []);
|
||||
}, [onInlinePathPress]);
|
||||
|
||||
const markdownRules = useMemo(() => {
|
||||
return {
|
||||
@@ -775,12 +838,16 @@ export const AssistantMessage = memo(function AssistantMessage({
|
||||
code_inline: (
|
||||
node: any,
|
||||
_children: ReactNode[],
|
||||
_parent: any,
|
||||
_styles: any,
|
||||
parent: any,
|
||||
styles: any,
|
||||
inheritedStyles: any = {}
|
||||
) => {
|
||||
const content = node.content ?? "";
|
||||
const parsed = onInlinePathPress
|
||||
const isLinkedInlineCode =
|
||||
nodeHasParentType(parent, "link") ||
|
||||
(!Array.isArray(parent) &&
|
||||
typeof parent?.attributes?.href === "string");
|
||||
const parsed = onInlinePathPress && !isLinkedInlineCode
|
||||
? parseInlinePathToken(content)
|
||||
: null;
|
||||
|
||||
@@ -803,30 +870,21 @@ export const AssistantMessage = memo(function AssistantMessage({
|
||||
const inlineCodeLinkUrl = getInlineCodeAutoLinkUrl(markdownParser, content);
|
||||
if (inlineCodeLinkUrl) {
|
||||
return (
|
||||
<Text
|
||||
<MarkdownLink
|
||||
key={node.key}
|
||||
accessibilityRole="link"
|
||||
onPress={() => {
|
||||
handleLinkPress(inlineCodeLinkUrl);
|
||||
}}
|
||||
style={[
|
||||
inheritedStyles,
|
||||
assistantMessageStylesheet.markdownCodeInline,
|
||||
assistantMessageStylesheet.markdownCodeInlineLink,
|
||||
]}
|
||||
href={inlineCodeLinkUrl}
|
||||
style={[inheritedStyles, styles.code_inline, styles.link]}
|
||||
onPress={handleLinkPress}
|
||||
>
|
||||
{content}
|
||||
</Text>
|
||||
</MarkdownLink>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Text
|
||||
key={node.key}
|
||||
style={[
|
||||
inheritedStyles,
|
||||
assistantMessageStylesheet.markdownCodeInline,
|
||||
]}
|
||||
style={[inheritedStyles, styles.code_inline]}
|
||||
>
|
||||
{content}
|
||||
</Text>
|
||||
@@ -877,6 +935,25 @@ export const AssistantMessage = memo(function AssistantMessage({
|
||||
</View>
|
||||
);
|
||||
},
|
||||
link: (
|
||||
node: any,
|
||||
children: ReactNode[],
|
||||
_parent: any,
|
||||
styles: any,
|
||||
) => (
|
||||
<MarkdownLink
|
||||
key={node.key}
|
||||
href={node.attributes?.href ?? ""}
|
||||
style={styles.link}
|
||||
onPress={handleLinkPress}
|
||||
>
|
||||
{Children.map(children, (child) =>
|
||||
isValidElement(child)
|
||||
? cloneElement(child, { style: [(child.props as any).style, { color: styles.link.color }] } as any)
|
||||
: child
|
||||
)}
|
||||
</MarkdownLink>
|
||||
),
|
||||
};
|
||||
}, [handleLinkPress, markdownParser, onInlinePathPress]);
|
||||
|
||||
@@ -1849,9 +1926,11 @@ export const ToolCall = memo(function ToolCall({
|
||||
const hasDetails =
|
||||
Boolean(error) ||
|
||||
(effectiveDetail
|
||||
? effectiveDetail.type !== "unknown" ||
|
||||
effectiveDetail.input !== null ||
|
||||
effectiveDetail.output !== null
|
||||
? effectiveDetail.type === "plain_text"
|
||||
? Boolean(effectiveDetail.text)
|
||||
: effectiveDetail.type !== "unknown" ||
|
||||
effectiveDetail.input !== null ||
|
||||
effectiveDetail.output !== null
|
||||
: false);
|
||||
|
||||
const handleToggle = useCallback(() => {
|
||||
|
||||
@@ -2,13 +2,11 @@ import { ActivityIndicator, Pressable, View } from "react-native";
|
||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||
import { Mic, MicOff, Square } from "lucide-react-native";
|
||||
import { FOOTER_HEIGHT } from "@/constants/layout";
|
||||
import { useVoiceTelemetry } from "@/contexts/voice-context";
|
||||
import { VolumeMeter } from "./volume-meter";
|
||||
|
||||
interface RealtimeVoiceOverlayProps {
|
||||
volume: number;
|
||||
isMuted: boolean;
|
||||
isDetecting: boolean;
|
||||
isSpeaking: boolean;
|
||||
isSwitching: boolean;
|
||||
onToggleMute: () => void;
|
||||
onStop: () => void;
|
||||
@@ -18,16 +16,13 @@ const OVERLAY_BUTTON_SIZE = 44;
|
||||
const OVERLAY_VERTICAL_PADDING = (FOOTER_HEIGHT - OVERLAY_BUTTON_SIZE) / 2;
|
||||
|
||||
export function RealtimeVoiceOverlay({
|
||||
volume,
|
||||
isMuted,
|
||||
isDetecting,
|
||||
isSpeaking,
|
||||
isSwitching,
|
||||
onToggleMute,
|
||||
onStop,
|
||||
}: RealtimeVoiceOverlayProps) {
|
||||
const { theme } = useUnistyles();
|
||||
|
||||
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
|
||||
return (
|
||||
<View style={styles.container}>
|
||||
<View style={styles.meterContainer}>
|
||||
|
||||
@@ -59,7 +59,6 @@ import {
|
||||
import { SyncedLoader } from '@/components/synced-loader'
|
||||
import { useToast } from '@/contexts/toast-context'
|
||||
import { useCheckoutGitActionsStore } from '@/stores/checkout-git-actions-store'
|
||||
import { buildSidebarShortcutModel } from '@/utils/sidebar-shortcuts'
|
||||
import { hasVisibleOrderChanged, mergeWithRemainder } from '@/utils/sidebar-reorder'
|
||||
import { decideLongPressMove } from '@/utils/sidebar-gesture-arbitration'
|
||||
import { confirmDialog } from '@/utils/confirm-dialog'
|
||||
@@ -79,10 +78,17 @@ function toProjectIconDataUri(icon: { mimeType: string; data: string } | null):
|
||||
return `data:${icon.mimeType};base64,${icon.data}`
|
||||
}
|
||||
|
||||
const workspaceKeyExtractor = (workspace: SidebarWorkspaceEntry) =>
|
||||
workspace.workspaceKey
|
||||
|
||||
const projectKeyExtractor = (project: SidebarProjectEntry) => project.projectKey
|
||||
|
||||
interface SidebarWorkspaceListProps {
|
||||
isOpen?: boolean
|
||||
projects: SidebarProjectEntry[]
|
||||
serverId: string | null
|
||||
collapsedProjectKeys: ReadonlySet<string>
|
||||
onToggleProjectCollapsed: (projectKey: string) => void
|
||||
shortcutIndexByWorkspaceKey: Map<string, number>
|
||||
isRefreshing?: boolean
|
||||
onRefresh?: () => void
|
||||
onWorkspacePress?: () => void
|
||||
@@ -122,6 +128,7 @@ interface WorkspaceRowInnerProps {
|
||||
archiveStatus?: 'idle' | 'pending' | 'success'
|
||||
archivePendingLabel?: string
|
||||
onArchive?: () => void
|
||||
onCopyBranchName?: () => void
|
||||
onCopyPath?: () => void
|
||||
}
|
||||
|
||||
@@ -250,7 +257,6 @@ function NewWorktreeButton({
|
||||
function useLongPressDragInteraction(input: {
|
||||
drag: () => void
|
||||
menuController: ReturnType<typeof useContextMenu> | null
|
||||
debugId: string
|
||||
}) {
|
||||
const didLongPressRef = useRef(false)
|
||||
const dragArmedRef = useRef(false)
|
||||
@@ -288,15 +294,11 @@ function useLongPressDragInteraction(input: {
|
||||
input.menuController.setOpen(true)
|
||||
menuOpenedRef.current = true
|
||||
didLongPressRef.current = true
|
||||
console.log('[sidebar-dnd-debug] context menu opened', { id: input.debugId })
|
||||
}, [input.debugId, input.menuController])
|
||||
}, [input.menuController])
|
||||
|
||||
const handleLongPress = useCallback(() => {
|
||||
// Manual timers own long-press behavior on mobile.
|
||||
console.log('[sidebar-dnd-debug] native onLongPress ignored (manual state machine active)', {
|
||||
id: input.debugId,
|
||||
})
|
||||
}, [input.debugId])
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
@@ -325,16 +327,11 @@ function useLongPressDragInteraction(input: {
|
||||
const dy = current.y - start.y
|
||||
const distance = Math.sqrt(dx * dx + dy * dy)
|
||||
if (distance > DRAG_ARM_STATIONARY_SLOP_PX) {
|
||||
console.log('[sidebar-dnd-debug] drag arm cancelled (movement)', {
|
||||
id: input.debugId,
|
||||
distance,
|
||||
})
|
||||
return
|
||||
}
|
||||
dragArmedRef.current = true
|
||||
dragActivatedRef.current = true
|
||||
didLongPressRef.current = true
|
||||
console.log('[sidebar-dnd-debug] drag armed', { id: input.debugId })
|
||||
void Haptics.selectionAsync().catch(() => {})
|
||||
input.drag()
|
||||
}, DRAG_ARM_DELAY_MS)
|
||||
@@ -356,17 +353,12 @@ function useLongPressDragInteraction(input: {
|
||||
const dy = current.y - start.y
|
||||
const distance = Math.sqrt(dx * dx + dy * dy)
|
||||
if (distance > CONTEXT_MENU_STATIONARY_SLOP_PX) {
|
||||
console.log('[sidebar-dnd-debug] context menu cancelled (movement)', {
|
||||
id: input.debugId,
|
||||
distance,
|
||||
})
|
||||
return
|
||||
}
|
||||
console.log('[sidebar-dnd-debug] long-press armed', { id: input.debugId })
|
||||
void Haptics.selectionAsync().catch(() => {})
|
||||
openContextMenuAtStartPoint()
|
||||
}, CONTEXT_MENU_DELAY_MS)
|
||||
}, [clearTimers, input.debugId, input.menuController, openContextMenuAtStartPoint])
|
||||
}, [clearTimers, input.menuController, openContextMenuAtStartPoint])
|
||||
|
||||
const handleDragIntent = useCallback(
|
||||
(details: { dx: number; dy: number; distance: number }) => {
|
||||
@@ -376,10 +368,9 @@ function useLongPressDragInteraction(input: {
|
||||
didStartDragRef.current = true
|
||||
didLongPressRef.current = true
|
||||
clearTimers()
|
||||
console.log('[sidebar-dnd-debug] drag movement detected', { id: input.debugId, ...details })
|
||||
void Haptics.impactAsync(Haptics.ImpactFeedbackStyle.Medium).catch(() => {})
|
||||
},
|
||||
[clearTimers, input.debugId]
|
||||
[clearTimers]
|
||||
)
|
||||
|
||||
const handleScrollIntent = useCallback(
|
||||
@@ -387,18 +378,16 @@ function useLongPressDragInteraction(input: {
|
||||
scrollIntentRef.current = true
|
||||
didLongPressRef.current = true
|
||||
clearTimers()
|
||||
console.log('[sidebar-dnd-debug] scroll intent detected', { id: input.debugId, ...details })
|
||||
},
|
||||
[clearTimers, input.debugId]
|
||||
[clearTimers]
|
||||
)
|
||||
|
||||
const handleSwipeIntent = useCallback(
|
||||
(details: { dx: number; dy: number; distance: number }) => {
|
||||
didLongPressRef.current = true
|
||||
clearTimers()
|
||||
console.log('[sidebar-dnd-debug] swipe intent detected', { id: input.debugId, ...details })
|
||||
},
|
||||
[clearTimers, input.debugId]
|
||||
[clearTimers]
|
||||
)
|
||||
|
||||
const handlePressIn = useCallback((event: GestureResponderEvent) => {
|
||||
@@ -416,13 +405,8 @@ function useLongPressDragInteraction(input: {
|
||||
x: event.nativeEvent.pageX,
|
||||
y: event.nativeEvent.pageY,
|
||||
}
|
||||
console.log('[sidebar-dnd-debug] press-in', {
|
||||
id: input.debugId,
|
||||
x: event.nativeEvent.pageX,
|
||||
y: event.nativeEvent.pageY,
|
||||
})
|
||||
armTimers()
|
||||
}, [armTimers, input.debugId])
|
||||
}, [armTimers])
|
||||
|
||||
const handleTouchMove = useCallback(
|
||||
(event: any) => {
|
||||
@@ -469,20 +453,11 @@ function useLongPressDragInteraction(input: {
|
||||
|
||||
const handlePressOut = useCallback(() => {
|
||||
clearTimers()
|
||||
console.log('[sidebar-dnd-debug] press-out no context-menu', {
|
||||
id: input.debugId,
|
||||
didLongPress: didLongPressRef.current,
|
||||
didStartDrag: didStartDragRef.current,
|
||||
dragActivated: dragActivatedRef.current,
|
||||
scrollIntent: scrollIntentRef.current,
|
||||
dragArmed: dragArmedRef.current,
|
||||
menuOpened: menuOpenedRef.current,
|
||||
})
|
||||
dragArmedRef.current = false
|
||||
dragActivatedRef.current = false
|
||||
touchStartRef.current = null
|
||||
touchCurrentRef.current = null
|
||||
}, [clearTimers, input.debugId])
|
||||
}, [clearTimers])
|
||||
|
||||
return {
|
||||
didLongPressRef,
|
||||
@@ -511,7 +486,6 @@ function ProjectHeaderRow({
|
||||
const interaction = useLongPressDragInteraction({
|
||||
drag,
|
||||
menuController,
|
||||
debugId: `project:${project.projectKey}`,
|
||||
})
|
||||
|
||||
const handlePress = useCallback(() => {
|
||||
@@ -610,6 +584,7 @@ function WorkspaceRowInner({
|
||||
archiveStatus = 'idle',
|
||||
archivePendingLabel,
|
||||
onArchive,
|
||||
onCopyBranchName,
|
||||
onCopyPath,
|
||||
}: WorkspaceRowInnerProps) {
|
||||
const { theme } = useUnistyles()
|
||||
@@ -618,7 +593,6 @@ function WorkspaceRowInner({
|
||||
const interaction = useLongPressDragInteraction({
|
||||
drag,
|
||||
menuController,
|
||||
debugId: `workspace:${workspace.workspaceKey}`,
|
||||
})
|
||||
|
||||
const handlePress = useCallback(() => {
|
||||
@@ -694,6 +668,15 @@ function WorkspaceRowInner({
|
||||
Copy path
|
||||
</DropdownMenuItem>
|
||||
) : null}
|
||||
{onCopyBranchName ? (
|
||||
<DropdownMenuItem
|
||||
testID={`sidebar-workspace-menu-copy-branch-name-${workspace.workspaceKey}`}
|
||||
leading={<Copy size={14} color={theme.colors.foregroundMuted} />}
|
||||
onSelect={onCopyBranchName}
|
||||
>
|
||||
Copy branch name
|
||||
</DropdownMenuItem>
|
||||
) : null}
|
||||
<DropdownMenuItem
|
||||
testID={`sidebar-workspace-menu-archive-${workspace.workspaceKey}`}
|
||||
leading={<Archive size={14} color={theme.colors.foregroundMuted} />}
|
||||
@@ -731,6 +714,7 @@ function WorkspaceRowWithMenu({
|
||||
drag,
|
||||
isDragging,
|
||||
dragHandleProps,
|
||||
canCopyBranchName,
|
||||
}: {
|
||||
workspace: SidebarWorkspaceEntry
|
||||
selected: boolean
|
||||
@@ -740,6 +724,7 @@ function WorkspaceRowWithMenu({
|
||||
drag: () => void
|
||||
isDragging: boolean
|
||||
dragHandleProps?: DraggableListDragHandleProps
|
||||
canCopyBranchName: boolean
|
||||
}) {
|
||||
const toast = useToast()
|
||||
const archiveWorktree = useCheckoutGitActionsStore((state) => state.archiveWorktree)
|
||||
@@ -825,6 +810,11 @@ function WorkspaceRowWithMenu({
|
||||
toast.copied('Path copied')
|
||||
}, [toast, workspace.workspaceId])
|
||||
|
||||
const handleCopyBranchName = useCallback(() => {
|
||||
void Clipboard.setStringAsync(workspace.name)
|
||||
toast.copied('Branch name copied')
|
||||
}, [toast, workspace.name])
|
||||
|
||||
return (
|
||||
<WorkspaceRowInner
|
||||
workspace={workspace}
|
||||
@@ -841,6 +831,7 @@ function WorkspaceRowWithMenu({
|
||||
archiveStatus={isWorktree ? archiveStatus : isArchivingWorkspace ? 'pending' : 'idle'}
|
||||
archivePendingLabel={isWorktree ? 'Archiving...' : 'Hiding...'}
|
||||
onArchive={isWorktree ? handleArchiveWorktree : handleArchiveWorkspace}
|
||||
onCopyBranchName={canCopyBranchName ? handleCopyBranchName : undefined}
|
||||
onCopyPath={handleCopyPath}
|
||||
/>
|
||||
)
|
||||
@@ -1006,6 +997,7 @@ function WorkspaceRow({
|
||||
drag,
|
||||
isDragging,
|
||||
dragHandleProps,
|
||||
canCopyBranchName,
|
||||
}: {
|
||||
workspace: SidebarWorkspaceEntry
|
||||
selected: boolean
|
||||
@@ -1015,6 +1007,7 @@ function WorkspaceRow({
|
||||
drag: () => void
|
||||
isDragging: boolean
|
||||
dragHandleProps?: DraggableListDragHandleProps
|
||||
canCopyBranchName: boolean
|
||||
}) {
|
||||
return (
|
||||
<WorkspaceRowWithMenu
|
||||
@@ -1026,6 +1019,7 @@ function WorkspaceRow({
|
||||
drag={drag}
|
||||
isDragging={isDragging}
|
||||
dragHandleProps={dragHandleProps}
|
||||
canCopyBranchName={canCopyBranchName}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -1093,6 +1087,7 @@ function ProjectBlock({
|
||||
selected={isSelected}
|
||||
shortcutNumber={shortcutIndexByWorkspaceKey.get(item.workspaceKey) ?? null}
|
||||
showShortcutBadge={showShortcutBadges}
|
||||
canCopyBranchName={project.projectKind === 'git'}
|
||||
onPress={() => {
|
||||
if (!serverId) {
|
||||
return
|
||||
@@ -1131,6 +1126,13 @@ function ProjectBlock({
|
||||
[renderWorkspaceRow]
|
||||
)
|
||||
|
||||
const handleWorkspaceDragEnd = useCallback(
|
||||
(workspaces: SidebarWorkspaceEntry[]) => {
|
||||
onWorkspaceReorder(project.projectKey, workspaces)
|
||||
},
|
||||
[onWorkspaceReorder, project.projectKey]
|
||||
)
|
||||
|
||||
return (
|
||||
<View style={styles.projectBlock}>
|
||||
{flattenedWorkspace ? (
|
||||
@@ -1176,9 +1178,9 @@ function ProjectBlock({
|
||||
<DraggableList
|
||||
testID={`sidebar-workspace-list-${project.projectKey}`}
|
||||
data={project.workspaces}
|
||||
keyExtractor={(workspace) => workspace.workspaceKey}
|
||||
keyExtractor={workspaceKeyExtractor}
|
||||
renderItem={renderWorkspace}
|
||||
onDragEnd={(workspaces) => onWorkspaceReorder(project.projectKey, workspaces)}
|
||||
onDragEnd={handleWorkspaceDragEnd}
|
||||
scrollEnabled={false}
|
||||
useDragHandle
|
||||
nestable={useNestable}
|
||||
@@ -1193,9 +1195,11 @@ function ProjectBlock({
|
||||
}
|
||||
|
||||
export function SidebarWorkspaceList({
|
||||
isOpen = true,
|
||||
projects,
|
||||
serverId,
|
||||
collapsedProjectKeys,
|
||||
onToggleProjectCollapsed,
|
||||
shortcutIndexByWorkspaceKey,
|
||||
isRefreshing = false,
|
||||
onRefresh,
|
||||
onWorkspacePress,
|
||||
@@ -1205,16 +1209,9 @@ export function SidebarWorkspaceList({
|
||||
const isMobile = UnistylesRuntime.breakpoint === 'xs' || UnistylesRuntime.breakpoint === 'sm'
|
||||
const isNative = Platform.OS !== 'web'
|
||||
const pathname = usePathname()
|
||||
const [collapsedProjectKeys, setCollapsedProjectKeys] = useState<Set<string>>(new Set())
|
||||
const isTauri = getIsTauri()
|
||||
const altDown = useKeyboardShortcutsStore((state) => state.altDown)
|
||||
const cmdOrCtrlDown = useKeyboardShortcutsStore((state) => state.cmdOrCtrlDown)
|
||||
const setSidebarShortcutWorkspaceTargets = useKeyboardShortcutsStore(
|
||||
(state) => state.setSidebarShortcutWorkspaceTargets
|
||||
)
|
||||
const setVisibleWorkspaceTargets = useKeyboardShortcutsStore(
|
||||
(state) => state.setVisibleWorkspaceTargets
|
||||
)
|
||||
const showShortcutBadges = altDown || (isTauri && cmdOrCtrlDown)
|
||||
|
||||
const getProjectOrder = useSidebarOrderStore((state) => state.getProjectOrder)
|
||||
@@ -1236,21 +1233,8 @@ export function SidebarWorkspaceList({
|
||||
}
|
||||
}, [pathname])
|
||||
|
||||
useEffect(() => {
|
||||
setCollapsedProjectKeys((prev) => {
|
||||
const validProjectKeys = new Set(projects.map((project) => project.projectKey))
|
||||
const next = new Set<string>()
|
||||
for (const key of prev) {
|
||||
if (validProjectKeys.has(key)) {
|
||||
next.add(key)
|
||||
}
|
||||
}
|
||||
return next
|
||||
})
|
||||
}, [projects])
|
||||
|
||||
const projectIconRequests = useMemo(() => {
|
||||
if (!isOpen || !serverId) {
|
||||
if (!serverId) {
|
||||
return []
|
||||
}
|
||||
const unique = new Map<string, { serverId: string; cwd: string }>()
|
||||
@@ -1262,7 +1246,7 @@ export function SidebarWorkspaceList({
|
||||
unique.set(`${serverId}:${cwd}`, { serverId, cwd })
|
||||
}
|
||||
return Array.from(unique.values())
|
||||
}, [isOpen, projects, serverId])
|
||||
}, [projects, serverId])
|
||||
|
||||
const projectIconQueries = useQueries({
|
||||
queries: projectIconRequests.map((request) => ({
|
||||
@@ -1277,7 +1261,6 @@ export function SidebarWorkspaceList({
|
||||
},
|
||||
select: toProjectIconDataUri,
|
||||
enabled: Boolean(
|
||||
isOpen &&
|
||||
getHostRuntimeStore().getClient(request.serverId) &&
|
||||
isHostRuntimeConnected(getHostRuntimeStore().getSnapshot(request.serverId)) &&
|
||||
request.cwd
|
||||
@@ -1316,44 +1299,6 @@ export function SidebarWorkspaceList({
|
||||
return byProject
|
||||
}, [projectIconQueries, projectIconRequests, projects, serverId])
|
||||
|
||||
const shortcutModel = useMemo(
|
||||
() =>
|
||||
buildSidebarShortcutModel({
|
||||
projects,
|
||||
collapsedProjectKeys,
|
||||
}),
|
||||
[collapsedProjectKeys, projects]
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
setSidebarShortcutWorkspaceTargets(shortcutModel.shortcutTargets)
|
||||
setVisibleWorkspaceTargets(shortcutModel.visibleTargets)
|
||||
}, [
|
||||
setSidebarShortcutWorkspaceTargets,
|
||||
setVisibleWorkspaceTargets,
|
||||
shortcutModel.shortcutTargets,
|
||||
shortcutModel.visibleTargets,
|
||||
])
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
setSidebarShortcutWorkspaceTargets([])
|
||||
setVisibleWorkspaceTargets([])
|
||||
}
|
||||
}, [setSidebarShortcutWorkspaceTargets, setVisibleWorkspaceTargets])
|
||||
|
||||
const toggleProjectCollapsed = useCallback((projectKey: string) => {
|
||||
setCollapsedProjectKeys((prev) => {
|
||||
const next = new Set(prev)
|
||||
if (next.has(projectKey)) {
|
||||
next.delete(projectKey)
|
||||
} else {
|
||||
next.add(projectKey)
|
||||
}
|
||||
return next
|
||||
})
|
||||
}, [])
|
||||
|
||||
const handleProjectDragEnd = useCallback(
|
||||
(reorderedProjects: SidebarProjectEntry[]) => {
|
||||
if (!serverId) {
|
||||
@@ -1437,9 +1382,9 @@ export function SidebarWorkspaceList({
|
||||
serverId={serverId}
|
||||
activeWorkspaceSelection={activeWorkspaceSelection}
|
||||
showShortcutBadges={showShortcutBadges}
|
||||
shortcutIndexByWorkspaceKey={shortcutModel.shortcutIndexByWorkspaceKey}
|
||||
shortcutIndexByWorkspaceKey={shortcutIndexByWorkspaceKey}
|
||||
parentGestureRef={parentGestureRef}
|
||||
onToggleCollapsed={() => toggleProjectCollapsed(item.projectKey)}
|
||||
onToggleCollapsed={() => onToggleProjectCollapsed(item.projectKey)}
|
||||
onWorkspacePress={onWorkspacePress}
|
||||
onWorkspaceReorder={handleWorkspaceReorder}
|
||||
onCreateWorktree={handleCreateWorktree}
|
||||
@@ -1456,12 +1401,12 @@ export function SidebarWorkspaceList({
|
||||
handleCreateWorktree,
|
||||
handleWorkspaceReorder,
|
||||
onWorkspacePress,
|
||||
onToggleProjectCollapsed,
|
||||
parentGestureRef,
|
||||
projectIconByProjectKey,
|
||||
serverId,
|
||||
shortcutModel.shortcutIndexByWorkspaceKey,
|
||||
shortcutIndexByWorkspaceKey,
|
||||
showShortcutBadges,
|
||||
toggleProjectCollapsed,
|
||||
isNative,
|
||||
]
|
||||
)
|
||||
@@ -1474,7 +1419,7 @@ export function SidebarWorkspaceList({
|
||||
<DraggableList
|
||||
testID="sidebar-project-list"
|
||||
data={projects}
|
||||
keyExtractor={(project) => project.projectKey}
|
||||
keyExtractor={projectKeyExtractor}
|
||||
renderItem={renderProject}
|
||||
onDragEnd={handleProjectDragEnd}
|
||||
scrollEnabled={false}
|
||||
@@ -1495,9 +1440,7 @@ export function SidebarWorkspaceList({
|
||||
style={styles.list}
|
||||
contentContainerStyle={styles.listContent}
|
||||
showsVerticalScrollIndicator={false}
|
||||
onScrollBeginDrag={() =>
|
||||
console.log('[sidebar-dnd-debug] outer scroll begin')
|
||||
}
|
||||
|
||||
testID="sidebar-project-workspace-list-scroll"
|
||||
>
|
||||
{content}
|
||||
@@ -1507,9 +1450,7 @@ export function SidebarWorkspaceList({
|
||||
style={styles.list}
|
||||
contentContainerStyle={styles.listContent}
|
||||
showsVerticalScrollIndicator={false}
|
||||
onScrollBeginDrag={() =>
|
||||
console.log('[sidebar-dnd-debug] outer scroll begin')
|
||||
}
|
||||
|
||||
testID="sidebar-project-workspace-list-scroll"
|
||||
>
|
||||
{content}
|
||||
|
||||
@@ -24,7 +24,6 @@ const USER_SCROLL_DELTA_EPSILON = 1
|
||||
const AUTO_SCROLL_BOTTOM_THRESHOLD_PX = 64
|
||||
const AUTO_SCROLL_RESUME_THRESHOLD_PX = 1
|
||||
const WEB_STREAM_SCROLLBAR_STYLE_ID = 'web-stream-viewport-scrollbar-style'
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__)
|
||||
const WEB_STREAM_SCROLLBAR_STYLE = `
|
||||
#agent-chat-scroll-web-dom-scroll,
|
||||
#agent-chat-scroll-web-dom-virtualized {
|
||||
@@ -40,11 +39,8 @@ const WEB_STREAM_SCROLLBAR_STYLE = `
|
||||
}
|
||||
`
|
||||
|
||||
function logWebStickyBottom(event: string, details: Record<string, unknown>): void {
|
||||
if (!IS_DEV) {
|
||||
return
|
||||
}
|
||||
console.log('[WebStickyBottom]', event, details)
|
||||
function logWebStickyBottom(_event: string, _details: Record<string, unknown>): void {
|
||||
// Intentionally disabled: this path is too noisy during voice debugging.
|
||||
}
|
||||
|
||||
function getDebugNow(): number | null {
|
||||
|
||||
@@ -64,6 +64,7 @@ export interface ComboboxProps {
|
||||
title?: string
|
||||
open?: boolean
|
||||
onOpenChange?: (open: boolean) => void
|
||||
enableDismissOnClose?: boolean
|
||||
desktopPlacement?: 'top-start' | 'bottom-start'
|
||||
/**
|
||||
* Prevents an initial frame at 0,0 by hiding desktop content until floating
|
||||
@@ -224,6 +225,7 @@ export function Combobox({
|
||||
title = 'Select',
|
||||
open,
|
||||
onOpenChange,
|
||||
enableDismissOnClose,
|
||||
desktopPlacement = 'top-start',
|
||||
desktopPreventInitialFlash = true,
|
||||
anchorRef,
|
||||
@@ -235,6 +237,7 @@ export function Combobox({
|
||||
!isMobile && Platform.OS === 'web' && effectiveOptionsPosition === 'above-search'
|
||||
const { height: windowHeight } = useWindowDimensions()
|
||||
const bottomSheetRef = useRef<BottomSheetModal>(null)
|
||||
const hasPresentedBottomSheetRef = useRef(false)
|
||||
const snapPoints = useMemo(() => ['60%', '90%'], [])
|
||||
const [availableSize, setAvailableSize] = useState<{ width?: number; height?: number } | null>(
|
||||
null
|
||||
@@ -379,11 +382,20 @@ export function Combobox({
|
||||
useEffect(() => {
|
||||
if (!isMobile) return
|
||||
if (isOpen) {
|
||||
bottomSheetRef.current?.present()
|
||||
if (enableDismissOnClose === false && hasPresentedBottomSheetRef.current) {
|
||||
bottomSheetRef.current?.snapToIndex(0)
|
||||
} else {
|
||||
hasPresentedBottomSheetRef.current = true
|
||||
bottomSheetRef.current?.present()
|
||||
}
|
||||
} else {
|
||||
bottomSheetRef.current?.dismiss()
|
||||
if (enableDismissOnClose === false) {
|
||||
bottomSheetRef.current?.close()
|
||||
} else {
|
||||
bottomSheetRef.current?.dismiss()
|
||||
}
|
||||
}
|
||||
}, [isOpen, isMobile])
|
||||
}, [enableDismissOnClose, isOpen, isMobile])
|
||||
|
||||
const handleSheetChange = useCallback(
|
||||
(index: number) => {
|
||||
@@ -622,6 +634,7 @@ export function Combobox({
|
||||
onChange={handleSheetChange}
|
||||
backdropComponent={renderBackdrop}
|
||||
enablePanDownToClose
|
||||
enableDismissOnClose={enableDismissOnClose}
|
||||
backgroundComponent={ComboboxSheetBackground}
|
||||
handleIndicatorStyle={styles.bottomSheetHandle}
|
||||
keyboardBehavior="extend"
|
||||
|
||||
@@ -9,6 +9,8 @@ import {
|
||||
useState,
|
||||
type PropsWithChildren,
|
||||
type ReactElement,
|
||||
type Ref,
|
||||
type MutableRefObject,
|
||||
} from "react";
|
||||
import {
|
||||
ActivityIndicator,
|
||||
@@ -182,6 +184,16 @@ function coerceEventPoint(event: unknown): { pageX: number; pageY: number } | nu
|
||||
return null;
|
||||
}
|
||||
|
||||
function assignRef<T>(ref: Ref<T> | undefined, value: T): void {
|
||||
if (typeof ref === "function") {
|
||||
ref(value);
|
||||
return;
|
||||
}
|
||||
if (ref && typeof ref === "object") {
|
||||
(ref as MutableRefObject<T>).current = value;
|
||||
}
|
||||
}
|
||||
|
||||
export function ContextMenu({
|
||||
open,
|
||||
defaultOpen,
|
||||
@@ -231,6 +243,7 @@ export function ContextMenuTrigger({
|
||||
enabledOnMobile = false,
|
||||
enabledOnWeb = true,
|
||||
longPressDelayMs,
|
||||
triggerRef,
|
||||
...props
|
||||
}: PropsWithChildren<
|
||||
Omit<PressableProps, "style"> & {
|
||||
@@ -239,6 +252,7 @@ export function ContextMenuTrigger({
|
||||
enabledOnMobile?: boolean;
|
||||
enabledOnWeb?: boolean;
|
||||
longPressDelayMs?: number;
|
||||
triggerRef?: Ref<View | null>;
|
||||
}
|
||||
>): ReactElement {
|
||||
const ctx = useContextMenuContext("ContextMenuTrigger");
|
||||
@@ -268,10 +282,18 @@ export function ContextMenuTrigger({
|
||||
[ctx, disabled, shouldEnableOnThisPlatform]
|
||||
);
|
||||
|
||||
const handleRef = useCallback(
|
||||
(node: View | null) => {
|
||||
assignRef(ctx.triggerRef, node);
|
||||
assignRef(triggerRef, node);
|
||||
},
|
||||
[ctx.triggerRef, triggerRef]
|
||||
);
|
||||
|
||||
return (
|
||||
<Pressable
|
||||
{...props}
|
||||
ref={ctx.triggerRef}
|
||||
ref={handleRef}
|
||||
collapsable={false}
|
||||
disabled={disabled}
|
||||
delayLongPress={longPressDelayMs}
|
||||
|
||||
@@ -170,10 +170,6 @@ export function useDropdownFloating({
|
||||
}, []);
|
||||
|
||||
const update = useCallback(async () => {
|
||||
console.log("[useDropdownFloating] update called", {
|
||||
hasReferenceEl: !!referenceEl,
|
||||
hasFloatingEl: !!floatingElRef.current,
|
||||
});
|
||||
|
||||
if (!referenceEl || !floatingElRef.current) {
|
||||
return;
|
||||
@@ -185,12 +181,6 @@ export function useDropdownFloating({
|
||||
measureElement(floatingElRef.current),
|
||||
]);
|
||||
|
||||
console.log("[useDropdownFloating] measured", {
|
||||
fromRect,
|
||||
contentRect,
|
||||
displayArea,
|
||||
});
|
||||
|
||||
const result = computeGeometry({
|
||||
fromRect,
|
||||
contentSize: { width: contentRect.width, height: contentRect.height },
|
||||
@@ -201,8 +191,6 @@ export function useDropdownFloating({
|
||||
padding,
|
||||
});
|
||||
|
||||
console.log("[useDropdownFloating] geometry result", result);
|
||||
|
||||
setGeometry(result);
|
||||
} catch (e) {
|
||||
console.warn("[useDropdownFloating] measure failed:", e);
|
||||
@@ -210,7 +198,6 @@ export function useDropdownFloating({
|
||||
}, [referenceEl, displayArea, placement, alignment, offset, padding]);
|
||||
|
||||
const floatingRef = useCallback((el: View | null) => {
|
||||
console.log("[useDropdownFloating] floatingRef called", { hasEl: !!el });
|
||||
floatingElRef.current = el;
|
||||
}, []);
|
||||
|
||||
@@ -218,17 +205,10 @@ export function useDropdownFloating({
|
||||
const [floatingReady, setFloatingReady] = useState(false);
|
||||
|
||||
const handleFloatingLayout = useCallback(() => {
|
||||
console.log("[useDropdownFloating] handleFloatingLayout called");
|
||||
setFloatingReady(true);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
console.log("[useDropdownFloating] useEffect", {
|
||||
open,
|
||||
floatingReady,
|
||||
hasReferenceEl: !!referenceEl,
|
||||
hasFloatingEl: !!floatingElRef.current,
|
||||
});
|
||||
|
||||
if (!open) {
|
||||
setGeometry(null);
|
||||
@@ -237,7 +217,6 @@ export function useDropdownFloating({
|
||||
}
|
||||
|
||||
if (floatingReady && referenceEl && floatingElRef.current) {
|
||||
console.log("[useDropdownFloating] calling update from useEffect");
|
||||
update();
|
||||
}
|
||||
}, [open, floatingReady, referenceEl, update]);
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import {
|
||||
Children,
|
||||
cloneElement,
|
||||
createContext,
|
||||
isValidElement,
|
||||
useCallback,
|
||||
useContext,
|
||||
useEffect,
|
||||
@@ -8,6 +11,7 @@ import {
|
||||
useState,
|
||||
type PropsWithChildren,
|
||||
type ReactElement,
|
||||
type Ref,
|
||||
} from "react";
|
||||
import {
|
||||
Dimensions,
|
||||
@@ -53,6 +57,26 @@ function useTooltipContext(componentName: string): TooltipContextValue {
|
||||
return ctx;
|
||||
}
|
||||
|
||||
function composeEventHandlers<E>(
|
||||
original?: (event: E) => void,
|
||||
injected?: (event: E) => void
|
||||
): (event: E) => void {
|
||||
return (event: E) => {
|
||||
original?.(event);
|
||||
injected?.(event);
|
||||
};
|
||||
}
|
||||
|
||||
function assignRef<T>(ref: Ref<T> | undefined, value: T): void {
|
||||
if (typeof ref === "function") {
|
||||
ref(value);
|
||||
return;
|
||||
}
|
||||
if (ref && typeof ref === "object") {
|
||||
(ref as { current: T }).current = value;
|
||||
}
|
||||
}
|
||||
|
||||
function useControllableOpenState({
|
||||
open,
|
||||
defaultOpen,
|
||||
@@ -224,8 +248,15 @@ export function TooltipTrigger({
|
||||
onFocus,
|
||||
onBlur,
|
||||
onPress,
|
||||
asChild = false,
|
||||
triggerRefProp = "ref",
|
||||
...props
|
||||
}: PropsWithChildren<PressableProps>): ReactElement {
|
||||
}: PropsWithChildren<
|
||||
PressableProps & {
|
||||
asChild?: boolean;
|
||||
triggerRefProp?: string;
|
||||
}
|
||||
>): ReactElement {
|
||||
const ctx = useTooltipContext("TooltipTrigger");
|
||||
const openTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
@@ -276,37 +307,86 @@ export function TooltipTrigger({
|
||||
[onHoverOut, close]
|
||||
);
|
||||
|
||||
const handleFocus = useCallback(
|
||||
(e: any) => {
|
||||
onFocus?.(e);
|
||||
if (!ctx.enabled || disabled) return;
|
||||
clearOpenTimer();
|
||||
ctx.setOpen(true);
|
||||
},
|
||||
[clearOpenTimer, ctx, disabled, onFocus]
|
||||
);
|
||||
|
||||
const handleBlur = useCallback(
|
||||
(e: any) => {
|
||||
onBlur?.(e);
|
||||
close();
|
||||
},
|
||||
[close, onBlur]
|
||||
);
|
||||
|
||||
const handlePress = useCallback(
|
||||
(e: any) => {
|
||||
onPress?.(e);
|
||||
close();
|
||||
},
|
||||
[close, onPress]
|
||||
);
|
||||
|
||||
const triggerProps = {
|
||||
...props,
|
||||
disabled,
|
||||
onHoverIn: handleHoverIn,
|
||||
onHoverOut: handleHoverOut,
|
||||
onFocus: handleFocus,
|
||||
onBlur: handleBlur,
|
||||
onPress: handlePress,
|
||||
...(Platform.OS === "web"
|
||||
? ({
|
||||
// RN Web's hover handling can vary across environments; pointer events are the most reliable.
|
||||
onPointerEnter: handleHoverIn,
|
||||
onPointerLeave: handleHoverOut,
|
||||
onMouseEnter: handleHoverIn,
|
||||
onMouseLeave: handleHoverOut,
|
||||
} as any)
|
||||
: null),
|
||||
};
|
||||
|
||||
if (asChild) {
|
||||
const child = Children.only(children);
|
||||
if (!isValidElement(child)) {
|
||||
throw new Error("TooltipTrigger with asChild expects a single React element child");
|
||||
}
|
||||
|
||||
const childProps = child.props as Record<string, any>;
|
||||
const mergedProps = {
|
||||
...childProps,
|
||||
...triggerProps,
|
||||
onHoverIn: composeEventHandlers(childProps.onHoverIn, handleHoverIn),
|
||||
onHoverOut: composeEventHandlers(childProps.onHoverOut, handleHoverOut),
|
||||
onFocus: composeEventHandlers(childProps.onFocus, handleFocus),
|
||||
onBlur: composeEventHandlers(childProps.onBlur, handleBlur),
|
||||
onPress: composeEventHandlers(childProps.onPress, handlePress),
|
||||
onPointerEnter: composeEventHandlers(childProps.onPointerEnter, handleHoverIn),
|
||||
onPointerLeave: composeEventHandlers(childProps.onPointerLeave, handleHoverOut),
|
||||
onMouseEnter: composeEventHandlers(childProps.onMouseEnter, handleHoverIn),
|
||||
onMouseLeave: composeEventHandlers(childProps.onMouseLeave, handleHoverOut),
|
||||
} as Record<string, any>;
|
||||
|
||||
const existingRefProp = childProps[triggerRefProp] as Ref<View | null> | undefined;
|
||||
mergedProps[triggerRefProp] = (node: View | null) => {
|
||||
assignRef(existingRefProp, node);
|
||||
assignRef(ctx.triggerRef, node);
|
||||
};
|
||||
|
||||
return cloneElement(child, mergedProps);
|
||||
}
|
||||
|
||||
return (
|
||||
<Pressable
|
||||
{...props}
|
||||
{...triggerProps}
|
||||
ref={ctx.triggerRef}
|
||||
collapsable={false}
|
||||
disabled={disabled}
|
||||
onHoverIn={handleHoverIn}
|
||||
onHoverOut={handleHoverOut}
|
||||
onFocus={(e) => {
|
||||
onFocus?.(e);
|
||||
if (!ctx.enabled || disabled) return;
|
||||
clearOpenTimer();
|
||||
ctx.setOpen(true);
|
||||
}}
|
||||
onBlur={(e) => {
|
||||
onBlur?.(e);
|
||||
close();
|
||||
}}
|
||||
onPress={(e) => {
|
||||
onPress?.(e);
|
||||
close();
|
||||
}}
|
||||
{...(Platform.OS === "web"
|
||||
? ({
|
||||
// RN Web's hover handling can vary across environments; pointer events are the most reliable.
|
||||
onPointerEnter: handleHoverIn,
|
||||
onPointerLeave: handleHoverOut,
|
||||
onMouseEnter: handleHoverIn,
|
||||
onMouseLeave: handleHoverOut,
|
||||
} as any)
|
||||
: null)}
|
||||
>
|
||||
{children}
|
||||
</Pressable>
|
||||
|
||||
@@ -132,8 +132,6 @@ function createDriverHarness(input?: {
|
||||
context.measurementState.offsetY = 720;
|
||||
});
|
||||
const modeChanges: BottomAnchorMode[] = [];
|
||||
const warnings: Array<{ agentId: string; reason: string }> = [];
|
||||
const logs: Array<{ event: string; details: Record<string, unknown> }> = [];
|
||||
const driver = __private__.createBottomAnchorControllerDriver({
|
||||
getAgentId: () => context.agentId,
|
||||
getIsAuthoritativeHistoryReady: () => context.authoritativeReady,
|
||||
@@ -145,10 +143,6 @@ function createDriverHarness(input?: {
|
||||
onModeChange: (mode) => {
|
||||
modeChanges.push(mode);
|
||||
},
|
||||
log: (event, details) => {
|
||||
logs.push({ event, details });
|
||||
},
|
||||
warn: (details) => warnings.push(details),
|
||||
scheduleFrame: (params) => scheduler.schedule(params),
|
||||
cancelFrame: (handle) => scheduler.cancel(handle),
|
||||
});
|
||||
@@ -159,8 +153,6 @@ function createDriverHarness(input?: {
|
||||
scheduler,
|
||||
scrollToBottom,
|
||||
modeChanges,
|
||||
logs,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -367,7 +359,6 @@ describe("bottom anchor controller driver", () => {
|
||||
harness.scheduler.flushAll();
|
||||
|
||||
expect(harness.scrollToBottom).toHaveBeenCalledTimes(1);
|
||||
expect(harness.warnings).toEqual([]);
|
||||
expect(harness.driver.getSnapshot().pendingRequest).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
@@ -110,8 +110,6 @@ type CreateBottomAnchorControllerDriverInput = {
|
||||
isNearBottom: () => boolean;
|
||||
scrollToBottom: (animated: boolean) => void;
|
||||
onModeChange: (mode: BottomAnchorMode) => void;
|
||||
log: (event: BottomAnchorEvent, details: Record<string, unknown>) => void;
|
||||
warn: (details: { agentId: string; reason: BottomAnchorRequestReason }) => void;
|
||||
scheduleFrame: (params: {
|
||||
kind: "attempt" | "verification";
|
||||
callback: () => void;
|
||||
@@ -123,17 +121,6 @@ type CreateBottomAnchorControllerDriverInput = {
|
||||
const MAX_VERIFICATION_RETRIES = 3;
|
||||
const WEB_PARTIAL_VIRTUALIZED_CONFIRMATION_DELAY_FRAMES = 1;
|
||||
const USER_SCROLL_AWAY_DELTA_PX = 24;
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
|
||||
|
||||
function logBottomAnchorEvent(
|
||||
event: BottomAnchorEvent,
|
||||
details: Record<string, unknown>
|
||||
): void {
|
||||
if (!IS_DEV) {
|
||||
return;
|
||||
}
|
||||
console.debug("[BottomAnchor]", event, details);
|
||||
}
|
||||
|
||||
function scheduleAnimationFrameWithDelay(input: {
|
||||
callback: () => void;
|
||||
@@ -323,10 +310,6 @@ function createBottomAnchorControllerDriver(
|
||||
return;
|
||||
}
|
||||
blockedReason = nextBlockedReason;
|
||||
input.log(
|
||||
"blocked_reason_changed",
|
||||
getLogContext({ nextBlockedReason })
|
||||
);
|
||||
};
|
||||
|
||||
const setModeInternal = (nextMode: BottomAnchorMode) => {
|
||||
@@ -367,13 +350,6 @@ function createBottomAnchorControllerDriver(
|
||||
setBlockedReason(null);
|
||||
return;
|
||||
}
|
||||
input.log(
|
||||
"request_cancelled",
|
||||
getLogContext({
|
||||
cancelledRequestReason: currentRequest.reason,
|
||||
cancelReason: reason,
|
||||
})
|
||||
);
|
||||
pendingRequest = null;
|
||||
cancelPendingAttempt();
|
||||
setBlockedReason(null);
|
||||
@@ -398,19 +374,6 @@ function createBottomAnchorControllerDriver(
|
||||
if (verificationHandle) {
|
||||
input.cancelFrame(verificationHandle);
|
||||
}
|
||||
input.log(
|
||||
"verification_scheduled",
|
||||
getLogContext({
|
||||
retries: attemptContext.retries,
|
||||
startedContentHeight: attemptContext.startedContentHeight ?? null,
|
||||
startedOffsetY: attemptContext.startedOffsetY ?? null,
|
||||
startedViewportHeight: attemptContext.startedViewportHeight ?? null,
|
||||
scheduledMeasurementState:
|
||||
getDetailedMeasurementState(scheduledMeasurementState),
|
||||
verificationDelayFrames:
|
||||
delayFramesOverride ?? input.getTransportBehavior().verificationDelayFrames,
|
||||
})
|
||||
);
|
||||
verificationHandle = input.scheduleFrame({
|
||||
kind: "verification",
|
||||
delayFrames:
|
||||
@@ -427,15 +390,6 @@ function createBottomAnchorControllerDriver(
|
||||
});
|
||||
|
||||
if (verificationBlockedReason) {
|
||||
input.log(
|
||||
"attempt_verified",
|
||||
getLogContext({
|
||||
verificationPhase: "blocked",
|
||||
verificationBlockedReason,
|
||||
retries: attemptContext.retries,
|
||||
measurementState: getDetailedMeasurementState(measurementState),
|
||||
})
|
||||
);
|
||||
pendingVerification = attemptContext;
|
||||
setBlockedReason(verificationBlockedReason);
|
||||
return;
|
||||
@@ -451,26 +405,6 @@ function createBottomAnchorControllerDriver(
|
||||
input.getTransportBehavior().verificationRetryMode,
|
||||
});
|
||||
|
||||
input.log(
|
||||
"attempt_verified",
|
||||
getLogContext({
|
||||
verifiedNearBottom,
|
||||
retries: attemptContext.retries,
|
||||
retryDisposition,
|
||||
contentHeightDeltaSinceAttempt:
|
||||
measurementState.contentHeight -
|
||||
(attemptContext.startedContentHeight ?? measurementState.contentHeight),
|
||||
offsetDeltaSinceAttempt:
|
||||
measurementState.offsetY -
|
||||
(attemptContext.startedOffsetY ?? measurementState.offsetY),
|
||||
viewportHeightDeltaSinceAttempt:
|
||||
measurementState.viewportHeight -
|
||||
(attemptContext.startedViewportHeight ??
|
||||
measurementState.viewportHeight),
|
||||
measurementState: getDetailedMeasurementState(measurementState),
|
||||
})
|
||||
);
|
||||
|
||||
if (verifiedNearBottom) {
|
||||
if (
|
||||
isRequestAttempt &&
|
||||
@@ -494,7 +428,6 @@ function createBottomAnchorControllerDriver(
|
||||
pendingVerification = null;
|
||||
markStickyMeasurementVerified();
|
||||
if (isRequestAttempt) {
|
||||
input.log("request_fulfilled", getLogContext());
|
||||
pendingRequest = null;
|
||||
}
|
||||
setBlockedReason(null);
|
||||
@@ -520,21 +453,7 @@ function createBottomAnchorControllerDriver(
|
||||
return;
|
||||
}
|
||||
|
||||
input.log(
|
||||
"attempt_failed",
|
||||
getLogContext({
|
||||
retries: attemptContext.retries,
|
||||
retryDisposition,
|
||||
measurementState: getDetailedMeasurementState(measurementState),
|
||||
})
|
||||
);
|
||||
pendingVerification = null;
|
||||
if (isRequestAttempt && currentRequest) {
|
||||
input.warn({
|
||||
agentId: input.getAgentId(),
|
||||
reason: currentRequest.reason,
|
||||
});
|
||||
}
|
||||
setBlockedReason(
|
||||
isRequestAttempt ? "waiting_for_post_layout_verification" : null
|
||||
);
|
||||
@@ -552,14 +471,6 @@ function createBottomAnchorControllerDriver(
|
||||
startedViewportHeight: measurementState.viewportHeight,
|
||||
};
|
||||
pendingVerification = attemptContext;
|
||||
input.log(
|
||||
"attempt_started",
|
||||
getLogContext({
|
||||
animated,
|
||||
retries: attemptContext.retries,
|
||||
measurementState: getDetailedMeasurementState(measurementState),
|
||||
})
|
||||
);
|
||||
input.scrollToBottom(animated);
|
||||
scheduleVerification(attemptContext);
|
||||
setBlockedReason(deriveDriverBlockedReason(input.getMeasurementState()));
|
||||
@@ -576,18 +487,6 @@ function createBottomAnchorControllerDriver(
|
||||
| "manual_reevaluate"
|
||||
| "retry_scroll"
|
||||
) => {
|
||||
input.log(
|
||||
"evaluate_called",
|
||||
getLogContext({
|
||||
evaluateReason: reason,
|
||||
animated,
|
||||
hasAttemptHandle: attemptHandle !== null,
|
||||
hasVerificationHandle: verificationHandle !== null,
|
||||
pendingVerificationRequestId: pendingVerification?.requestId ?? null,
|
||||
pendingVerificationRetries: pendingVerification?.retries ?? null,
|
||||
measurementState: getDetailedMeasurementState(input.getMeasurementState()),
|
||||
})
|
||||
);
|
||||
if (attemptHandle) {
|
||||
return;
|
||||
}
|
||||
@@ -610,17 +509,6 @@ function createBottomAnchorControllerDriver(
|
||||
!shouldAttemptForPendingRequest &&
|
||||
!shouldAttemptForStickyVerification
|
||||
) {
|
||||
input.log(
|
||||
"attempt_started",
|
||||
getLogContext({
|
||||
attemptPhase: "skipped",
|
||||
evaluateReason: reason,
|
||||
nextBlockedReason,
|
||||
shouldAttemptForPendingRequest,
|
||||
shouldAttemptForStickyVerification,
|
||||
measurementState: getDetailedMeasurementState(measurementState),
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -630,17 +518,6 @@ function createBottomAnchorControllerDriver(
|
||||
};
|
||||
|
||||
const createRequest = (request: BottomAnchorRouteRequest | BottomAnchorLocalRequest) => {
|
||||
const existing = pendingRequest;
|
||||
if (existing) {
|
||||
input.log(
|
||||
"request_cancelled",
|
||||
getLogContext({
|
||||
cancelledRequestReason: existing.reason,
|
||||
cancelReason: "replaced_by_new_request",
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
cancelPendingAttempt();
|
||||
const nextRequest: BottomAnchorRequest = {
|
||||
id: requestSequence + 1,
|
||||
@@ -659,10 +536,6 @@ function createBottomAnchorControllerDriver(
|
||||
? "sticky-bottom"
|
||||
: __private__.deriveModeForLocalRequest({ reason: request.reason })
|
||||
);
|
||||
input.log(
|
||||
"request_created",
|
||||
getLogContext({ requestReason: request.reason })
|
||||
);
|
||||
evaluate(request.reason === "jump-to-bottom", "request_created");
|
||||
};
|
||||
|
||||
@@ -707,7 +580,6 @@ function createBottomAnchorControllerDriver(
|
||||
}
|
||||
cancelPendingRequest("user_scrolled_away");
|
||||
setModeInternal("detached");
|
||||
input.log("detached_by_user", getLogContext());
|
||||
},
|
||||
handleViewportMetricsChange(params) {
|
||||
if (
|
||||
@@ -905,10 +777,6 @@ export function useBottomAnchorController(input: {
|
||||
isNearBottom: () => isNearBottomRef.current(),
|
||||
scrollToBottom: (animated) => scrollToBottomRef.current(animated),
|
||||
onModeChange: (nextMode) => setMode(nextMode),
|
||||
log: (event, details) => logBottomAnchorEvent(event, details),
|
||||
warn: (details) => {
|
||||
console.warn("[BottomAnchor] request could not be fulfilled", details);
|
||||
},
|
||||
scheduleFrame: ({ callback, delayFrames }) =>
|
||||
scheduleAnimationFrameWithDelay({ callback, delayFrames }),
|
||||
cancelFrame: (handle) => cancelScheduledAnimationFrame(handle),
|
||||
|
||||
@@ -2,21 +2,18 @@ import { ActivityIndicator, Alert, Pressable, View } from "react-native";
|
||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||
import { Mic, MicOff, Square } from "lucide-react-native";
|
||||
import { VolumeMeter } from "@/components/volume-meter";
|
||||
import { useVoice } from "@/contexts/voice-context";
|
||||
import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context";
|
||||
|
||||
export function VoiceCompactIndicator() {
|
||||
const { theme } = useUnistyles();
|
||||
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
|
||||
const {
|
||||
isVoiceMode,
|
||||
isVoiceSwitching,
|
||||
volume,
|
||||
isMuted,
|
||||
isDetecting,
|
||||
isSpeaking,
|
||||
toggleMute,
|
||||
stopVoice,
|
||||
} = useVoice();
|
||||
|
||||
if (!isVoiceMode) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -2,22 +2,19 @@ import { View, Pressable } from "react-native";
|
||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||
import { MicOff, Square } from "lucide-react-native";
|
||||
import { VolumeMeter } from "./volume-meter";
|
||||
import { useVoice } from "@/contexts/voice-context";
|
||||
import { useVoice, useVoiceTelemetry } from "@/contexts/voice-context";
|
||||
import { useHosts } from "@/runtime/host-runtime";
|
||||
|
||||
export function VoicePanel() {
|
||||
const { theme } = useUnistyles();
|
||||
const daemons = useHosts();
|
||||
const { volume, isDetecting, isSpeaking } = useVoiceTelemetry();
|
||||
const {
|
||||
volume,
|
||||
isMuted,
|
||||
isDetecting,
|
||||
isSpeaking,
|
||||
stopVoice,
|
||||
toggleMute,
|
||||
activeServerId,
|
||||
} = useVoice();
|
||||
|
||||
const hostLabel = activeServerId
|
||||
? daemons.find((daemon) => daemon.serverId === activeServerId)?.label ?? null
|
||||
: null;
|
||||
|
||||
@@ -3,10 +3,9 @@ import { View } from "react-native";
|
||||
import ReanimatedAnimated, {
|
||||
useSharedValue,
|
||||
useAnimatedStyle,
|
||||
withSpring,
|
||||
withTiming,
|
||||
withRepeat,
|
||||
withSequence,
|
||||
withTiming,
|
||||
Easing,
|
||||
} from "react-native-reanimated";
|
||||
import { StyleSheet, useUnistyles } from "react-native-unistyles";
|
||||
@@ -41,9 +40,7 @@ export function VolumeMeter({
|
||||
orientation === "horizontal" ? (isCompact ? 8 : 12) : isCompact ? 14 : 20;
|
||||
|
||||
// Create shared values for 3 dots unconditionally
|
||||
const line1Height = useSharedValue(MIN_HEIGHT);
|
||||
const line2Height = useSharedValue(MIN_HEIGHT);
|
||||
const line3Height = useSharedValue(MIN_HEIGHT);
|
||||
const animatedVolume = useSharedValue(0);
|
||||
const line1Pulse = useSharedValue(1);
|
||||
const line2Pulse = useSharedValue(1);
|
||||
const line3Pulse = useSharedValue(1);
|
||||
@@ -90,54 +87,19 @@ export function VolumeMeter({
|
||||
);
|
||||
}, [isMuted]);
|
||||
|
||||
// Update heights based on volume with different responsiveness for all dots
|
||||
// Drive a single animated volume value and derive the individual bar heights
|
||||
// on the UI thread instead of scheduling three independent springs per sample.
|
||||
useEffect(() => {
|
||||
if (isMuted) {
|
||||
// When muted, keep all lines at minimum height without animation
|
||||
line1Height.value = MIN_HEIGHT;
|
||||
line2Height.value = MIN_HEIGHT;
|
||||
line3Height.value = MIN_HEIGHT;
|
||||
animatedVolume.value = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if (volume > 0.001) {
|
||||
// Active volume - animate heights based on volume
|
||||
const target1 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.2);
|
||||
const target2 = MIN_HEIGHT + (MAX_HEIGHT * volume * 1.05);
|
||||
const target3 = MIN_HEIGHT + (MAX_HEIGHT * volume * 0.9);
|
||||
|
||||
line1Height.value = withSpring(target1, {
|
||||
damping: 10,
|
||||
stiffness: 200,
|
||||
});
|
||||
|
||||
line2Height.value = withSpring(target2, {
|
||||
damping: 12.5,
|
||||
stiffness: 175,
|
||||
});
|
||||
|
||||
line3Height.value = withSpring(target3, {
|
||||
damping: 15,
|
||||
stiffness: 150,
|
||||
});
|
||||
} else {
|
||||
// No volume - return to minimum
|
||||
line1Height.value = withSpring(MIN_HEIGHT, {
|
||||
damping: 20,
|
||||
stiffness: 150,
|
||||
});
|
||||
|
||||
line2Height.value = withSpring(MIN_HEIGHT, {
|
||||
damping: 20,
|
||||
stiffness: 150,
|
||||
});
|
||||
|
||||
line3Height.value = withSpring(MIN_HEIGHT, {
|
||||
damping: 20,
|
||||
stiffness: 150,
|
||||
});
|
||||
}
|
||||
}, [volume, isMuted]);
|
||||
animatedVolume.value = withTiming(volume, {
|
||||
duration: volume > animatedVolume.value ? 70 : 140,
|
||||
easing: Easing.out(Easing.cubic),
|
||||
});
|
||||
}, [animatedVolume, isMuted, volume]);
|
||||
|
||||
const lineColor = color ?? theme.colors.foreground;
|
||||
const containerHeight =
|
||||
@@ -147,9 +109,12 @@ export function VolumeMeter({
|
||||
const line1Style = useAnimatedStyle(() => {
|
||||
const isActive = isSpeaking;
|
||||
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
|
||||
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
|
||||
const currentVolume = isMuted ? 0 : animatedVolume.value;
|
||||
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 1.2);
|
||||
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
|
||||
return {
|
||||
height: line1Height.value * (isMuted || volume > 0.001 ? 1 : line1Pulse.value),
|
||||
height:
|
||||
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line1Pulse.value),
|
||||
opacity: baseOpacity + volumeBoost,
|
||||
};
|
||||
});
|
||||
@@ -157,9 +122,12 @@ export function VolumeMeter({
|
||||
const line2Style = useAnimatedStyle(() => {
|
||||
const isActive = isSpeaking;
|
||||
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
|
||||
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
|
||||
const currentVolume = isMuted ? 0 : animatedVolume.value;
|
||||
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 1.05);
|
||||
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
|
||||
return {
|
||||
height: line2Height.value * (isMuted || volume > 0.001 ? 1 : line2Pulse.value),
|
||||
height:
|
||||
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line2Pulse.value),
|
||||
opacity: baseOpacity + volumeBoost,
|
||||
};
|
||||
});
|
||||
@@ -167,9 +135,12 @@ export function VolumeMeter({
|
||||
const line3Style = useAnimatedStyle(() => {
|
||||
const isActive = isSpeaking;
|
||||
const baseOpacity = isMuted ? 0.3 : isActive ? 0.9 : 0.5;
|
||||
const volumeBoost = isMuted || !isActive ? 0 : volume * 0.3;
|
||||
const currentVolume = isMuted ? 0 : animatedVolume.value;
|
||||
const currentHeight = MIN_HEIGHT + (MAX_HEIGHT * currentVolume * 0.9);
|
||||
const volumeBoost = isMuted || !isActive ? 0 : currentVolume * 0.3;
|
||||
return {
|
||||
height: line3Height.value * (isMuted || volume > 0.001 ? 1 : line3Pulse.value),
|
||||
height:
|
||||
currentHeight * (isMuted || currentVolume > 0.001 ? 1 : line3Pulse.value),
|
||||
opacity: baseOpacity + volumeBoost,
|
||||
};
|
||||
});
|
||||
|
||||
@@ -12,18 +12,6 @@ import { usePanelStore } from "@/stores/panel-store";
|
||||
|
||||
const ANIMATION_DURATION = 220;
|
||||
const ANIMATION_EASING = Easing.bezier(0.25, 0.1, 0.25, 1);
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
|
||||
|
||||
function logExplorerAnimation(
|
||||
event: string,
|
||||
details: Record<string, unknown>
|
||||
): void {
|
||||
if (!IS_DEV) {
|
||||
return;
|
||||
}
|
||||
console.log(`[ExplorerAnimation] ${event}`, details);
|
||||
}
|
||||
|
||||
interface ExplorerSidebarAnimationContextValue {
|
||||
translateX: SharedValue<number>;
|
||||
backdropOpacity: SharedValue<number>;
|
||||
@@ -66,23 +54,9 @@ export function ExplorerSidebarAnimationProvider({ children }: { children: React
|
||||
|
||||
// Don't animate if we're in the middle of a gesture - the gesture handler will handle it
|
||||
if (isGesturing.value) {
|
||||
logExplorerAnimation("sync-skipped-during-gesture", {
|
||||
previousIsOpen,
|
||||
nextIsOpen: isOpen,
|
||||
mobileView,
|
||||
desktopFileExplorerOpen,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
logExplorerAnimation("sync-state-change", {
|
||||
previousIsOpen,
|
||||
nextIsOpen: isOpen,
|
||||
mobileView,
|
||||
desktopFileExplorerOpen,
|
||||
windowWidth,
|
||||
});
|
||||
|
||||
if (isOpen) {
|
||||
translateX.value = withTiming(0, {
|
||||
duration: ANIMATION_DURATION,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useRef, ReactNode, useCallback, useEffect, useMemo } from "react";
|
||||
import { Buffer } from "buffer";
|
||||
import { AppState, Platform } from "react-native";
|
||||
import { useQueryClient } from "@tanstack/react-query";
|
||||
import { useAudioPlayer } from "@/hooks/use-audio-player";
|
||||
import { useClientActivity } from "@/hooks/use-client-activity";
|
||||
import { usePushTokenRegistration } from "@/hooks/use-push-token-registration";
|
||||
import { clearArchiveAgentPending } from "@/hooks/use-archive-agent";
|
||||
@@ -29,8 +29,13 @@ import type { DaemonClient } from "@server/client/daemon-client";
|
||||
import { File } from "expo-file-system";
|
||||
import {
|
||||
getHostRuntimeStore,
|
||||
useHostRuntimeSession,
|
||||
useHostRuntimeIsConnected,
|
||||
} from "@/runtime/host-runtime";
|
||||
import {
|
||||
useVoiceAudioEngineOptional,
|
||||
useVoiceRuntimeOptional,
|
||||
} from "@/contexts/voice-context";
|
||||
import type { AudioPlaybackSource } from "@/voice/audio-engine-types";
|
||||
import {
|
||||
useSessionStore,
|
||||
type Agent,
|
||||
@@ -41,6 +46,7 @@ import {
|
||||
import { useDraftStore } from "@/stores/draft-store";
|
||||
import type { AgentDirectoryEntry } from "@/types/agent-directory";
|
||||
import { sendOsNotification } from "@/utils/os-notifications";
|
||||
import { getIsAppActivelyVisible } from "@/utils/app-visibility";
|
||||
import {
|
||||
getInitKey,
|
||||
getInitDeferred,
|
||||
@@ -72,6 +78,55 @@ export type {
|
||||
const HISTORY_STALE_AFTER_MS = 60_000;
|
||||
const AUTHORITATIVE_REVALIDATION_DEBOUNCE_MS = 300;
|
||||
|
||||
type AudioOutputPayload = Extract<
|
||||
SessionOutboundMessage,
|
||||
{ type: "audio_output" }
|
||||
>["payload"];
|
||||
|
||||
interface BufferedAudioChunk {
|
||||
chunkIndex: number;
|
||||
audio: string;
|
||||
format: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
function decodeBase64Chunk(base64: string): Uint8Array {
|
||||
return Buffer.from(base64, "base64");
|
||||
}
|
||||
|
||||
function buildAudioPlaybackSource(
|
||||
chunks: BufferedAudioChunk[]
|
||||
): AudioPlaybackSource {
|
||||
const decodedChunks = chunks.map((chunk) => decodeBase64Chunk(chunk.audio));
|
||||
const totalSize = decodedChunks.reduce((sum, chunk) => sum + chunk.length, 0);
|
||||
const output = new Uint8Array(totalSize);
|
||||
let offset = 0;
|
||||
for (const chunk of decodedChunks) {
|
||||
output.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
const format = chunks[0]?.format ?? "pcm";
|
||||
const mimeType =
|
||||
format === "pcm"
|
||||
? "audio/pcm;rate=24000;bits=16"
|
||||
: format === "mp3"
|
||||
? "audio/mpeg"
|
||||
: `audio/${format}`;
|
||||
|
||||
const bytes = output.slice();
|
||||
return {
|
||||
size: bytes.byteLength,
|
||||
type: mimeType,
|
||||
async arrayBuffer() {
|
||||
return bytes.buffer.slice(
|
||||
bytes.byteOffset,
|
||||
bytes.byteOffset + bytes.byteLength
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const findLatestAssistantMessageText = (items: StreamItem[]): string | null => {
|
||||
for (let i = items.length - 1; i >= 0; i -= 1) {
|
||||
const item = items[i];
|
||||
@@ -205,8 +260,10 @@ function SessionProviderInternal({
|
||||
serverId,
|
||||
client,
|
||||
}: SessionProviderClientProps) {
|
||||
const voiceRuntime = useVoiceRuntimeOptional();
|
||||
const voiceAudioEngine = useVoiceAudioEngineOptional();
|
||||
const queryClient = useQueryClient();
|
||||
const { isConnected } = useHostRuntimeSession(serverId);
|
||||
const isConnected = useHostRuntimeIsConnected(serverId);
|
||||
|
||||
// Zustand store actions
|
||||
const initializeSession = useSessionStore((state) => state.initializeSession);
|
||||
@@ -275,16 +332,6 @@ function SessionProviderInternal({
|
||||
(state) => state.sessions[serverId]?.agents
|
||||
);
|
||||
|
||||
// State for voice detection flags (will be set by RealtimeContext)
|
||||
const isDetectingRef = useRef(false);
|
||||
const isSpeakingRef = useRef(false);
|
||||
|
||||
const audioPlayer = useAudioPlayer({
|
||||
isDetecting: () => isDetectingRef.current,
|
||||
isSpeaking: () => isSpeakingRef.current,
|
||||
});
|
||||
|
||||
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
|
||||
const previousAgentStatusRef = useRef<Map<string, AgentLifecycleStatus>>(
|
||||
new Map()
|
||||
);
|
||||
@@ -307,6 +354,10 @@ function SessionProviderInternal({
|
||||
const revalidationInFlightRef = useRef<Promise<void> | null>(null);
|
||||
const revalidationQueuedRef = useRef(false);
|
||||
const wasConnectedRef = useRef(isConnected);
|
||||
const audioOutputBuffersRef = useRef<Map<string, BufferedAudioChunk[]>>(
|
||||
new Map()
|
||||
);
|
||||
const activeAudioGroupsRef = useRef<Set<string>>(new Set());
|
||||
|
||||
useEffect(() => {
|
||||
const subscription = AppState.addEventListener("change", (nextState) => {
|
||||
@@ -452,11 +503,6 @@ function SessionProviderInternal({
|
||||
const queue = session?.queuedMessages.get(agent.id);
|
||||
if (queue && queue.length > 0) {
|
||||
const [next, ...rest] = queue;
|
||||
console.log(
|
||||
"[Session] Flushing queued message for agent:",
|
||||
agent.id,
|
||||
next.text
|
||||
);
|
||||
if (sendAgentMessageRef.current) {
|
||||
void sendAgentMessageRef.current(agent.id, next.text, next.images);
|
||||
}
|
||||
@@ -570,18 +616,10 @@ function SessionProviderInternal({
|
||||
if (params.reason === "error") {
|
||||
return;
|
||||
}
|
||||
const isActive = appState ? appState === "active" : true;
|
||||
const isAwayFromAgent = !isActive || focusedAgentId !== params.agentId;
|
||||
const isActivelyVisible = getIsAppActivelyVisible(appState);
|
||||
const isAwayFromAgent =
|
||||
!isActivelyVisible || focusedAgentId !== params.agentId;
|
||||
if (!isAwayFromAgent) {
|
||||
console.log(
|
||||
"[OSNotifications] Skipping attention notification: user already focused on agent",
|
||||
{
|
||||
agentId: params.agentId,
|
||||
reason: params.reason,
|
||||
appState,
|
||||
focusedAgentId,
|
||||
}
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -624,24 +662,55 @@ function SessionProviderInternal({
|
||||
[serverId]
|
||||
);
|
||||
|
||||
// Buffer for streaming audio chunks
|
||||
interface AudioChunk {
|
||||
chunkIndex: number;
|
||||
audio: string; // base64
|
||||
format: string;
|
||||
id: string;
|
||||
}
|
||||
const audioChunkBuffersRef = useRef<Map<string, AudioChunk[]>>(new Map());
|
||||
|
||||
// Initialize session in store
|
||||
useEffect(() => {
|
||||
initializeSession(serverId, client, audioPlayer);
|
||||
}, [serverId, client, audioPlayer, initializeSession]);
|
||||
initializeSession(serverId, client);
|
||||
}, [serverId, client, initializeSession]);
|
||||
|
||||
useEffect(() => {
|
||||
updateSessionClient(serverId, client);
|
||||
}, [serverId, client, updateSessionClient]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!voiceRuntime) {
|
||||
return;
|
||||
}
|
||||
|
||||
return voiceRuntime.registerSession({
|
||||
serverId,
|
||||
setVoiceMode: async (enabled, agentId) => {
|
||||
if (!client) {
|
||||
throw new Error("Daemon unavailable");
|
||||
}
|
||||
await client.setVoiceMode(enabled, agentId);
|
||||
},
|
||||
sendVoiceAudioChunk: async (audioData, mimeType, isLast) => {
|
||||
if (!client) {
|
||||
throw new Error("Daemon unavailable");
|
||||
}
|
||||
await client.sendVoiceAudioChunk(audioData, mimeType, isLast);
|
||||
},
|
||||
abortRequest: async () => {
|
||||
if (!client) {
|
||||
throw new Error("Daemon unavailable");
|
||||
}
|
||||
await client.abortRequest();
|
||||
},
|
||||
setAssistantAudioPlaying: (isPlaying) => {
|
||||
setIsPlayingAudio(serverId, isPlaying);
|
||||
},
|
||||
});
|
||||
}, [
|
||||
client,
|
||||
serverId,
|
||||
setIsPlayingAudio,
|
||||
voiceRuntime,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
voiceRuntime?.updateSessionConnection(serverId, isConnected);
|
||||
}, [isConnected, serverId, voiceRuntime]);
|
||||
|
||||
// If the client drops mid-initialization, clear pending flags
|
||||
useEffect(() => {
|
||||
if (!isConnected) {
|
||||
@@ -969,6 +1038,14 @@ function SessionProviderInternal({
|
||||
const { agentId, event, timestamp, seq, epoch } = message.payload;
|
||||
const parsedTimestamp = new Date(timestamp);
|
||||
const streamEvent = event as AgentStreamEventPayload;
|
||||
if (
|
||||
event.type === "turn_started" ||
|
||||
event.type === "turn_completed" ||
|
||||
event.type === "turn_failed" ||
|
||||
event.type === "turn_canceled"
|
||||
) {
|
||||
voiceRuntime?.onTurnEvent(serverId, agentId, event.type);
|
||||
}
|
||||
|
||||
// Attention notification stays in React (not extractable to pure reducer)
|
||||
if (event.type === "attention_required") {
|
||||
@@ -1118,13 +1195,6 @@ function SessionProviderInternal({
|
||||
if (message.type !== "agent_permission_request") return;
|
||||
const { agentId, request } = message.payload;
|
||||
|
||||
console.log(
|
||||
"[Session] Permission request:",
|
||||
request.id,
|
||||
"for agent:",
|
||||
agentId
|
||||
);
|
||||
|
||||
setPendingPermissions(serverId, (prev) => {
|
||||
const next = new Map(prev);
|
||||
const key = derivePendingPermissionKey(agentId, request);
|
||||
@@ -1140,13 +1210,6 @@ function SessionProviderInternal({
|
||||
if (message.type !== "agent_permission_resolved") return;
|
||||
const { requestId, agentId } = message.payload;
|
||||
|
||||
console.log(
|
||||
"[Session] Permission resolved:",
|
||||
requestId,
|
||||
"for agent:",
|
||||
agentId
|
||||
);
|
||||
|
||||
setPendingPermissions(serverId, (prev) => {
|
||||
const next = new Map(prev);
|
||||
const derivedKey = `${agentId}:${requestId}`;
|
||||
@@ -1168,92 +1231,70 @@ function SessionProviderInternal({
|
||||
|
||||
const unsubAudioOutput = client.on("audio_output", async (message) => {
|
||||
if (message.type !== "audio_output") return;
|
||||
const data = message.payload;
|
||||
const playbackGroupId = data.groupId ?? data.id;
|
||||
const isFinalChunk = data.isLastChunk ?? true;
|
||||
const chunkIndex = data.chunkIndex ?? 0;
|
||||
if (!voiceAudioEngine) {
|
||||
return;
|
||||
}
|
||||
|
||||
const payload: AudioOutputPayload = message.payload;
|
||||
const playbackGroupId = payload.groupId ?? payload.id;
|
||||
const chunkIndex = payload.chunkIndex ?? 0;
|
||||
const isFinalChunk = payload.isLastChunk ?? true;
|
||||
|
||||
if (!audioOutputBuffersRef.current.has(playbackGroupId)) {
|
||||
audioOutputBuffersRef.current.set(playbackGroupId, []);
|
||||
}
|
||||
|
||||
const bufferedChunks = audioOutputBuffersRef.current.get(playbackGroupId)!;
|
||||
bufferedChunks.push({
|
||||
chunkIndex,
|
||||
audio: payload.audio,
|
||||
format: payload.format,
|
||||
id: payload.id,
|
||||
});
|
||||
|
||||
activeAudioGroupsRef.current.add(playbackGroupId);
|
||||
setIsPlayingAudio(serverId, true);
|
||||
|
||||
if (!audioChunkBuffersRef.current.has(playbackGroupId)) {
|
||||
audioChunkBuffersRef.current.set(playbackGroupId, []);
|
||||
}
|
||||
|
||||
const buffer = audioChunkBuffersRef.current.get(playbackGroupId)!;
|
||||
buffer.push({
|
||||
chunkIndex,
|
||||
audio: data.audio,
|
||||
format: data.format,
|
||||
id: data.id,
|
||||
});
|
||||
|
||||
if (!isFinalChunk) {
|
||||
return;
|
||||
}
|
||||
buffer.sort((a, b) => a.chunkIndex - b.chunkIndex);
|
||||
|
||||
let playbackFailed = false;
|
||||
const chunkIds = buffer.map((chunk) => chunk.id);
|
||||
|
||||
const confirmAudioPlayed = (ids: string[]) => {
|
||||
if (!client) {
|
||||
console.warn("[Session] audio_played skipped: daemon unavailable");
|
||||
return;
|
||||
}
|
||||
ids.forEach((chunkId) => {
|
||||
void client.audioPlayed(chunkId).catch((error) => {
|
||||
console.warn("[Session] Failed to confirm audio playback:", error);
|
||||
});
|
||||
});
|
||||
bufferedChunks.sort((left, right) => left.chunkIndex - right.chunkIndex);
|
||||
const chunkIds = bufferedChunks.map((chunk) => chunk.id);
|
||||
const shouldPlay =
|
||||
!payload.isVoiceMode ||
|
||||
(voiceRuntime?.shouldPlayVoiceAudio(serverId) ?? false);
|
||||
const audioBlob = buildAudioPlaybackSource(bufferedChunks);
|
||||
const confirmAudioPlayed = async () => {
|
||||
await Promise.all(
|
||||
chunkIds.map((chunkId) =>
|
||||
client.audioPlayed(chunkId).catch((error) => {
|
||||
console.warn("[Session] Failed to confirm audio playback:", error);
|
||||
})
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
let startedVoicePlayback = false;
|
||||
try {
|
||||
const mimeType =
|
||||
data.format === "mp3" ? "audio/mpeg" : `audio/${data.format}`;
|
||||
|
||||
const decodedChunks: Uint8Array[] = [];
|
||||
let totalSize = 0;
|
||||
|
||||
for (const chunk of buffer) {
|
||||
const binaryString = atob(chunk.audio);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
if (shouldPlay) {
|
||||
if (payload.isVoiceMode) {
|
||||
startedVoicePlayback = true;
|
||||
voiceRuntime?.onAssistantAudioStarted(serverId);
|
||||
}
|
||||
decodedChunks.push(bytes);
|
||||
totalSize += bytes.length;
|
||||
await voiceAudioEngine.play(audioBlob);
|
||||
}
|
||||
|
||||
const concatenatedBytes = new Uint8Array(totalSize);
|
||||
let offset = 0;
|
||||
for (const chunk of decodedChunks) {
|
||||
concatenatedBytes.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
|
||||
const audioBlob = {
|
||||
type: mimeType,
|
||||
size: totalSize,
|
||||
arrayBuffer: async () => {
|
||||
return concatenatedBytes.buffer;
|
||||
},
|
||||
} as Blob;
|
||||
|
||||
await audioPlayer.play(audioBlob);
|
||||
|
||||
confirmAudioPlayed(chunkIds);
|
||||
} catch (error: any) {
|
||||
playbackFailed = true;
|
||||
await confirmAudioPlayed();
|
||||
} catch (error) {
|
||||
console.error("[Session] Audio playback error:", error);
|
||||
|
||||
confirmAudioPlayed(chunkIds);
|
||||
await confirmAudioPlayed();
|
||||
} finally {
|
||||
audioChunkBuffersRef.current.delete(playbackGroupId);
|
||||
audioOutputBuffersRef.current.delete(playbackGroupId);
|
||||
activeAudioGroupsRef.current.delete(playbackGroupId);
|
||||
setIsPlayingAudio(serverId, activeAudioGroupsRef.current.size > 0);
|
||||
|
||||
if (activeAudioGroupsRef.current.size === 0) {
|
||||
setIsPlayingAudio(serverId, false);
|
||||
if (startedVoicePlayback) {
|
||||
voiceRuntime?.onAssistantAudioFinished(serverId);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1261,7 +1302,6 @@ function SessionProviderInternal({
|
||||
const unsubActivity = client.on("activity_log", (message) => {
|
||||
if (message.type !== "activity_log") return;
|
||||
const data = message.payload;
|
||||
|
||||
if (data.type === "system" && data.content.includes("Transcribing")) {
|
||||
return;
|
||||
}
|
||||
@@ -1381,13 +1421,12 @@ function SessionProviderInternal({
|
||||
if (message.type !== "transcription_result") return;
|
||||
|
||||
const transcriptText = message.payload.text.trim();
|
||||
|
||||
if (!transcriptText) {
|
||||
} else {
|
||||
audioPlayer.stop();
|
||||
setIsPlayingAudio(serverId, false);
|
||||
setCurrentAssistantMessage(serverId, "");
|
||||
voiceRuntime?.onTranscriptionResult(serverId, transcriptText);
|
||||
return;
|
||||
}
|
||||
|
||||
setCurrentAssistantMessage(serverId, "");
|
||||
});
|
||||
|
||||
const unsubAgentDeleted = client.on("agent_deleted", (message) => {
|
||||
@@ -1395,7 +1434,6 @@ function SessionProviderInternal({
|
||||
return;
|
||||
}
|
||||
const { agentId } = message.payload;
|
||||
console.log("[Session] Agent deleted:", agentId);
|
||||
deletePendingAgentUpdate(serverId, agentId);
|
||||
clearArchiveAgentPending({ queryClient, serverId, agentId });
|
||||
|
||||
@@ -1471,7 +1509,6 @@ function SessionProviderInternal({
|
||||
return;
|
||||
}
|
||||
const { agentId, archivedAt } = message.payload;
|
||||
console.log("[Session] Agent archived:", agentId);
|
||||
clearArchiveAgentPending({ queryClient, serverId, agentId });
|
||||
|
||||
setAgents(serverId, (prev) => {
|
||||
@@ -1505,7 +1542,6 @@ function SessionProviderInternal({
|
||||
};
|
||||
}, [
|
||||
client,
|
||||
audioPlayer,
|
||||
queryClient,
|
||||
serverId,
|
||||
setIsPlayingAudio,
|
||||
@@ -1528,6 +1564,8 @@ function SessionProviderInternal({
|
||||
requestCanonicalCatchUp,
|
||||
applyAgentUpdatePayload,
|
||||
applyTimelineResponse,
|
||||
voiceRuntime,
|
||||
voiceAudioEngine,
|
||||
]);
|
||||
|
||||
const sendAgentMessage = useCallback(
|
||||
@@ -1654,11 +1692,6 @@ function SessionProviderInternal({
|
||||
worktreeName?: string;
|
||||
requestId?: string;
|
||||
}) => {
|
||||
console.log(
|
||||
"[Session] createAgent called with images:",
|
||||
images?.length ?? 0,
|
||||
images
|
||||
);
|
||||
if (!client) {
|
||||
console.warn("[Session] createAgent skipped: daemon unavailable");
|
||||
return;
|
||||
@@ -1667,14 +1700,6 @@ function SessionProviderInternal({
|
||||
let imagesData: Array<{ data: string; mimeType: string }> | undefined;
|
||||
try {
|
||||
imagesData = await encodeImages(images);
|
||||
console.log(
|
||||
"[Session] encodeImages result:",
|
||||
imagesData?.length ?? 0,
|
||||
imagesData?.map((img) => ({
|
||||
dataLength: img.data?.length ?? 0,
|
||||
mimeType: img.mimeType,
|
||||
}))
|
||||
);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"[Session] Failed to prepare images for agent creation:",
|
||||
@@ -1749,20 +1774,12 @@ function SessionProviderInternal({
|
||||
[client]
|
||||
);
|
||||
|
||||
const setVoiceDetectionFlags = useCallback(
|
||||
(isDetecting: boolean, isSpeaking: boolean) => {
|
||||
isDetectingRef.current = isDetecting;
|
||||
isSpeakingRef.current = isSpeaking;
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
// Cleanup on unmount
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
clearSession(serverId);
|
||||
};
|
||||
}, [serverId, clearSession]);
|
||||
}, [clearSession, serverId]);
|
||||
|
||||
return children;
|
||||
}
|
||||
|
||||
@@ -1,41 +1,106 @@
|
||||
import { createContext, useContext, useState, ReactNode, useCallback, useEffect, useRef } from "react";
|
||||
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
|
||||
import type { SessionState } from "@/stores/session-store";
|
||||
import { useSessionStore } from "@/stores/session-store";
|
||||
import { useHostRuntimeSession } from "@/runtime/host-runtime";
|
||||
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
|
||||
import {
|
||||
createContext,
|
||||
useContext,
|
||||
useEffect,
|
||||
useRef,
|
||||
useSyncExternalStore,
|
||||
type ReactNode,
|
||||
} from "react";
|
||||
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
|
||||
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
|
||||
import { useSessionStore } from "@/stores/session-store";
|
||||
import { createAudioEngine } from "@/voice/audio-engine";
|
||||
import type { AudioEngine } from "@/voice/audio-engine-types";
|
||||
import {
|
||||
createVoiceRuntime,
|
||||
type VoiceRuntime,
|
||||
type VoiceRuntimeSnapshot,
|
||||
type VoiceRuntimeTelemetrySnapshot,
|
||||
} from "@/voice/voice-runtime";
|
||||
|
||||
const KEEP_AWAKE_TAG = "paseo:voice";
|
||||
interface VoiceContextValue {
|
||||
isVoiceMode: boolean;
|
||||
isVoiceSwitching: boolean;
|
||||
volume: number;
|
||||
isMuted: boolean;
|
||||
isDetecting: boolean;
|
||||
isSpeaking: boolean;
|
||||
segmentDuration: number;
|
||||
interface VoiceContextValue extends VoiceRuntimeSnapshot {
|
||||
startVoice: (serverId: string, agentId: string) => Promise<void>;
|
||||
stopVoice: () => Promise<void>;
|
||||
isVoiceModeForAgent: (serverId: string, agentId: string) => boolean;
|
||||
toggleMute: () => void;
|
||||
activeServerId: string | null;
|
||||
activeAgentId: string | null;
|
||||
}
|
||||
|
||||
const VoiceContext = createContext<VoiceContextValue | null>(null);
|
||||
const EMPTY_SNAPSHOT: VoiceRuntimeSnapshot = {
|
||||
phase: "disabled",
|
||||
isVoiceMode: false,
|
||||
isVoiceSwitching: false,
|
||||
isMuted: false,
|
||||
activeServerId: null,
|
||||
activeAgentId: null,
|
||||
};
|
||||
|
||||
const EMPTY_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
|
||||
volume: 0,
|
||||
isDetecting: false,
|
||||
isSpeaking: false,
|
||||
segmentDuration: 0,
|
||||
};
|
||||
|
||||
const VoiceRuntimeContext = createContext<VoiceRuntime | null>(null);
|
||||
const VoiceAudioEngineContext = createContext<AudioEngine | null>(null);
|
||||
|
||||
const noopSubscribe = () => () => {};
|
||||
const getEmptySnapshot = () => EMPTY_SNAPSHOT;
|
||||
const getEmptyTelemetry = () => EMPTY_TELEMETRY;
|
||||
|
||||
export function useVoice() {
|
||||
const context = useContext(VoiceContext);
|
||||
if (!context) {
|
||||
const value = useVoiceOptional();
|
||||
if (!value) {
|
||||
throw new Error("useVoice must be used within VoiceProvider");
|
||||
}
|
||||
return context;
|
||||
return value;
|
||||
}
|
||||
|
||||
export function useVoiceOptional(): VoiceContextValue | null {
|
||||
return useContext(VoiceContext);
|
||||
const runtime = useContext(VoiceRuntimeContext);
|
||||
const snapshot = useSyncExternalStore(
|
||||
runtime ? runtime.subscribe : noopSubscribe,
|
||||
runtime ? runtime.getSnapshot : getEmptySnapshot,
|
||||
runtime ? runtime.getSnapshot : getEmptySnapshot
|
||||
);
|
||||
|
||||
if (!runtime) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
...snapshot,
|
||||
startVoice: runtime.startVoice,
|
||||
stopVoice: runtime.stopVoice,
|
||||
isVoiceModeForAgent: runtime.isVoiceModeForAgent,
|
||||
toggleMute: runtime.toggleMute,
|
||||
};
|
||||
}
|
||||
|
||||
export function useVoiceTelemetry() {
|
||||
const telemetry = useVoiceTelemetryOptional();
|
||||
if (!telemetry) {
|
||||
throw new Error("useVoiceTelemetry must be used within VoiceProvider");
|
||||
}
|
||||
return telemetry;
|
||||
}
|
||||
|
||||
export function useVoiceTelemetryOptional(): VoiceRuntimeTelemetrySnapshot | null {
|
||||
const runtime = useContext(VoiceRuntimeContext);
|
||||
const snapshot = useSyncExternalStore(
|
||||
runtime ? runtime.subscribeTelemetry : noopSubscribe,
|
||||
runtime ? runtime.getTelemetrySnapshot : getEmptyTelemetry,
|
||||
runtime ? runtime.getTelemetrySnapshot : getEmptyTelemetry
|
||||
);
|
||||
|
||||
return runtime ? snapshot : null;
|
||||
}
|
||||
|
||||
export function useVoiceRuntimeOptional(): VoiceRuntime | null {
|
||||
return useContext(VoiceRuntimeContext);
|
||||
}
|
||||
|
||||
export function useVoiceAudioEngineOptional(): AudioEngine | null {
|
||||
return useContext(VoiceAudioEngineContext);
|
||||
}
|
||||
|
||||
interface VoiceProviderProps {
|
||||
@@ -43,405 +108,55 @@ interface VoiceProviderProps {
|
||||
}
|
||||
|
||||
export function VoiceProvider({ children }: VoiceProviderProps) {
|
||||
const getSession = useSessionStore((state) => state.getSession);
|
||||
const [activeServerId, setActiveServerId] = useState<string | null>(null);
|
||||
const [activeAgentId, setActiveAgentId] = useState<string | null>(null);
|
||||
const { client: runtimeClient, isConnected: activeRuntimeConnected } = useHostRuntimeSession(
|
||||
activeServerId ?? ""
|
||||
);
|
||||
const activeSession = useSessionStore(
|
||||
useCallback(
|
||||
(state: ReturnType<typeof useSessionStore.getState>) => {
|
||||
if (!activeServerId) {
|
||||
return null;
|
||||
}
|
||||
return state.sessions[activeServerId] ?? null;
|
||||
const engineRef = useRef<AudioEngine | null>(null);
|
||||
const runtimeRef = useRef<VoiceRuntime | null>(null);
|
||||
|
||||
if (!engineRef.current) {
|
||||
let runtime: VoiceRuntime | null = null;
|
||||
const engine = createAudioEngine({
|
||||
onCaptureData: (pcm) => {
|
||||
runtime?.handleCapturePcm(pcm);
|
||||
},
|
||||
[activeServerId]
|
||||
)
|
||||
);
|
||||
const realtimeSessionRef = useRef<SessionState | null>(null);
|
||||
const [isVoiceMode, setIsVoiceMode] = useState(false);
|
||||
const [isVoiceSwitching, setIsVoiceSwitching] = useState(false);
|
||||
const bargeInPlaybackStopRef = useRef<number | null>(null);
|
||||
const wasVoiceSocketConnectedRef = useRef(false);
|
||||
const lastVoiceModeSyncedClientRef = useRef<SessionState["client"] | null>(null);
|
||||
const voiceTransportReadyRef = useRef(false);
|
||||
const voiceResyncInFlightRef = useRef(false);
|
||||
const silenceGraceStartMsRef = useRef<number | null>(null);
|
||||
const speechInterruptTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const speechInterruptStartMsRef = useRef<number | null>(null);
|
||||
const speechStartInterruptSentRef = useRef(false);
|
||||
const isVoiceModeRef = useRef(false);
|
||||
const vadStateRef = useRef<{ isDetecting: boolean; isSpeaking: boolean }>({
|
||||
isDetecting: false,
|
||||
isSpeaking: false,
|
||||
});
|
||||
onVolumeLevel: (level) => {
|
||||
runtime?.handleCaptureVolume(level);
|
||||
},
|
||||
onError: (error) => {
|
||||
console.error("[VoiceEngine] Capture error:", error);
|
||||
},
|
||||
});
|
||||
|
||||
const clearSpeechStartInterruptTimer = useCallback((reason: string) => {
|
||||
const timer = speechInterruptTimerRef.current;
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
speechInterruptTimerRef.current = null;
|
||||
const startedAt = speechInterruptStartMsRef.current;
|
||||
speechInterruptStartMsRef.current = null;
|
||||
if (startedAt !== null) {
|
||||
console.log("[Voice] Cleared speech-start interrupt timer", {
|
||||
reason,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
} else {
|
||||
console.log("[Voice] Cleared speech-start interrupt timer", { reason });
|
||||
}
|
||||
return;
|
||||
}
|
||||
speechInterruptStartMsRef.current = null;
|
||||
}, []);
|
||||
runtime = createVoiceRuntime({
|
||||
engine,
|
||||
getServerInfo: (serverId) =>
|
||||
useSessionStore.getState().getSession(serverId)?.serverInfo ?? null,
|
||||
activateKeepAwake: async (tag) => {
|
||||
await activateKeepAwakeAsync(tag);
|
||||
},
|
||||
deactivateKeepAwake: async (tag) => {
|
||||
await deactivateKeepAwake(tag);
|
||||
},
|
||||
});
|
||||
|
||||
const interruptActiveVoiceTurn = useCallback((source: string) => {
|
||||
const session = realtimeSessionRef.current;
|
||||
const sessionAudioPlayer = session?.audioPlayer ?? null;
|
||||
const sessionClient = session?.client ?? null;
|
||||
const sessionIsPlayingAudio = session?.isPlayingAudio ?? false;
|
||||
engineRef.current = engine;
|
||||
runtimeRef.current = runtime;
|
||||
}
|
||||
|
||||
if (sessionIsPlayingAudio && sessionAudioPlayer) {
|
||||
if (bargeInPlaybackStopRef.current === null) {
|
||||
bargeInPlaybackStopRef.current = Date.now();
|
||||
}
|
||||
sessionAudioPlayer.stop();
|
||||
}
|
||||
|
||||
try {
|
||||
if (sessionClient) {
|
||||
void sessionClient.abortRequest().catch((error) => {
|
||||
console.error("[Voice] Failed to send abort_request:", error);
|
||||
});
|
||||
}
|
||||
console.log("[Voice] Sent abort_request before streaming audio", { source });
|
||||
} catch (error) {
|
||||
console.error("[Voice] Failed to send abort_request:", error);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const realtimeAudio = useSpeechmaticsAudio({
|
||||
onSpeechStart: () => {
|
||||
console.log("[Voice] Segment started (speech confirmed)");
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
const silenceMs =
|
||||
silenceGraceStartMsRef.current === null
|
||||
? null
|
||||
: Date.now() - silenceGraceStartMsRef.current;
|
||||
if (silenceMs === null) {
|
||||
console.log("[Voice] Segment finalized");
|
||||
} else {
|
||||
console.log("[Voice] Segment finalized", { silenceMs });
|
||||
}
|
||||
silenceGraceStartMsRef.current = null;
|
||||
clearSpeechStartInterruptTimer("speech ended");
|
||||
speechStartInterruptSentRef.current = false;
|
||||
},
|
||||
onAudioSegment: ({ audioData, isLast }) => {
|
||||
if (!voiceTransportReadyRef.current) {
|
||||
console.log("[Voice] Skipping audio segment: voice transport not ready");
|
||||
return;
|
||||
}
|
||||
console.log(
|
||||
"[Voice] Sending audio segment, length:",
|
||||
audioData.length,
|
||||
"isLast:",
|
||||
isLast
|
||||
);
|
||||
|
||||
// Send audio segment to server (realtime always goes to orchestrator)
|
||||
const session = realtimeSessionRef.current;
|
||||
try {
|
||||
if (session?.client) {
|
||||
void session.client
|
||||
.sendVoiceAudioChunk(
|
||||
audioData,
|
||||
"audio/pcm;rate=16000;bits=16",
|
||||
isLast
|
||||
)
|
||||
.catch((error) => {
|
||||
console.error("[Voice] Failed to send audio segment:", error);
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("[Voice] Failed to send audio segment:", error);
|
||||
}
|
||||
},
|
||||
onError: (error) => {
|
||||
console.error("[Voice] Audio error:", error);
|
||||
const session = realtimeSessionRef.current;
|
||||
if (session?.client) {
|
||||
// Send error through websocket instead of directly manipulating messages
|
||||
console.error("[Voice] Cannot handle error - setMessages not available from SessionState");
|
||||
}
|
||||
},
|
||||
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
|
||||
confirmedDropGracePeriod: REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
|
||||
silenceDuration: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
|
||||
speechConfirmationDuration: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
|
||||
detectionGracePeriod: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
realtimeSessionRef.current = activeSession;
|
||||
}, [activeSession]);
|
||||
|
||||
useEffect(() => {
|
||||
isVoiceModeRef.current = isVoiceMode;
|
||||
}, [isVoiceMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isVoiceMode) {
|
||||
clearSpeechStartInterruptTimer("voice mode disabled");
|
||||
speechStartInterruptSentRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (realtimeAudio.isSpeaking) {
|
||||
if (
|
||||
speechStartInterruptSentRef.current ||
|
||||
speechInterruptTimerRef.current !== null
|
||||
) {
|
||||
return;
|
||||
}
|
||||
speechInterruptStartMsRef.current = Date.now();
|
||||
speechInterruptTimerRef.current = setTimeout(() => {
|
||||
speechInterruptTimerRef.current = null;
|
||||
speechInterruptStartMsRef.current = null;
|
||||
if (
|
||||
!isVoiceModeRef.current ||
|
||||
!vadStateRef.current.isSpeaking ||
|
||||
speechStartInterruptSentRef.current
|
||||
) {
|
||||
return;
|
||||
}
|
||||
speechStartInterruptSentRef.current = true;
|
||||
console.log("[Voice] Speech persisted beyond grace; interrupting turn", {
|
||||
graceMs: REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs,
|
||||
});
|
||||
interruptActiveVoiceTurn("speech_start_grace_elapsed");
|
||||
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
|
||||
return;
|
||||
}
|
||||
|
||||
clearSpeechStartInterruptTimer("speech stopped before grace");
|
||||
}, [
|
||||
clearSpeechStartInterruptTimer,
|
||||
interruptActiveVoiceTurn,
|
||||
isVoiceMode,
|
||||
realtimeAudio.isSpeaking,
|
||||
]);
|
||||
const engine = engineRef.current!;
|
||||
const runtime = runtimeRef.current!;
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
clearSpeechStartInterruptTimer("voice provider unmounted");
|
||||
};
|
||||
}, [clearSpeechStartInterruptTimer]);
|
||||
|
||||
useEffect(() => {
|
||||
const next = {
|
||||
isDetecting: realtimeAudio.isDetecting,
|
||||
isSpeaking: realtimeAudio.isSpeaking,
|
||||
};
|
||||
const prev = vadStateRef.current;
|
||||
|
||||
if (!isVoiceMode) {
|
||||
silenceGraceStartMsRef.current = null;
|
||||
vadStateRef.current = next;
|
||||
return;
|
||||
}
|
||||
|
||||
// Confirmed speech dropped below threshold: grace window starts.
|
||||
if (prev.isSpeaking && !next.isSpeaking && next.isDetecting) {
|
||||
silenceGraceStartMsRef.current = Date.now();
|
||||
console.log("[Voice] Grace started (speech dropped below threshold)");
|
||||
}
|
||||
|
||||
// User resumed speaking before grace timeout elapsed.
|
||||
if (!prev.isSpeaking && next.isSpeaking && silenceGraceStartMsRef.current !== null) {
|
||||
const resumedAfterMs = Date.now() - silenceGraceStartMsRef.current;
|
||||
console.log("[Voice] Speech resumed during grace", { resumedAfterMs });
|
||||
silenceGraceStartMsRef.current = null;
|
||||
}
|
||||
|
||||
// Fully idle (neither detecting nor speaking).
|
||||
if (!next.isDetecting && !next.isSpeaking) {
|
||||
silenceGraceStartMsRef.current = null;
|
||||
speechStartInterruptSentRef.current = false;
|
||||
}
|
||||
|
||||
vadStateRef.current = next;
|
||||
}, [isVoiceMode, realtimeAudio.isDetecting, realtimeAudio.isSpeaking]);
|
||||
|
||||
useEffect(() => {
|
||||
const connected = activeRuntimeConnected;
|
||||
const client = runtimeClient;
|
||||
if (!connected) {
|
||||
voiceTransportReadyRef.current = false;
|
||||
}
|
||||
|
||||
if (!isVoiceMode || !activeServerId || !activeAgentId || !client) {
|
||||
wasVoiceSocketConnectedRef.current = connected;
|
||||
if (!isVoiceMode) {
|
||||
lastVoiceModeSyncedClientRef.current = null;
|
||||
}
|
||||
voiceTransportReadyRef.current = false;
|
||||
return;
|
||||
}
|
||||
|
||||
const connectionRecovered = connected && !wasVoiceSocketConnectedRef.current;
|
||||
const clientChanged = lastVoiceModeSyncedClientRef.current !== client;
|
||||
if (connected && (connectionRecovered || clientChanged)) {
|
||||
if (!voiceResyncInFlightRef.current) {
|
||||
voiceResyncInFlightRef.current = true;
|
||||
voiceTransportReadyRef.current = false;
|
||||
setIsVoiceSwitching(true);
|
||||
void client.setVoiceMode(true, activeAgentId).then(
|
||||
() => {
|
||||
console.log("[Voice] Re-synced voice mode after reconnect");
|
||||
lastVoiceModeSyncedClientRef.current = client;
|
||||
voiceTransportReadyRef.current = true;
|
||||
},
|
||||
(error) => {
|
||||
console.error("[Voice] Failed to re-sync voice mode:", error);
|
||||
}
|
||||
).finally(() => {
|
||||
voiceResyncInFlightRef.current = false;
|
||||
setIsVoiceSwitching(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
wasVoiceSocketConnectedRef.current = connected;
|
||||
}, [activeAgentId, activeRuntimeConnected, activeServerId, isVoiceMode, runtimeClient]);
|
||||
|
||||
const isPlayingAudio = activeSession?.isPlayingAudio ?? false;
|
||||
|
||||
useEffect(() => {
|
||||
if (!isPlayingAudio && bargeInPlaybackStopRef.current !== null) {
|
||||
const latencyMs = Date.now() - bargeInPlaybackStopRef.current;
|
||||
console.log("[Telemetry] barge_in.playback_stop_latency", {
|
||||
latencyMs,
|
||||
startedAt: new Date(bargeInPlaybackStopRef.current).toISOString(),
|
||||
completedAt: new Date().toISOString(),
|
||||
void runtime.destroy().catch((error) => {
|
||||
console.error("[VoiceProvider] Failed to destroy voice runtime", error);
|
||||
});
|
||||
bargeInPlaybackStopRef.current = null;
|
||||
}
|
||||
}, [isPlayingAudio]);
|
||||
|
||||
const startVoice = useCallback(
|
||||
async (serverId: string, agentId: string) => {
|
||||
const session = getSession(serverId) ?? null;
|
||||
if (!session) {
|
||||
throw new Error(`Host ${serverId} is not connected`);
|
||||
}
|
||||
const unavailableMessage = resolveVoiceUnavailableMessage({
|
||||
serverInfo: session.serverInfo,
|
||||
mode: "voice",
|
||||
});
|
||||
if (unavailableMessage) {
|
||||
throw new Error(unavailableMessage);
|
||||
}
|
||||
|
||||
setIsVoiceSwitching(true);
|
||||
voiceTransportReadyRef.current = false;
|
||||
try {
|
||||
const previousSession = realtimeSessionRef.current;
|
||||
if (
|
||||
isVoiceMode &&
|
||||
previousSession?.client &&
|
||||
(activeServerId !== serverId || activeAgentId !== agentId)
|
||||
) {
|
||||
await previousSession.client.setVoiceMode(false);
|
||||
}
|
||||
|
||||
realtimeSessionRef.current = session;
|
||||
setActiveServerId(serverId);
|
||||
setActiveAgentId(agentId);
|
||||
await activateKeepAwakeAsync(KEEP_AWAKE_TAG).catch((error) => {
|
||||
console.warn("[Voice] Failed to activate keep-awake:", error);
|
||||
});
|
||||
if (session?.client) {
|
||||
await session.client.setVoiceMode(true, agentId);
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
await session.audioPlayer?.warmup?.();
|
||||
await realtimeAudio.start();
|
||||
voiceTransportReadyRef.current = true;
|
||||
setIsVoiceMode(true);
|
||||
lastVoiceModeSyncedClientRef.current = session.client;
|
||||
console.log("[Voice] Mode enabled");
|
||||
} catch (error: any) {
|
||||
console.error("[Voice] Failed to start:", error);
|
||||
await realtimeAudio.stop().catch(() => undefined);
|
||||
setActiveServerId((current) => (current === serverId ? null : current));
|
||||
setActiveAgentId((current) => (current === agentId ? null : current));
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsVoiceSwitching(false);
|
||||
}
|
||||
},
|
||||
[activeAgentId, activeServerId, getSession, isVoiceMode, realtimeAudio]
|
||||
);
|
||||
|
||||
const stopVoice = useCallback(async () => {
|
||||
setIsVoiceSwitching(true);
|
||||
voiceTransportReadyRef.current = false;
|
||||
try {
|
||||
const session = realtimeSessionRef.current;
|
||||
session?.audioPlayer?.stop();
|
||||
if (session?.client) {
|
||||
await session.client.setVoiceMode(false);
|
||||
lastVoiceModeSyncedClientRef.current = session.client;
|
||||
} else {
|
||||
console.warn("[Voice] setVoiceMode skipped: daemon unavailable");
|
||||
}
|
||||
await realtimeAudio.stop();
|
||||
setIsVoiceMode(false);
|
||||
setActiveServerId(null);
|
||||
setActiveAgentId(null);
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
console.log("[Voice] Mode disabled");
|
||||
} catch (error: any) {
|
||||
console.error("[Voice] Failed to stop:", error);
|
||||
await deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
setIsVoiceSwitching(false);
|
||||
}
|
||||
}, [realtimeAudio]);
|
||||
|
||||
const isVoiceModeForAgent = useCallback(
|
||||
(serverId: string, agentId: string) =>
|
||||
isVoiceMode && activeServerId === serverId && activeAgentId === agentId,
|
||||
[activeAgentId, activeServerId, isVoiceMode]
|
||||
);
|
||||
|
||||
const value: VoiceContextValue = {
|
||||
isVoiceMode,
|
||||
isVoiceSwitching,
|
||||
volume: realtimeAudio.volume,
|
||||
isMuted: realtimeAudio.isMuted,
|
||||
isDetecting: realtimeAudio.isDetecting,
|
||||
isSpeaking: realtimeAudio.isSpeaking,
|
||||
segmentDuration: realtimeAudio.segmentDuration,
|
||||
startVoice,
|
||||
stopVoice,
|
||||
isVoiceModeForAgent,
|
||||
toggleMute: realtimeAudio.toggleMute,
|
||||
activeServerId,
|
||||
activeAgentId,
|
||||
};
|
||||
};
|
||||
}, [runtime]);
|
||||
|
||||
return (
|
||||
<VoiceContext.Provider value={value}>
|
||||
{children}
|
||||
</VoiceContext.Provider>
|
||||
<VoiceAudioEngineContext.Provider value={engine}>
|
||||
<VoiceRuntimeContext.Provider value={runtime}>
|
||||
{children}
|
||||
</VoiceRuntimeContext.Provider>
|
||||
</VoiceAudioEngineContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { invokeDesktopCommand } from "@/desktop/tauri/invoke-desktop-command";
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
|
||||
export const DESKTOP_NOTIFICATION_CLICK_EVENT = "desktop-notification-click";
|
||||
|
||||
export interface DesktopNotificationInput {
|
||||
title: string;
|
||||
body?: string;
|
||||
data?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface DesktopNotificationClickPayload {
|
||||
data?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type DesktopNotificationClickHandler = (
|
||||
payload: DesktopNotificationClickPayload
|
||||
) => void;
|
||||
|
||||
export type DesktopNotificationClickUnlisten = () => void;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export async function sendDesktopNotification(
|
||||
input: DesktopNotificationInput
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
await invokeDesktopCommand("send_desktop_notification", { input });
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn("[OSNotifications][Desktop] Failed to send desktop notification", error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function listenToDesktopNotificationClicks(
|
||||
handler: DesktopNotificationClickHandler
|
||||
): Promise<DesktopNotificationClickUnlisten> {
|
||||
const listen = getTauri()?.event?.listen;
|
||||
if (typeof listen !== "function") {
|
||||
throw new Error("Tauri event API is unavailable.");
|
||||
}
|
||||
|
||||
const unlisten = await listen(DESKTOP_NOTIFICATION_CLICK_EVENT, (event: unknown) => {
|
||||
const payload = isRecord(event) && isRecord(event.payload) ? event.payload : null;
|
||||
if (!payload) {
|
||||
return;
|
||||
}
|
||||
|
||||
handler({
|
||||
data: isRecord(payload.data) ? payload.data : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
return typeof unlisten === "function" ? unlisten : () => {};
|
||||
}
|
||||
107
packages/app/src/hooks/image-attachment-picker.test.ts
Normal file
107
packages/app/src/hooks/image-attachment-picker.test.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
|
||||
const tauriState = vi.hoisted(() => ({
|
||||
api: null as any,
|
||||
}));
|
||||
|
||||
vi.mock("@/utils/tauri", () => ({
|
||||
getTauri: () => tauriState.api,
|
||||
}));
|
||||
|
||||
import {
|
||||
normalizePickedImageAssets,
|
||||
openImagePathsWithTauriDialog,
|
||||
} from "./image-attachment-picker";
|
||||
|
||||
describe("image-attachment-picker", () => {
|
||||
beforeEach(() => {
|
||||
tauriState.api = null;
|
||||
});
|
||||
|
||||
it("normalizes a picked File into a blob source", async () => {
|
||||
const file = new File(["hello"], "picked.png", { type: "image/png" });
|
||||
|
||||
const result = await normalizePickedImageAssets([
|
||||
{
|
||||
uri: "blob:test",
|
||||
mimeType: "image/png",
|
||||
fileName: null,
|
||||
file,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]?.source.kind).toBe("blob");
|
||||
expect(result[0]?.fileName).toBe("picked.png");
|
||||
expect(result[0]?.mimeType).toBe("image/png");
|
||||
});
|
||||
|
||||
it("keeps filesystem picker results as file uris", async () => {
|
||||
const result = await normalizePickedImageAssets([
|
||||
{
|
||||
uri: "file:///tmp/picked.png",
|
||||
mimeType: "image/png",
|
||||
fileName: "picked.png",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toEqual([
|
||||
{
|
||||
source: { kind: "file_uri", uri: "file:///tmp/picked.png" },
|
||||
mimeType: "image/png",
|
||||
fileName: "picked.png",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("converts data urls into blob sources when no file path exists", async () => {
|
||||
const result = await normalizePickedImageAssets([
|
||||
{
|
||||
uri: "data:image/png;base64,AAEC",
|
||||
mimeType: "image/png",
|
||||
fileName: "inline.png",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0]?.source.kind).toBe("blob");
|
||||
expect(result[0]?.fileName).toBe("inline.png");
|
||||
expect(result[0]?.mimeType).toBe("image/png");
|
||||
});
|
||||
|
||||
it("uses the tauri dialog api when available", async () => {
|
||||
const open = vi.fn().mockResolvedValue(["/tmp/one.png", "/tmp/two.jpg"]);
|
||||
tauriState.api = {
|
||||
dialog: { open },
|
||||
};
|
||||
|
||||
const result = await openImagePathsWithTauriDialog();
|
||||
|
||||
expect(open).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
multiple: true,
|
||||
directory: false,
|
||||
title: "Attach images",
|
||||
})
|
||||
);
|
||||
expect(result).toEqual(["/tmp/one.png", "/tmp/two.jpg"]);
|
||||
});
|
||||
|
||||
it("falls back to core invoke for the tauri dialog plugin", async () => {
|
||||
const invoke = vi.fn().mockResolvedValue("/tmp/one.png");
|
||||
tauriState.api = {
|
||||
core: { invoke },
|
||||
};
|
||||
|
||||
const result = await openImagePathsWithTauriDialog();
|
||||
|
||||
expect(invoke).toHaveBeenCalledWith("plugin:dialog|open", {
|
||||
options: expect.objectContaining({
|
||||
multiple: true,
|
||||
directory: false,
|
||||
title: "Attach images",
|
||||
}),
|
||||
});
|
||||
expect(result).toEqual(["/tmp/one.png"]);
|
||||
});
|
||||
});
|
||||
110
packages/app/src/hooks/image-attachment-picker.ts
Normal file
110
packages/app/src/hooks/image-attachment-picker.ts
Normal file
@@ -0,0 +1,110 @@
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
|
||||
export type PickedImageSource =
|
||||
| { kind: "file_uri"; uri: string }
|
||||
| { kind: "blob"; blob: Blob };
|
||||
|
||||
export interface PickedImageAttachmentInput {
|
||||
source: PickedImageSource;
|
||||
mimeType?: string | null;
|
||||
fileName?: string | null;
|
||||
}
|
||||
|
||||
export interface ExpoImagePickerAssetLike {
|
||||
uri: string;
|
||||
mimeType?: string | null;
|
||||
fileName?: string | null;
|
||||
file?: File | null;
|
||||
}
|
||||
|
||||
const IMAGE_FILE_EXTENSIONS = [
|
||||
"png",
|
||||
"jpg",
|
||||
"jpeg",
|
||||
"gif",
|
||||
"webp",
|
||||
"avif",
|
||||
"heic",
|
||||
"heif",
|
||||
"tiff",
|
||||
"bmp",
|
||||
"svg",
|
||||
];
|
||||
|
||||
function isAbsoluteWindowsPath(value: string): boolean {
|
||||
return /^[a-zA-Z]:[\\/]/.test(value);
|
||||
}
|
||||
|
||||
function shouldTreatAsFileUri(uri: string): boolean {
|
||||
return uri.startsWith("file://") || uri.startsWith("/") || isAbsoluteWindowsPath(uri);
|
||||
}
|
||||
|
||||
async function blobFromUri(uri: string): Promise<Blob> {
|
||||
const response = await fetch(uri);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to read picked image from '${uri}'.`);
|
||||
}
|
||||
return await response.blob();
|
||||
}
|
||||
|
||||
export async function normalizePickedImageAssets(
|
||||
assets: readonly ExpoImagePickerAssetLike[]
|
||||
): Promise<PickedImageAttachmentInput[]> {
|
||||
return await Promise.all(
|
||||
assets.map(async (asset) => {
|
||||
if (asset.file instanceof Blob) {
|
||||
return {
|
||||
source: { kind: "blob", blob: asset.file },
|
||||
mimeType: asset.mimeType ?? asset.file.type ?? null,
|
||||
fileName: asset.fileName ?? asset.file.name ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
if (shouldTreatAsFileUri(asset.uri)) {
|
||||
return {
|
||||
source: { kind: "file_uri", uri: asset.uri },
|
||||
mimeType: asset.mimeType ?? null,
|
||||
fileName: asset.fileName ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
source: { kind: "blob", blob: await blobFromUri(asset.uri) },
|
||||
mimeType: asset.mimeType ?? null,
|
||||
fileName: asset.fileName ?? null,
|
||||
};
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeTauriDialogSelection(selection: string | string[] | null): string[] {
|
||||
if (!selection) {
|
||||
return [];
|
||||
}
|
||||
return Array.isArray(selection) ? selection : [selection];
|
||||
}
|
||||
|
||||
export async function openImagePathsWithTauriDialog(): Promise<string[]> {
|
||||
const tauri = getTauri();
|
||||
const options = {
|
||||
directory: false,
|
||||
multiple: true,
|
||||
filters: [{ name: "Images", extensions: IMAGE_FILE_EXTENSIONS }],
|
||||
title: "Attach images",
|
||||
};
|
||||
|
||||
const dialogOpen = tauri?.dialog?.open;
|
||||
if (typeof dialogOpen === "function") {
|
||||
return normalizeTauriDialogSelection(await dialogOpen(options));
|
||||
}
|
||||
|
||||
const invoke = tauri?.core?.invoke;
|
||||
if (typeof invoke !== "function") {
|
||||
throw new Error("Tauri dialog API is not available.");
|
||||
}
|
||||
|
||||
const result = await invoke("plugin:dialog|open", { options });
|
||||
return normalizeTauriDialogSelection(
|
||||
Array.isArray(result) || typeof result === "string" || result === null ? result : null
|
||||
);
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
shouldClearAgentAttention,
|
||||
type AgentAttentionClearTrigger,
|
||||
} from "@/utils/agent-attention";
|
||||
import { getIsAppActivelyVisible } from "@/utils/app-visibility";
|
||||
|
||||
type AttentionReason = "finished" | "error" | "permission" | null | undefined;
|
||||
|
||||
@@ -23,20 +24,6 @@ interface AgentAttentionClearController {
|
||||
clearOnAgentBlur: () => void;
|
||||
}
|
||||
|
||||
function getIsAppVisible(): boolean {
|
||||
const isAppStateActive = AppState.currentState === "active";
|
||||
if (Platform.OS !== "web") {
|
||||
return isAppStateActive;
|
||||
}
|
||||
const documentVisible =
|
||||
typeof document === "undefined" || document.visibilityState === "visible";
|
||||
const windowFocused =
|
||||
typeof document === "undefined" ||
|
||||
typeof document.hasFocus !== "function" ||
|
||||
document.hasFocus();
|
||||
return isAppStateActive && documentVisible && windowFocused;
|
||||
}
|
||||
|
||||
export function useAgentAttentionClear({
|
||||
agentId,
|
||||
client,
|
||||
@@ -45,12 +32,14 @@ export function useAgentAttentionClear({
|
||||
attentionReason,
|
||||
isScreenFocused,
|
||||
}: UseAgentAttentionClearParams): AgentAttentionClearController {
|
||||
const [isAppVisible, setIsAppVisible] = useState<boolean>(() => getIsAppVisible());
|
||||
const [isAppVisible, setIsAppVisible] = useState<boolean>(() =>
|
||||
getIsAppActivelyVisible()
|
||||
);
|
||||
const deferredFocusEntryClearRef = useRef(false);
|
||||
const prevRequiresAttentionRef = useRef(Boolean(requiresAttention));
|
||||
const prevActivelyViewedRef = useRef(isScreenFocused && getIsAppVisible());
|
||||
const prevActivelyViewedRef = useRef(isScreenFocused && getIsAppActivelyVisible());
|
||||
const prevScreenFocusedRef = useRef(false);
|
||||
const prevAppVisibleRef = useRef(getIsAppVisible());
|
||||
const prevAppVisibleRef = useRef(getIsAppActivelyVisible());
|
||||
|
||||
const clearAttention = useCallback(
|
||||
(trigger: AgentAttentionClearTrigger) => {
|
||||
@@ -78,7 +67,7 @@ export function useAgentAttentionClear({
|
||||
|
||||
useEffect(() => {
|
||||
const updateVisibility = () => {
|
||||
setIsAppVisible(getIsAppVisible());
|
||||
setIsAppVisible(getIsAppActivelyVisible());
|
||||
};
|
||||
|
||||
const appStateSubscription = AppState.addEventListener(
|
||||
|
||||
1
packages/app/src/hooks/use-audio-player.d.ts
vendored
1
packages/app/src/hooks/use-audio-player.d.ts
vendored
@@ -1 +0,0 @@
|
||||
export * from "./use-audio-player.native";
|
||||
@@ -1,366 +0,0 @@
|
||||
import { useState, useRef } from "react";
|
||||
import {
|
||||
initialize,
|
||||
playPCMData,
|
||||
stopPlayback,
|
||||
pausePlayback,
|
||||
resumePlayback,
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
|
||||
interface QueuedAudio {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resample PCM16 audio between sample rates.
|
||||
* Speechmatics expects 16kHz.
|
||||
*/
|
||||
function resamplePcm16(pcm: Uint8Array, fromRate: number, toRate: number): Uint8Array {
|
||||
if (fromRate === toRate) {
|
||||
return pcm;
|
||||
}
|
||||
|
||||
const inputSamples = Math.floor(pcm.length / 2);
|
||||
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
|
||||
const out = new Uint8Array(outputSamples * 2);
|
||||
|
||||
const ratio = fromRate / toRate;
|
||||
|
||||
const readInt16 = (sampleIndex: number): number => {
|
||||
const i = sampleIndex * 2;
|
||||
if (i + 1 >= pcm.length) {
|
||||
return 0;
|
||||
}
|
||||
const lo = pcm[i]!;
|
||||
const hi = pcm[i + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value = value - 0x10000;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const writeInt16 = (sampleIndex: number, value: number): void => {
|
||||
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
|
||||
const i = sampleIndex * 2;
|
||||
out[i] = clamped & 0xff;
|
||||
out[i + 1] = (clamped >> 8) & 0xff;
|
||||
};
|
||||
|
||||
for (let i = 0; i < outputSamples; i++) {
|
||||
const srcPos = i * ratio;
|
||||
const i0 = Math.floor(srcPos);
|
||||
const frac = srcPos - i0;
|
||||
const s0 = readInt16(i0);
|
||||
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
|
||||
writeInt16(i, s0 + (s1 - s0) * frac);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
}
|
||||
|
||||
export interface AudioPlayerOptions {
|
||||
isDetecting?: () => boolean;
|
||||
isSpeaking?: () => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio playback using Speechmatics two-way audio with echo cancellation
|
||||
*/
|
||||
export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
const [isPlaying, setIsPlaying] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
const queueRef = useRef<QueuedAudio[]>([]);
|
||||
const suppressedQueueRef = useRef<QueuedAudio[]>([]);
|
||||
const isProcessingQueueRef = useRef(false);
|
||||
const activePlaybackRef = useRef<{
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
} | null>(null);
|
||||
const playbackTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
|
||||
async function play(audioData: Blob): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Check if we should suppress playback due to voice detection/speaking
|
||||
const shouldSuppress =
|
||||
(options?.isDetecting && options.isDetecting()) ||
|
||||
(options?.isSpeaking && options.isSpeaking());
|
||||
|
||||
if (shouldSuppress) {
|
||||
console.log("[AudioPlayer] Suppressing playback - voice detection/speaking active");
|
||||
// Add to suppressed queue instead
|
||||
suppressedQueueRef.current.push({ audioData, resolve, reject });
|
||||
|
||||
// Start checking for when flags clear
|
||||
startCheckingForClearFlags();
|
||||
return;
|
||||
}
|
||||
|
||||
// Add to queue with its promise handlers
|
||||
queueRef.current.push({ audioData, resolve, reject });
|
||||
|
||||
// Start processing queue if not already processing
|
||||
if (!isProcessingQueueRef.current) {
|
||||
processQueue();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function startCheckingForClearFlags(): void {
|
||||
// Already checking
|
||||
if (checkIntervalRef.current !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("[AudioPlayer] Starting to check for clear flags");
|
||||
|
||||
checkIntervalRef.current = setInterval(() => {
|
||||
const isStillBlocked =
|
||||
(options?.isDetecting && options.isDetecting()) ||
|
||||
(options?.isSpeaking && options.isSpeaking());
|
||||
|
||||
if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
|
||||
console.log("[AudioPlayer] Flags cleared - moving suppressed queue to main queue");
|
||||
|
||||
// Move all suppressed items to main queue
|
||||
const suppressedItems = [...suppressedQueueRef.current];
|
||||
suppressedQueueRef.current = [];
|
||||
|
||||
// Add to front of main queue (they were waiting)
|
||||
queueRef.current = [...suppressedItems, ...queueRef.current];
|
||||
|
||||
// Stop checking
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
|
||||
// Start processing if not already
|
||||
if (!isProcessingQueueRef.current) {
|
||||
processQueue();
|
||||
}
|
||||
} else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
|
||||
// No more suppressed items and flags are clear - stop checking
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
}
|
||||
}, 100); // Check every 100ms
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = true;
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
// Before processing each item, check if flags became active
|
||||
const shouldSuppress =
|
||||
(options?.isDetecting && options.isDetecting()) ||
|
||||
(options?.isSpeaking && options.isSpeaking());
|
||||
|
||||
if (shouldSuppress) {
|
||||
console.log("[AudioPlayer] Flags became active during processing - moving remaining queue to suppressed");
|
||||
// Move remaining queue to suppressed
|
||||
suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
|
||||
queueRef.current = [];
|
||||
startCheckingForClearFlags();
|
||||
break;
|
||||
}
|
||||
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
async function processNextInQueue(): Promise<void> {
|
||||
if (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
} else {
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audioData: Blob): Promise<number> {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
activePlaybackRef.current = { resolve, reject };
|
||||
try {
|
||||
console.log(
|
||||
`[AudioPlayer] Playing audio (${audioData.size} bytes, type: ${audioData.type})`
|
||||
);
|
||||
|
||||
// Initialize audio if not already initialized
|
||||
if (!audioInitialized) {
|
||||
console.log("[AudioPlayer] Initializing audio...");
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log(
|
||||
"[AudioPlayer] ✅ Initialized (Speechmatics two-way audio)"
|
||||
);
|
||||
}
|
||||
|
||||
// Workaround: Resume playback before playing new audio to ensure the audio engine is ready
|
||||
// This fixes the issue where playback doesn't work after calling stopPlayback()
|
||||
console.log("[AudioPlayer] Resuming playback engine...");
|
||||
resumePlayback();
|
||||
|
||||
// Get PCM data from blob (server sends PCM16)
|
||||
const arrayBuffer = await audioData.arrayBuffer();
|
||||
const pcm = new Uint8Array(arrayBuffer);
|
||||
|
||||
const inputRate = parsePcmSampleRate(audioData.type || "") ?? 24000;
|
||||
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
|
||||
|
||||
// Calculate total duration
|
||||
const samples = pcm16k.length / 2; // 16-bit = 2 bytes per sample
|
||||
const durationSec = samples / 16000; // 16kHz sample rate
|
||||
|
||||
const audioSizeKb = (pcm16k.length / 1024).toFixed(2);
|
||||
console.log(
|
||||
"[AudioPlayer] 🔊 Playing audio:",
|
||||
audioSizeKb,
|
||||
"KB, duration:",
|
||||
durationSec.toFixed(2),
|
||||
"s"
|
||||
);
|
||||
|
||||
setIsPlaying(true);
|
||||
|
||||
// Play entire PCM data at once through Speechmatics
|
||||
playPCMData(pcm16k);
|
||||
|
||||
// Clear any existing timeout
|
||||
if (playbackTimeoutRef.current) {
|
||||
clearTimeout(playbackTimeoutRef.current);
|
||||
}
|
||||
|
||||
// Wait for playback to finish (estimate based on duration)
|
||||
playbackTimeoutRef.current = setTimeout(() => {
|
||||
console.log("[AudioPlayer] ✅ Playback finished");
|
||||
setIsPlaying(false);
|
||||
playbackTimeoutRef.current = null;
|
||||
activePlaybackRef.current = null;
|
||||
resolve(durationSec);
|
||||
}, durationSec * 1000);
|
||||
} catch (error) {
|
||||
console.error("[AudioPlayer] Error playing audio:", error);
|
||||
|
||||
// Clear timeout on error
|
||||
if (playbackTimeoutRef.current) {
|
||||
clearTimeout(playbackTimeoutRef.current);
|
||||
playbackTimeoutRef.current = null;
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
activePlaybackRef.current = null;
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (isPlaying) {
|
||||
console.log("[AudioPlayer] 🛑 Stopping playback (interrupted)");
|
||||
|
||||
// Stop native playback
|
||||
stopPlayback();
|
||||
|
||||
// Clear playback timeout
|
||||
if (playbackTimeoutRef.current) {
|
||||
clearTimeout(playbackTimeoutRef.current);
|
||||
playbackTimeoutRef.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
setIsPlaying(false);
|
||||
|
||||
// Reject the currently playing promise, if any.
|
||||
if (activePlaybackRef.current) {
|
||||
activePlaybackRef.current.reject(new Error("Playback stopped"));
|
||||
activePlaybackRef.current = null;
|
||||
}
|
||||
|
||||
// Reject all pending promises in the main queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error("Playback stopped"));
|
||||
}
|
||||
|
||||
// Reject all pending promises in the suppressed queue
|
||||
while (suppressedQueueRef.current.length > 0) {
|
||||
const item = suppressedQueueRef.current.shift()!;
|
||||
item.reject(new Error("Playback stopped"));
|
||||
}
|
||||
|
||||
// Clear check interval
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
}
|
||||
|
||||
function clearQueue(): void {
|
||||
// Reject all pending promises in the main queue
|
||||
while (queueRef.current.length > 0) {
|
||||
const item = queueRef.current.shift()!;
|
||||
item.reject(new Error("Queue cleared"));
|
||||
}
|
||||
|
||||
// Reject all pending promises in the suppressed queue
|
||||
while (suppressedQueueRef.current.length > 0) {
|
||||
const item = suppressedQueueRef.current.shift()!;
|
||||
item.reject(new Error("Queue cleared"));
|
||||
}
|
||||
|
||||
// Clear check interval
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
play,
|
||||
stop,
|
||||
isPlaying: () => isPlaying,
|
||||
clearQueue,
|
||||
warmup: async () => {
|
||||
if (!audioInitialized) {
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
}
|
||||
// Ensure playback engine isn't suspended after a previous stop.
|
||||
resumePlayback();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,275 +0,0 @@
|
||||
import { useMemo, useRef, useState } from "react";
|
||||
|
||||
export interface AudioPlayerOptions {
|
||||
isDetecting?: () => boolean;
|
||||
isSpeaking?: () => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Web audio player for server-provided audio chunks.
|
||||
*
|
||||
* Supports:
|
||||
* - `audio/pcm` (assumed PCM16 LE, mono, default 24kHz unless `rate=` is present)
|
||||
* - formats supported by `AudioContext.decodeAudioData` (e.g. mp3)
|
||||
*/
|
||||
export function useAudioPlayer(options?: AudioPlayerOptions) {
|
||||
const [isPlayingState, setIsPlayingState] = useState(false);
|
||||
|
||||
type QueuedAudio = {
|
||||
audioData: Blob;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
};
|
||||
|
||||
const audioContextRef = useRef<AudioContext | null>(null);
|
||||
const queueRef = useRef<QueuedAudio[]>([]);
|
||||
const suppressedQueueRef = useRef<QueuedAudio[]>([]);
|
||||
const isProcessingQueueRef = useRef(false);
|
||||
const checkIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||
const activePlaybackRef = useRef<{
|
||||
source: AudioBufferSourceNode;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
settled: boolean;
|
||||
} | null>(null);
|
||||
|
||||
const decodeAudioData = async (context: AudioContext, buffer: ArrayBuffer): Promise<AudioBuffer> => {
|
||||
const maybePromise = context.decodeAudioData(buffer);
|
||||
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
|
||||
return maybePromise as Promise<AudioBuffer>;
|
||||
}
|
||||
return await new Promise<AudioBuffer>((resolve, reject) => {
|
||||
context.decodeAudioData(buffer, resolve, reject);
|
||||
});
|
||||
};
|
||||
|
||||
const parsePcmSampleRate = (mimeType: string): number | null => {
|
||||
const match = /rate=(\\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
};
|
||||
|
||||
const pcm16LeToAudioBuffer = (context: AudioContext, bytes: Uint8Array, sampleRate: number): AudioBuffer => {
|
||||
const sampleCount = Math.floor(bytes.length / 2);
|
||||
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
|
||||
const channel = audioBuffer.getChannelData(0);
|
||||
for (let i = 0; i < sampleCount; i++) {
|
||||
const lo = bytes[i * 2]!;
|
||||
const hi = bytes[i * 2 + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value = value - 0x10000;
|
||||
}
|
||||
channel[i] = value / 0x8000;
|
||||
}
|
||||
return audioBuffer;
|
||||
};
|
||||
|
||||
const ensureContext = async (): Promise<AudioContext> => {
|
||||
if (audioContextRef.current) {
|
||||
if (audioContextRef.current.state === "suspended") {
|
||||
try {
|
||||
await audioContextRef.current.resume();
|
||||
} catch {
|
||||
// Best effort. If this fails due to autoplay policies, a later user gesture
|
||||
// (or explicit warmup call) can unlock it.
|
||||
}
|
||||
}
|
||||
return audioContextRef.current;
|
||||
}
|
||||
const context = new AudioContext();
|
||||
if (context.state === "suspended") {
|
||||
try {
|
||||
await context.resume();
|
||||
} catch {
|
||||
// See note above.
|
||||
}
|
||||
}
|
||||
audioContextRef.current = context;
|
||||
return context;
|
||||
};
|
||||
|
||||
const shouldSuppressPlayback = (): boolean => {
|
||||
return Boolean(options?.isDetecting?.()) || Boolean(options?.isSpeaking?.());
|
||||
};
|
||||
|
||||
const startCheckingForClearFlags = (): void => {
|
||||
if (checkIntervalRef.current !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
checkIntervalRef.current = setInterval(() => {
|
||||
const isStillBlocked = shouldSuppressPlayback();
|
||||
if (!isStillBlocked && suppressedQueueRef.current.length > 0) {
|
||||
const suppressedItems = [...suppressedQueueRef.current];
|
||||
suppressedQueueRef.current = [];
|
||||
queueRef.current = [...suppressedItems, ...queueRef.current];
|
||||
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
|
||||
if (!isProcessingQueueRef.current) {
|
||||
void processQueue();
|
||||
}
|
||||
} else if (!isStillBlocked && suppressedQueueRef.current.length === 0) {
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
}
|
||||
}, 100);
|
||||
};
|
||||
|
||||
const playAudio = async (audioData: Blob): Promise<number> => {
|
||||
const context = await ensureContext();
|
||||
const arrayBuffer = await audioData.arrayBuffer();
|
||||
|
||||
let audioBuffer: AudioBuffer;
|
||||
const type = (audioData.type || "").toLowerCase();
|
||||
|
||||
if (type.startsWith("audio/pcm")) {
|
||||
const sampleRate = parsePcmSampleRate(type) ?? 24000;
|
||||
audioBuffer = pcm16LeToAudioBuffer(context, new Uint8Array(arrayBuffer), sampleRate);
|
||||
} else {
|
||||
audioBuffer = await decodeAudioData(context, arrayBuffer);
|
||||
}
|
||||
|
||||
const durationSec = audioBuffer.duration;
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(context.destination);
|
||||
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
activePlaybackRef.current = { source, resolve, reject, settled: false };
|
||||
setIsPlayingState(true);
|
||||
|
||||
const settleOnce = (fn: () => void) => {
|
||||
const active = activePlaybackRef.current;
|
||||
if (!active || active.source !== source || active.settled) {
|
||||
return;
|
||||
}
|
||||
active.settled = true;
|
||||
activePlaybackRef.current = null;
|
||||
setIsPlayingState(false);
|
||||
fn();
|
||||
};
|
||||
|
||||
source.onended = () => {
|
||||
settleOnce(() => resolve(durationSec));
|
||||
};
|
||||
|
||||
try {
|
||||
source.start();
|
||||
} catch (e) {
|
||||
settleOnce(() => reject(e instanceof Error ? e : new Error(String(e))));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const processQueue = async (): Promise<void> => {
|
||||
if (isProcessingQueueRef.current || queueRef.current.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = true;
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
if (shouldSuppressPlayback()) {
|
||||
suppressedQueueRef.current = [...queueRef.current, ...suppressedQueueRef.current];
|
||||
queueRef.current = [];
|
||||
startCheckingForClearFlags();
|
||||
break;
|
||||
}
|
||||
|
||||
const item = queueRef.current.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audioData);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error as Error);
|
||||
}
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
};
|
||||
|
||||
const play = async (audioData: Blob): Promise<number> => {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
if (shouldSuppressPlayback()) {
|
||||
suppressedQueueRef.current.push({ audioData, resolve, reject });
|
||||
startCheckingForClearFlags();
|
||||
return;
|
||||
}
|
||||
|
||||
queueRef.current.push({ audioData, resolve, reject });
|
||||
if (!isProcessingQueueRef.current) {
|
||||
void processQueue();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const stop = (): void => {
|
||||
// Stop currently playing audio (and reject its promise)
|
||||
if (activePlaybackRef.current) {
|
||||
const active = activePlaybackRef.current;
|
||||
activePlaybackRef.current = null;
|
||||
try {
|
||||
active.source.onended = null;
|
||||
active.source.stop();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (!active.settled) {
|
||||
active.settled = true;
|
||||
active.reject(new Error("Playback stopped"));
|
||||
}
|
||||
}
|
||||
|
||||
setIsPlayingState(false);
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
queueRef.current.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
while (suppressedQueueRef.current.length > 0) {
|
||||
suppressedQueueRef.current.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false;
|
||||
};
|
||||
|
||||
const clearQueue = (): void => {
|
||||
while (queueRef.current.length > 0) {
|
||||
queueRef.current.shift()!.reject(new Error("Queue cleared"));
|
||||
}
|
||||
while (suppressedQueueRef.current.length > 0) {
|
||||
suppressedQueueRef.current.shift()!.reject(new Error("Queue cleared"));
|
||||
}
|
||||
if (checkIntervalRef.current !== null) {
|
||||
clearInterval(checkIntervalRef.current);
|
||||
checkIntervalRef.current = null;
|
||||
}
|
||||
};
|
||||
|
||||
return useMemo(
|
||||
() => ({
|
||||
play,
|
||||
stop,
|
||||
isPlaying: () => isPlayingState,
|
||||
clearQueue,
|
||||
warmup: async () => {
|
||||
await ensureContext();
|
||||
},
|
||||
}),
|
||||
[isPlayingState]
|
||||
);
|
||||
}
|
||||
@@ -18,41 +18,27 @@ export interface AudioCaptureConfig {
|
||||
*/
|
||||
async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
|
||||
try {
|
||||
console.log('[AudioRecorder] Searching for recording file created at:', createdAt.toISOString());
|
||||
|
||||
const audioDir = new Directory(Paths.cache, 'Audio');
|
||||
console.log('[AudioRecorder] Audio cache directory URI:', audioDir.uri);
|
||||
console.log('[AudioRecorder] Directory exists:', audioDir.exists);
|
||||
|
||||
if (!audioDir.exists) {
|
||||
console.log('[AudioRecorder] Audio cache directory does not exist');
|
||||
return null;
|
||||
}
|
||||
|
||||
const files = audioDir.list();
|
||||
console.log('[AudioRecorder] Found files in Audio cache:', files.length);
|
||||
|
||||
if (!files.length) {
|
||||
console.log('[AudioRecorder] No files found in Audio cache directory');
|
||||
return null;
|
||||
}
|
||||
|
||||
const validFiles = files
|
||||
.map(file => {
|
||||
const info = file.info();
|
||||
console.log('[AudioRecorder] File info:', {
|
||||
uri: info.uri,
|
||||
size: info.size,
|
||||
creationTime: info.creationTime ? new Date(info.creationTime).toISOString() : null,
|
||||
});
|
||||
return info;
|
||||
})
|
||||
.filter(f => f.size && f.size > 0);
|
||||
|
||||
console.log('[AudioRecorder] Valid files (size > 0):', validFiles.length);
|
||||
|
||||
if (validFiles.length === 0) {
|
||||
console.log('[AudioRecorder] No valid files found (all are zero-byte)');
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -62,10 +48,6 @@ async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
|
||||
for (const file of validFiles) {
|
||||
if (!file.creationTime || !file.uri) continue;
|
||||
const diff = Math.abs(file.creationTime - createdAt.getTime());
|
||||
console.log('[AudioRecorder] Time diff for file:', {
|
||||
uri: file.uri,
|
||||
diffMs: diff,
|
||||
});
|
||||
if (diff < minDiff) {
|
||||
closest = file;
|
||||
minDiff = diff;
|
||||
@@ -74,15 +56,9 @@ async function getActualRecordingUri(createdAt: Date): Promise<string | null> {
|
||||
|
||||
if (closest) {
|
||||
const resultUri = closest.uri?.slice(0, -1) ?? null;
|
||||
console.log('[AudioRecorder] Found closest file:', {
|
||||
uri: resultUri,
|
||||
size: closest.size,
|
||||
timeDiffMs: minDiff,
|
||||
});
|
||||
return resultUri;
|
||||
}
|
||||
|
||||
console.log('[AudioRecorder] No closest file found');
|
||||
return null;
|
||||
} catch (e) {
|
||||
console.error('[AudioRecorder] Error finding actual recording file:', e);
|
||||
@@ -208,7 +184,6 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
attemptGuardRef.current.assertCurrent(attemptId);
|
||||
|
||||
// Request microphone permissions
|
||||
console.log('[AudioRecorder] Requesting recording permissions...');
|
||||
const permissionResponse = await requestRecordingPermissionsAsync();
|
||||
attemptGuardRef.current.assertCurrent(attemptId);
|
||||
|
||||
@@ -217,38 +192,26 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
}
|
||||
|
||||
// Configure audio mode for recording
|
||||
console.log('[AudioRecorder] Configuring audio mode...');
|
||||
await setAudioModeAsync({
|
||||
playsInSilentMode: true,
|
||||
allowsRecording: true,
|
||||
});
|
||||
attemptGuardRef.current.assertCurrent(attemptId);
|
||||
|
||||
console.log('[AudioRecorder] Starting recording with options:', {
|
||||
sampleRate: recordingOptions.sampleRate,
|
||||
numberOfChannels: recordingOptions.numberOfChannels,
|
||||
bitRate: recordingOptions.bitRate,
|
||||
});
|
||||
|
||||
const startTime = new Date();
|
||||
setRecordingStartTime(startTime);
|
||||
attemptGuardRef.current.assertCurrent(attemptId);
|
||||
|
||||
// Prepare the recorder before recording (required step)
|
||||
console.log('[AudioRecorder] Preparing recorder...');
|
||||
await recorder.prepareToRecordAsync();
|
||||
attemptGuardRef.current.assertCurrent(attemptId);
|
||||
|
||||
console.log('[AudioRecorder] Starting recording...');
|
||||
await recorder.record();
|
||||
attemptGuardRef.current.assertCurrent(attemptId);
|
||||
|
||||
console.log('[AudioRecorder] Recording started at:', startTime.toISOString());
|
||||
console.log('[AudioRecorder] Recorder isRecording:', recorder.isRecording);
|
||||
} catch (error: any) {
|
||||
setRecordingStartTime(null);
|
||||
if (error instanceof AttemptCancelledError) {
|
||||
console.log('[AudioRecorder] Recording start cancelled.');
|
||||
return;
|
||||
}
|
||||
if (error?.message !== 'Recording cancelled') {
|
||||
@@ -278,15 +241,12 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
|
||||
// Get URI from recorder
|
||||
let uri = recorder.uri;
|
||||
console.log('[AudioRecorder] Initial URI from recorder:', uri);
|
||||
|
||||
// Workaround for Expo SDK 54 Android bug - find actual recording file
|
||||
if (recordingStartTime && (!uri || uri === '')) {
|
||||
console.log('[AudioRecorder] Using workaround to find actual recording file...');
|
||||
const actualUri = await getActualRecordingUri(recordingStartTime);
|
||||
if (actualUri) {
|
||||
uri = actualUri;
|
||||
console.log('[AudioRecorder] Found actual recording URI:', uri);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -299,7 +259,6 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
// Get file info
|
||||
const file = new File(uri);
|
||||
const exists = file.exists;
|
||||
console.log('[AudioRecorder] File exists:', exists);
|
||||
|
||||
if (!exists) {
|
||||
setRecordingStartTime(null);
|
||||
@@ -309,11 +268,6 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
// Convert URI to Blob
|
||||
const audioBlob = await uriToBlob(uri);
|
||||
|
||||
console.log('[AudioRecorder] Recording converted to blob:', {
|
||||
size: audioBlob.size,
|
||||
type: audioBlob.type,
|
||||
});
|
||||
|
||||
// Clean up the temporary file
|
||||
file.delete();
|
||||
|
||||
|
||||
@@ -163,16 +163,6 @@ export function useAudioRecorder(config?: AudioCaptureConfig) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
|
||||
console.log("[AudioRecorder][Web] Microphone preflight", {
|
||||
secureContext,
|
||||
currentOrigin,
|
||||
isTauri,
|
||||
hasMediaDevices:
|
||||
typeof navigator !== "undefined" &&
|
||||
!!navigator.mediaDevices &&
|
||||
typeof navigator.mediaDevices.getUserMedia === "function",
|
||||
});
|
||||
|
||||
if (!secureContext && !isTauri) {
|
||||
throw new Error(
|
||||
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`
|
||||
|
||||
@@ -1,44 +1,63 @@
|
||||
import { useCallback, useEffect, useRef } from "react";
|
||||
import { Buffer } from "buffer";
|
||||
import { useState } from "react";
|
||||
|
||||
import { useSpeechmaticsAudio } from "@/hooks/use-speechmatics-audio";
|
||||
import { createAudioEngine } from "@/voice/audio-engine";
|
||||
|
||||
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
|
||||
|
||||
export function useDictationAudioSource(config: DictationAudioSourceConfig): DictationAudioSource {
|
||||
const onPcmSegmentRef = useRef(config.onPcmSegment);
|
||||
const onErrorRef = useRef(config.onError);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const engineRef = useRef<ReturnType<typeof createAudioEngine> | null>(null);
|
||||
|
||||
const getOrCreateEngine = useCallback(() => {
|
||||
if (engineRef.current) {
|
||||
return engineRef.current;
|
||||
}
|
||||
|
||||
engineRef.current = createAudioEngine({
|
||||
onCaptureData: (pcm) => {
|
||||
onPcmSegmentRef.current(Buffer.from(pcm).toString("base64"));
|
||||
},
|
||||
onVolumeLevel: (level) => {
|
||||
setVolume(level);
|
||||
},
|
||||
onError: (error) => {
|
||||
onErrorRef.current?.(error);
|
||||
},
|
||||
});
|
||||
return engineRef.current;
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
onPcmSegmentRef.current = config.onPcmSegment;
|
||||
onErrorRef.current = config.onError;
|
||||
}, [config.onPcmSegment, config.onError]);
|
||||
|
||||
const speechmatics = useSpeechmaticsAudio({
|
||||
enableContinuousStreaming: true,
|
||||
onAudioSegment: ({ audioData }) => {
|
||||
onPcmSegmentRef.current(audioData);
|
||||
},
|
||||
onError: (err) => {
|
||||
onErrorRef.current?.(err);
|
||||
},
|
||||
volumeThreshold: 0.3,
|
||||
silenceDuration: 2000,
|
||||
speechConfirmationDuration: 300,
|
||||
detectionGracePeriod: 200,
|
||||
});
|
||||
|
||||
const start = useCallback(async () => {
|
||||
await speechmatics.start();
|
||||
}, [speechmatics]);
|
||||
const engine = getOrCreateEngine();
|
||||
await engine.initialize();
|
||||
await engine.startCapture();
|
||||
}, [getOrCreateEngine]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
await speechmatics.stop();
|
||||
}, [speechmatics]);
|
||||
await engineRef.current?.stopCapture();
|
||||
setVolume(0);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
const engine = engineRef.current;
|
||||
engineRef.current = null;
|
||||
void engine?.destroy().catch(() => undefined);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
volume: speechmatics.volume,
|
||||
volume,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { parsePcm16Wav } from "@/utils/pcm16-wav";
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
|
||||
import type { DictationAudioSource, DictationAudioSourceConfig } from "./use-dictation-audio-source.types";
|
||||
@@ -65,84 +66,6 @@ const int16ToFloat32 = (input: Int16Array): Float32Array => {
|
||||
return out;
|
||||
};
|
||||
|
||||
type Pcm16Wav = {
|
||||
sampleRate: number;
|
||||
samples: Int16Array;
|
||||
};
|
||||
|
||||
const parsePcm16Wav = (buffer: ArrayBuffer): Pcm16Wav | null => {
|
||||
if (buffer.byteLength < 44) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const view = new DataView(buffer);
|
||||
const readAscii = (offset: number, length: number): string => {
|
||||
let out = "";
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
out += String.fromCharCode(view.getUint8(offset + i));
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
if (readAscii(0, 4) !== "RIFF" || readAscii(8, 4) !== "WAVE") {
|
||||
return null;
|
||||
}
|
||||
|
||||
let offset = 12;
|
||||
let channels = 0;
|
||||
let sampleRate = 0;
|
||||
let bitsPerSample = 0;
|
||||
let dataOffset = 0;
|
||||
let dataSize = 0;
|
||||
|
||||
while (offset + 8 <= buffer.byteLength) {
|
||||
const chunkId = readAscii(offset, 4);
|
||||
const chunkSize = view.getUint32(offset + 4, true);
|
||||
const chunkDataOffset = offset + 8;
|
||||
if (chunkDataOffset + chunkSize > buffer.byteLength) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (chunkId === "fmt " && chunkSize >= 16) {
|
||||
const audioFormat = view.getUint16(chunkDataOffset, true);
|
||||
channels = view.getUint16(chunkDataOffset + 2, true);
|
||||
sampleRate = view.getUint32(chunkDataOffset + 4, true);
|
||||
bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
|
||||
if (audioFormat !== 1) {
|
||||
return null;
|
||||
}
|
||||
} else if (chunkId === "data") {
|
||||
dataOffset = chunkDataOffset;
|
||||
dataSize = chunkSize;
|
||||
break;
|
||||
}
|
||||
|
||||
offset = chunkDataOffset + chunkSize + (chunkSize % 2);
|
||||
}
|
||||
|
||||
if (!dataOffset || !dataSize || sampleRate <= 0 || bitsPerSample !== 16 || channels <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sampleCount = Math.floor(dataSize / 2);
|
||||
const interleaved = new Int16Array(buffer, dataOffset, sampleCount);
|
||||
|
||||
if (channels === 1) {
|
||||
return { sampleRate, samples: new Int16Array(interleaved) };
|
||||
}
|
||||
|
||||
const frameCount = Math.floor(interleaved.length / channels);
|
||||
const mono = new Int16Array(frameCount);
|
||||
for (let frame = 0; frame < frameCount; frame += 1) {
|
||||
let sum = 0;
|
||||
for (let ch = 0; ch < channels; ch += 1) {
|
||||
sum += interleaved[frame * channels + ch] ?? 0;
|
||||
}
|
||||
mono[frame] = Math.round(sum / channels);
|
||||
}
|
||||
return { sampleRate, samples: mono };
|
||||
};
|
||||
|
||||
const int16ToBase64 = (pcm: Int16Array): string => {
|
||||
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
||||
let binary = "";
|
||||
@@ -240,15 +163,6 @@ export function useDictationAudioSource(config: DictationAudioSourceConfig): Dic
|
||||
if (missingNavigator) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
console.log("[DictationAudio][Web] Microphone preflight", {
|
||||
secureContext,
|
||||
currentOrigin,
|
||||
isTauri,
|
||||
hasMediaDevices:
|
||||
typeof navigator !== "undefined" &&
|
||||
!!navigator.mediaDevices &&
|
||||
typeof navigator.mediaDevices.getUserMedia === "function",
|
||||
});
|
||||
if (!secureContext && !isTauri) {
|
||||
throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
|
||||
}
|
||||
|
||||
@@ -13,18 +13,6 @@ interface UseExplorerOpenGestureParams {
|
||||
onOpen: () => void;
|
||||
}
|
||||
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
|
||||
|
||||
function logExplorerOpenGesture(
|
||||
event: string,
|
||||
details: Record<string, unknown>
|
||||
): void {
|
||||
if (!IS_DEV) {
|
||||
return;
|
||||
}
|
||||
console.log(`[ExplorerOpenGesture] ${event}`, details);
|
||||
}
|
||||
|
||||
export function useExplorerOpenGesture({
|
||||
enabled,
|
||||
onOpen,
|
||||
@@ -82,7 +70,6 @@ export function useExplorerOpenGesture({
|
||||
})
|
||||
.onStart(() => {
|
||||
isGesturing.value = true;
|
||||
runOnJS(logExplorerOpenGesture)("start", { enabled });
|
||||
})
|
||||
.onUpdate((event) => {
|
||||
// Right sidebar: start from closed position (+windowWidth) and move towards 0.
|
||||
@@ -103,15 +90,6 @@ export function useExplorerOpenGesture({
|
||||
const shouldOpenByPosition = translateX.value < (windowWidth * 2) / 3;
|
||||
const shouldOpenByVelocity = event.velocityX < -500;
|
||||
const shouldOpen = shouldOpenByPosition || shouldOpenByVelocity;
|
||||
runOnJS(logExplorerOpenGesture)("end", {
|
||||
translationX: event.translationX,
|
||||
velocityX: event.velocityX,
|
||||
panelTranslateX: translateX.value,
|
||||
windowWidth,
|
||||
shouldOpenByPosition,
|
||||
shouldOpenByVelocity,
|
||||
shouldOpen,
|
||||
});
|
||||
if (shouldOpen) {
|
||||
animateToOpen();
|
||||
runOnJS(onOpen)();
|
||||
|
||||
@@ -1,11 +1,16 @@
|
||||
import { useCallback, useRef } from "react";
|
||||
import { Alert } from "react-native";
|
||||
import { Platform } from "react-native";
|
||||
import * as ImagePicker from "expo-image-picker";
|
||||
|
||||
type ImagePickerSuccessResult = ImagePicker.ImagePickerSuccessResult;
|
||||
import { isTauriEnvironment } from "@/utils/tauri";
|
||||
import {
|
||||
normalizePickedImageAssets,
|
||||
openImagePathsWithTauriDialog,
|
||||
type PickedImageAttachmentInput,
|
||||
} from "@/hooks/image-attachment-picker";
|
||||
|
||||
interface UseImageAttachmentPickerResult {
|
||||
pickImages: () => Promise<ImagePickerSuccessResult | null>;
|
||||
pickImages: () => Promise<PickedImageAttachmentInput[] | null>;
|
||||
}
|
||||
|
||||
export function useImageAttachmentPicker(): UseImageAttachmentPickerResult {
|
||||
@@ -37,6 +42,18 @@ export function useImageAttachmentPicker(): UseImageAttachmentPickerResult {
|
||||
isPickingRef.current = true;
|
||||
|
||||
try {
|
||||
if (Platform.OS === "web" && isTauriEnvironment()) {
|
||||
const selectedPaths = await openImagePathsWithTauriDialog();
|
||||
if (selectedPaths.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return selectedPaths.map((path) => ({
|
||||
source: { kind: "file_uri" as const, uri: path },
|
||||
mimeType: null,
|
||||
fileName: null,
|
||||
}));
|
||||
}
|
||||
|
||||
const hasPermission = await ensurePermission();
|
||||
if (!hasPermission) {
|
||||
return null;
|
||||
@@ -44,7 +61,7 @@ export function useImageAttachmentPicker(): UseImageAttachmentPickerResult {
|
||||
|
||||
const pendingResult = await ImagePicker.getPendingResultAsync();
|
||||
if (pendingResult && "canceled" in pendingResult && !pendingResult.canceled) {
|
||||
return pendingResult as ImagePickerSuccessResult;
|
||||
return await normalizePickedImageAssets(pendingResult.assets);
|
||||
}
|
||||
|
||||
const result = await ImagePicker.launchImageLibraryAsync({
|
||||
@@ -57,7 +74,7 @@ export function useImageAttachmentPicker(): UseImageAttachmentPickerResult {
|
||||
return null;
|
||||
}
|
||||
|
||||
return result;
|
||||
return await normalizePickedImageAssets(result.assets);
|
||||
} catch (error) {
|
||||
console.error("[ImageAttachmentPicker] Failed to pick image:", error);
|
||||
Alert.alert("Error", "Failed to select image");
|
||||
|
||||
71
packages/app/src/hooks/use-sidebar-shortcut-model.ts
Normal file
71
packages/app/src/hooks/use-sidebar-shortcut-model.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||
import type { SidebarProjectEntry } from '@/hooks/use-sidebar-workspaces-list'
|
||||
import { useKeyboardShortcutsStore } from '@/stores/keyboard-shortcuts-store'
|
||||
import { buildSidebarShortcutModel } from '@/utils/sidebar-shortcuts'
|
||||
|
||||
export function useSidebarShortcutModel(projects: SidebarProjectEntry[]) {
|
||||
const [collapsedProjectKeys, setCollapsedProjectKeys] = useState<Set<string>>(new Set())
|
||||
const setSidebarShortcutWorkspaceTargets = useKeyboardShortcutsStore(
|
||||
(state) => state.setSidebarShortcutWorkspaceTargets
|
||||
)
|
||||
const setVisibleWorkspaceTargets = useKeyboardShortcutsStore(
|
||||
(state) => state.setVisibleWorkspaceTargets
|
||||
)
|
||||
|
||||
const shortcutModel = useMemo(
|
||||
() =>
|
||||
buildSidebarShortcutModel({
|
||||
projects,
|
||||
collapsedProjectKeys,
|
||||
}),
|
||||
[collapsedProjectKeys, projects]
|
||||
)
|
||||
|
||||
useEffect(() => {
|
||||
setCollapsedProjectKeys((prev) => {
|
||||
const validProjectKeys = new Set(projects.map((project) => project.projectKey))
|
||||
const next = new Set<string>()
|
||||
for (const key of prev) {
|
||||
if (validProjectKeys.has(key)) {
|
||||
next.add(key)
|
||||
}
|
||||
}
|
||||
return next
|
||||
})
|
||||
}, [projects])
|
||||
|
||||
useEffect(() => {
|
||||
setSidebarShortcutWorkspaceTargets(shortcutModel.shortcutTargets)
|
||||
setVisibleWorkspaceTargets(shortcutModel.visibleTargets)
|
||||
}, [
|
||||
setSidebarShortcutWorkspaceTargets,
|
||||
setVisibleWorkspaceTargets,
|
||||
shortcutModel.shortcutTargets,
|
||||
shortcutModel.visibleTargets,
|
||||
])
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
setSidebarShortcutWorkspaceTargets([])
|
||||
setVisibleWorkspaceTargets([])
|
||||
}
|
||||
}, [setSidebarShortcutWorkspaceTargets, setVisibleWorkspaceTargets])
|
||||
|
||||
const toggleProjectCollapsed = useCallback((projectKey: string) => {
|
||||
setCollapsedProjectKeys((prev) => {
|
||||
const next = new Set(prev)
|
||||
if (next.has(projectKey)) {
|
||||
next.delete(projectKey)
|
||||
} else {
|
||||
next.add(projectKey)
|
||||
}
|
||||
return next
|
||||
})
|
||||
}, [])
|
||||
|
||||
return {
|
||||
collapsedProjectKeys,
|
||||
shortcutIndexByWorkspaceKey: shortcutModel.shortcutIndexByWorkspaceKey,
|
||||
toggleProjectCollapsed,
|
||||
}
|
||||
}
|
||||
@@ -267,8 +267,6 @@ export function useSidebarWorkspacesList(options?: {
|
||||
const value = options?.serverId
|
||||
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null
|
||||
}, [options?.serverId])
|
||||
const enabled = options?.enabled ?? true
|
||||
|
||||
const persistedProjectOrder = useSidebarOrderStore((state) =>
|
||||
serverId ? (state.projectOrderByServerId[serverId] ?? EMPTY_ORDER) : EMPTY_ORDER
|
||||
)
|
||||
@@ -345,7 +343,7 @@ export function useSidebarWorkspacesList(options?: {
|
||||
}, [persistedProjectOrder, persistedWorkspaceOrderByScope, projects, serverId])
|
||||
|
||||
const refreshAll = useCallback(() => {
|
||||
if (!isActive || !serverId || connectionStatus !== 'online' || !enabled) {
|
||||
if (!isActive || !serverId || connectionStatus !== 'online') {
|
||||
return
|
||||
}
|
||||
const client = runtime.getClient(serverId)
|
||||
@@ -377,7 +375,7 @@ export function useSidebarWorkspacesList(options?: {
|
||||
// ignore explicit refresh failures; hook keeps existing data
|
||||
}
|
||||
})()
|
||||
}, [connectionStatus, enabled, isActive, runtime, serverId])
|
||||
}, [connectionStatus, isActive, runtime, serverId])
|
||||
|
||||
const isLoading = isActive && Boolean(serverId) && connectionStatus === 'online' && !hasHydratedWorkspaces
|
||||
const isInitialLoad = isLoading && projects.length === 0
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
export * from "./use-speechmatics-audio.native";
|
||||
@@ -1,274 +0,0 @@
|
||||
import { useState, useEffect, useRef, useCallback } from "react";
|
||||
import {
|
||||
initialize,
|
||||
useMicrophonePermissions,
|
||||
toggleRecording,
|
||||
tearDown,
|
||||
useExpoTwoWayAudioEventListener,
|
||||
type MicrophoneDataCallback,
|
||||
type VolumeLevelCallback,
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
|
||||
import { SpeechSegmenter } from "@/voice/speech-segmenter";
|
||||
|
||||
export interface SpeechmaticsAudioConfig {
|
||||
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
/** When true, stream microphone PCM continuously without VAD gating. */
|
||||
enableContinuousStreaming?: boolean;
|
||||
volumeThreshold: number; // Volume threshold for speech detection (0-1)
|
||||
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
|
||||
confirmedDropGracePeriod?: number;
|
||||
silenceDuration: number; // ms of silence before ending segment
|
||||
speechConfirmationDuration: number; // ms of sustained speech before confirming
|
||||
detectionGracePeriod: number; // ms grace period for volume dips during detection
|
||||
}
|
||||
|
||||
export interface SpeechmaticsAudio {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
toggleMute: () => void;
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
isDetecting: boolean;
|
||||
isMuted: boolean;
|
||||
volume: number;
|
||||
segmentDuration: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for audio capture with echo cancellation using Speechmatics expo-two-way-audio
|
||||
*/
|
||||
export function useSpeechmaticsAudio(
|
||||
config: SpeechmaticsAudioConfig
|
||||
): SpeechmaticsAudio {
|
||||
const [microphonePermission, requestMicrophonePermission] =
|
||||
useMicrophonePermissions();
|
||||
const [isActive, setIsActive] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isDetecting, setIsDetecting] = useState(false);
|
||||
const [audioInitialized, setAudioInitialized] = useState(false);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const [isMuted, setIsMuted] = useState(false);
|
||||
const [segmentDuration, setSegmentDuration] = useState(0);
|
||||
|
||||
const enableContinuousStreaming = config.enableContinuousStreaming === true;
|
||||
|
||||
const isActiveRef = useRef(isActive);
|
||||
useEffect(() => {
|
||||
isActiveRef.current = isActive;
|
||||
}, [isActive]);
|
||||
|
||||
const isMutedRef = useRef(isMuted);
|
||||
useEffect(() => {
|
||||
isMutedRef.current = isMuted;
|
||||
}, [isMuted]);
|
||||
|
||||
const callbacksRef = useRef({
|
||||
onAudioSegment: config.onAudioSegment,
|
||||
onSpeechStart: config.onSpeechStart,
|
||||
onSpeechEnd: config.onSpeechEnd,
|
||||
});
|
||||
useEffect(() => {
|
||||
callbacksRef.current = {
|
||||
onAudioSegment: config.onAudioSegment,
|
||||
onSpeechStart: config.onSpeechStart,
|
||||
onSpeechEnd: config.onSpeechEnd,
|
||||
};
|
||||
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd]);
|
||||
|
||||
const segmenterRef = useRef<SpeechSegmenter | null>(null);
|
||||
if (segmenterRef.current === null) {
|
||||
segmenterRef.current = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming,
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
|
||||
silenceDurationMs: config.silenceDuration,
|
||||
speechConfirmationMs: config.speechConfirmationDuration,
|
||||
detectionGracePeriodMs: config.detectionGracePeriod,
|
||||
},
|
||||
{
|
||||
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
|
||||
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
|
||||
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
|
||||
onDetectingChange: (next) => setIsDetecting(next),
|
||||
onSpeakingChange: (next) => setIsSpeaking(next),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
segmenterRef.current?.updateConfig({
|
||||
enableContinuousStreaming,
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
|
||||
silenceDurationMs: config.silenceDuration,
|
||||
speechConfirmationMs: config.speechConfirmationDuration,
|
||||
detectionGracePeriodMs: config.detectionGracePeriod,
|
||||
});
|
||||
}, [
|
||||
enableContinuousStreaming,
|
||||
config.volumeThreshold,
|
||||
config.confirmedDropGracePeriod,
|
||||
config.silenceDuration,
|
||||
config.speechConfirmationDuration,
|
||||
config.detectionGracePeriod,
|
||||
]);
|
||||
|
||||
// Update segment duration timer
|
||||
useEffect(() => {
|
||||
if (!isDetecting && !isSpeaking) {
|
||||
setSegmentDuration(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
|
||||
const interval = setInterval(() => {
|
||||
const elapsed = Date.now() - startTime;
|
||||
setSegmentDuration(elapsed);
|
||||
}, 100);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isDetecting, isSpeaking]);
|
||||
|
||||
// Listen to microphone data
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
useCallback<MicrophoneDataCallback>(
|
||||
(event) => {
|
||||
if (!isActiveRef.current || isMutedRef.current) return;
|
||||
|
||||
const pcmData: Uint8Array = event.data;
|
||||
segmenterRef.current?.pushPcmChunk(pcmData);
|
||||
},
|
||||
[]
|
||||
)
|
||||
);
|
||||
|
||||
// Listen to volume level for VAD
|
||||
useExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
useCallback<VolumeLevelCallback>(
|
||||
(event) => {
|
||||
if (!isActiveRef.current) return;
|
||||
|
||||
const volumeLevel: number = event.data;
|
||||
setVolume(volumeLevel);
|
||||
|
||||
if (isMutedRef.current) return;
|
||||
segmenterRef.current?.pushVolumeLevel(volumeLevel, Date.now());
|
||||
},
|
||||
[]
|
||||
)
|
||||
);
|
||||
|
||||
const ensureMicrophonePermission = useCallback(async () => {
|
||||
let permissionStatus = microphonePermission;
|
||||
|
||||
if (!permissionStatus?.granted) {
|
||||
try {
|
||||
permissionStatus = await requestMicrophonePermission();
|
||||
} catch (err) {
|
||||
throw new Error("Failed to request microphone permission");
|
||||
}
|
||||
}
|
||||
|
||||
if (!permissionStatus?.granted) {
|
||||
throw new Error(
|
||||
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
|
||||
);
|
||||
}
|
||||
}, [microphonePermission, requestMicrophonePermission]);
|
||||
|
||||
async function start(): Promise<void> {
|
||||
if (isActive) {
|
||||
console.log("[SpeechmaticsAudio] Already active");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureMicrophonePermission();
|
||||
|
||||
// Initialize audio if not already initialized
|
||||
if (!audioInitialized) {
|
||||
console.log("[SpeechmaticsAudio] Initializing audio...");
|
||||
await initialize();
|
||||
setAudioInitialized(true);
|
||||
console.log("[SpeechmaticsAudio] Audio initialized");
|
||||
}
|
||||
|
||||
console.log("[SpeechmaticsAudio] Starting audio capture...");
|
||||
|
||||
// Start recording
|
||||
toggleRecording(true);
|
||||
|
||||
setIsActive(true);
|
||||
console.log("[SpeechmaticsAudio] Audio capture started successfully");
|
||||
} catch (error) {
|
||||
console.error("[SpeechmaticsAudio] Start error:", error);
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
config.onError?.(err);
|
||||
await stop();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function stop(): Promise<void> {
|
||||
console.log("[SpeechmaticsAudio] Stopping audio capture...");
|
||||
|
||||
// Stop recording
|
||||
if (isActive) {
|
||||
toggleRecording(false);
|
||||
}
|
||||
|
||||
segmenterRef.current?.stop(Date.now());
|
||||
|
||||
// Tear down audio session
|
||||
if (audioInitialized) {
|
||||
tearDown();
|
||||
setAudioInitialized(false);
|
||||
console.log("[SpeechmaticsAudio] Audio torn down");
|
||||
}
|
||||
|
||||
// Reset state
|
||||
segmenterRef.current?.reset();
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
setVolume(0);
|
||||
setIsMuted(false);
|
||||
|
||||
console.log("[SpeechmaticsAudio] Audio capture stopped");
|
||||
}
|
||||
|
||||
function toggleMute(): void {
|
||||
setIsMuted((prev) => {
|
||||
const newMuted = !prev;
|
||||
console.log("[SpeechmaticsAudio] Mute toggled:", newMuted);
|
||||
|
||||
if (newMuted) {
|
||||
// Clear any ongoing speech detection/speaking state
|
||||
segmenterRef.current?.reset();
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
}
|
||||
|
||||
return newMuted;
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
start,
|
||||
stop,
|
||||
toggleMute,
|
||||
isActive,
|
||||
isSpeaking,
|
||||
isDetecting,
|
||||
isMuted,
|
||||
volume,
|
||||
segmentDuration,
|
||||
};
|
||||
}
|
||||
@@ -1,410 +0,0 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import { SpeechSegmenter } from "@/voice/speech-segmenter";
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
|
||||
export interface SpeechmaticsAudioConfig {
|
||||
onAudioSegment?: (segment: { audioData: string; isLast: boolean }) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onSpeechEnd?: () => void;
|
||||
onError?: (error: Error) => void;
|
||||
/** When true, stream microphone PCM continuously without VAD gating. */
|
||||
enableContinuousStreaming?: boolean;
|
||||
volumeThreshold: number; // 0-1
|
||||
/** ms dip debounce before VAD transitions from confirmed speaking to non-speaking */
|
||||
confirmedDropGracePeriod?: number;
|
||||
silenceDuration: number; // ms of silence before ending segment
|
||||
speechConfirmationDuration: number; // ms of sustained speech before confirming
|
||||
detectionGracePeriod: number; // ms grace period for volume dips during detection
|
||||
}
|
||||
|
||||
export interface SpeechmaticsAudio {
|
||||
start: () => Promise<void>;
|
||||
stop: () => Promise<void>;
|
||||
toggleMute: () => void;
|
||||
isActive: boolean;
|
||||
isSpeaking: boolean;
|
||||
isDetecting: boolean;
|
||||
isMuted: boolean;
|
||||
volume: number;
|
||||
segmentDuration: number;
|
||||
}
|
||||
|
||||
const getAudioContextCtor = (): (typeof AudioContext) | null => {
|
||||
if (typeof window === "undefined") {
|
||||
return null;
|
||||
}
|
||||
const ctor =
|
||||
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).AudioContext ||
|
||||
(window as typeof window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
|
||||
return ctor ?? null;
|
||||
};
|
||||
|
||||
const floatToInt16 = (sample: number): number => {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
||||
};
|
||||
|
||||
const resampleToPcm16 = (input: Float32Array, inputRate: number, outputRate: number): Int16Array => {
|
||||
if (input.length === 0) {
|
||||
return new Int16Array(0);
|
||||
}
|
||||
if (inputRate === outputRate) {
|
||||
const out = new Int16Array(input.length);
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
out[i] = floatToInt16(input[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const ratio = inputRate / outputRate;
|
||||
const outputLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const out = new Int16Array(outputLength);
|
||||
for (let i = 0; i < outputLength; i++) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const i1 = Math.min(input.length - 1, i0 + 1);
|
||||
const frac = sourceIndex - i0;
|
||||
const sample = input[i0] * (1 - frac) + input[i1] * frac;
|
||||
out[i] = floatToInt16(sample);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
export function useSpeechmaticsAudio(config: SpeechmaticsAudioConfig): SpeechmaticsAudio {
|
||||
const [isActive, setIsActive] = useState(false);
|
||||
const [isSpeaking, setIsSpeaking] = useState(false);
|
||||
const [isDetecting, setIsDetecting] = useState(false);
|
||||
const [volume, setVolume] = useState(0);
|
||||
const [isMuted, setIsMuted] = useState(false);
|
||||
const [segmentDuration, setSegmentDuration] = useState(0);
|
||||
|
||||
const enableContinuousStreaming = config.enableContinuousStreaming === true;
|
||||
|
||||
const callbacksRef = useRef({
|
||||
onAudioSegment: config.onAudioSegment,
|
||||
onSpeechStart: config.onSpeechStart,
|
||||
onSpeechEnd: config.onSpeechEnd,
|
||||
onError: config.onError,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
callbacksRef.current = {
|
||||
onAudioSegment: config.onAudioSegment,
|
||||
onSpeechStart: config.onSpeechStart,
|
||||
onSpeechEnd: config.onSpeechEnd,
|
||||
onError: config.onError,
|
||||
};
|
||||
}, [config.onAudioSegment, config.onSpeechStart, config.onSpeechEnd, config.onError]);
|
||||
|
||||
const segmenterRef = useRef<SpeechSegmenter | null>(null);
|
||||
if (segmenterRef.current === null) {
|
||||
segmenterRef.current = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming,
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
|
||||
silenceDurationMs: config.silenceDuration,
|
||||
speechConfirmationMs: config.speechConfirmationDuration,
|
||||
detectionGracePeriodMs: config.detectionGracePeriod,
|
||||
},
|
||||
{
|
||||
onAudioSegment: (segment) => callbacksRef.current.onAudioSegment?.(segment),
|
||||
onSpeechStart: () => callbacksRef.current.onSpeechStart?.(),
|
||||
onSpeechEnd: () => callbacksRef.current.onSpeechEnd?.(),
|
||||
onDetectingChange: (next) => setIsDetecting(next),
|
||||
onSpeakingChange: (next) => setIsSpeaking(next),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
segmenterRef.current?.updateConfig({
|
||||
enableContinuousStreaming,
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
confirmedDropGracePeriodMs: config.confirmedDropGracePeriod,
|
||||
silenceDurationMs: config.silenceDuration,
|
||||
speechConfirmationMs: config.speechConfirmationDuration,
|
||||
detectionGracePeriodMs: config.detectionGracePeriod,
|
||||
});
|
||||
}, [
|
||||
enableContinuousStreaming,
|
||||
config.volumeThreshold,
|
||||
config.confirmedDropGracePeriod,
|
||||
config.silenceDuration,
|
||||
config.speechConfirmationDuration,
|
||||
config.detectionGracePeriod,
|
||||
]);
|
||||
|
||||
const refs = useRef<{
|
||||
started: boolean;
|
||||
stream: MediaStream | null;
|
||||
context: AudioContext | null;
|
||||
source: MediaStreamAudioSourceNode | null;
|
||||
processor: ScriptProcessorNode | null;
|
||||
gain: GainNode | null;
|
||||
}>({
|
||||
started: false,
|
||||
stream: null,
|
||||
context: null,
|
||||
source: null,
|
||||
processor: null,
|
||||
gain: null,
|
||||
});
|
||||
|
||||
const isMutedRef = useRef(isMuted);
|
||||
useEffect(() => {
|
||||
isMutedRef.current = isMuted;
|
||||
}, [isMuted]);
|
||||
|
||||
const vadConfigRef = useRef({
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
});
|
||||
useEffect(() => {
|
||||
vadConfigRef.current = {
|
||||
volumeThreshold: config.volumeThreshold,
|
||||
};
|
||||
}, [config.volumeThreshold]);
|
||||
|
||||
const vadLogRef = useRef({
|
||||
lastLogMs: 0,
|
||||
lastSpeaking: false,
|
||||
lastDetecting: false,
|
||||
});
|
||||
|
||||
// Update segment duration timer
|
||||
useEffect(() => {
|
||||
if (!isDetecting && !isSpeaking) {
|
||||
setSegmentDuration(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const startTime = segmenterRef.current?.getSpeechDetectionStartMs() ?? Date.now();
|
||||
const interval = setInterval(() => {
|
||||
setSegmentDuration(Date.now() - startTime);
|
||||
}, 100);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [isDetecting, isSpeaking]);
|
||||
|
||||
const stopInternal = useCallback(async () => {
|
||||
refs.current.started = false;
|
||||
|
||||
try {
|
||||
refs.current.processor?.disconnect();
|
||||
refs.current.source?.disconnect();
|
||||
refs.current.gain?.disconnect();
|
||||
} catch {
|
||||
// best-effort teardown
|
||||
}
|
||||
|
||||
if (refs.current.stream) {
|
||||
for (const track of refs.current.stream.getTracks()) {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const context = refs.current.context;
|
||||
if (context && context.state !== "closed") {
|
||||
try {
|
||||
await context.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
refs.current.stream = null;
|
||||
refs.current.context = null;
|
||||
refs.current.source = null;
|
||||
refs.current.processor = null;
|
||||
refs.current.gain = null;
|
||||
|
||||
segmenterRef.current?.stop(Date.now());
|
||||
setIsActive(false);
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
setVolume(0);
|
||||
setIsMuted(false);
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (refs.current.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const missingNavigator =
|
||||
typeof navigator === "undefined" ||
|
||||
!navigator.mediaDevices ||
|
||||
typeof navigator.mediaDevices.getUserMedia !== "function";
|
||||
|
||||
const secureContext =
|
||||
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
|
||||
? window.isSecureContext
|
||||
: true;
|
||||
const currentOrigin = typeof window !== "undefined" && window.location ? window.location.origin : "unknown";
|
||||
const isTauri = getTauri() !== null;
|
||||
|
||||
try {
|
||||
if (missingNavigator) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
console.log("[Voice][Web] Microphone preflight", {
|
||||
secureContext,
|
||||
currentOrigin,
|
||||
isTauri,
|
||||
hasMediaDevices:
|
||||
typeof navigator !== "undefined" &&
|
||||
!!navigator.mediaDevices &&
|
||||
typeof navigator.mediaDevices.getUserMedia === "function",
|
||||
});
|
||||
if (!secureContext && !isTauri) {
|
||||
throw new Error(`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`);
|
||||
}
|
||||
if (!secureContext && isTauri) {
|
||||
console.warn(
|
||||
"[Voice][Web] Insecure context reported under Tauri; attempting getUserMedia anyway",
|
||||
{ currentOrigin }
|
||||
);
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
if (context.state === "suspended") {
|
||||
await context.resume();
|
||||
}
|
||||
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const processor = context.createScriptProcessor(4096, 1, 1);
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 0;
|
||||
|
||||
refs.current = {
|
||||
started: true,
|
||||
stream,
|
||||
context,
|
||||
source,
|
||||
processor,
|
||||
gain,
|
||||
};
|
||||
|
||||
processor.onaudioprocess = (event) => {
|
||||
if (!refs.current.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const sample = input[i];
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
const normalized = Math.min(1, Math.max(0, rms * 2));
|
||||
setVolume(normalized);
|
||||
|
||||
if (isMutedRef.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
const nowMs = Date.now();
|
||||
segmenterRef.current?.pushVolumeLevel(normalized, nowMs);
|
||||
const segmenter = segmenterRef.current;
|
||||
if (segmenter) {
|
||||
const speakingNow = segmenter.getIsSpeaking();
|
||||
const detectingNow = segmenter.getIsDetecting();
|
||||
const shouldLog =
|
||||
nowMs - vadLogRef.current.lastLogMs >= 150 ||
|
||||
speakingNow !== vadLogRef.current.lastSpeaking ||
|
||||
detectingNow !== vadLogRef.current.lastDetecting;
|
||||
if (shouldLog) {
|
||||
const threshold = vadConfigRef.current.volumeThreshold;
|
||||
const releaseThreshold = segmenter.getVolumeReleaseThreshold();
|
||||
console.log("[Voice][Web][VAD] level", {
|
||||
volume: Number(normalized.toFixed(3)),
|
||||
threshold: Number(threshold.toFixed(3)),
|
||||
releaseThreshold: Number(releaseThreshold.toFixed(3)),
|
||||
confirmedDropGraceMs: config.confirmedDropGracePeriod ?? 250,
|
||||
isSpeaking: speakingNow,
|
||||
isDetecting: detectingNow,
|
||||
});
|
||||
vadLogRef.current.lastLogMs = nowMs;
|
||||
vadLogRef.current.lastSpeaking = speakingNow;
|
||||
vadLogRef.current.lastDetecting = detectingNow;
|
||||
}
|
||||
}
|
||||
|
||||
const pcm16 = resampleToPcm16(input, context.sampleRate, 16000);
|
||||
const pcmBytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength);
|
||||
segmenterRef.current?.pushPcmChunk(pcmBytes);
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
|
||||
setIsActive(true);
|
||||
return;
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
callbacksRef.current.onError?.(err);
|
||||
await stopInternal();
|
||||
throw err;
|
||||
}
|
||||
}, [stopInternal]);
|
||||
|
||||
const stop = useCallback(async () => {
|
||||
await stopInternal();
|
||||
}, [stopInternal]);
|
||||
|
||||
const toggleMute = useCallback(() => {
|
||||
setIsMuted((prev) => {
|
||||
const nextMuted = !prev;
|
||||
if (nextMuted) {
|
||||
segmenterRef.current?.reset();
|
||||
setIsSpeaking(false);
|
||||
setIsDetecting(false);
|
||||
}
|
||||
return nextMuted;
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (refs.current.started) {
|
||||
void stopInternal();
|
||||
}
|
||||
};
|
||||
}, [stopInternal]);
|
||||
|
||||
return useMemo(
|
||||
() => ({
|
||||
start,
|
||||
stop,
|
||||
toggleMute,
|
||||
isActive,
|
||||
isSpeaking,
|
||||
isDetecting,
|
||||
isMuted,
|
||||
volume,
|
||||
segmentDuration,
|
||||
}),
|
||||
[start, stop, toggleMute, isActive, isSpeaking, isDetecting, isMuted, volume, segmentDuration]
|
||||
);
|
||||
}
|
||||
@@ -922,8 +922,7 @@ describe("HostRuntimeStore", () => {
|
||||
|
||||
useSessionStore.getState().initializeSession(
|
||||
host.serverId,
|
||||
fakeClient as unknown as DaemonClient,
|
||||
null as any
|
||||
fakeClient as unknown as DaemonClient
|
||||
);
|
||||
store.syncHosts([host]);
|
||||
|
||||
@@ -1046,8 +1045,7 @@ describe("HostRuntimeStore", () => {
|
||||
|
||||
useSessionStore.getState().initializeSession(
|
||||
host.serverId,
|
||||
fakeClient as unknown as DaemonClient,
|
||||
null as any
|
||||
fakeClient as unknown as DaemonClient
|
||||
);
|
||||
store.syncHosts([host]);
|
||||
|
||||
@@ -1122,8 +1120,7 @@ describe("HostRuntimeStore", () => {
|
||||
|
||||
useSessionStore.getState().initializeSession(
|
||||
host.serverId,
|
||||
fakeClient as unknown as DaemonClient,
|
||||
null as any
|
||||
fakeClient as unknown as DaemonClient
|
||||
);
|
||||
store.syncHosts([host]);
|
||||
|
||||
|
||||
@@ -103,29 +103,6 @@ function toErrorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function toReasonCode(reason: string | null): string | null {
|
||||
if (!reason) {
|
||||
return null;
|
||||
}
|
||||
const normalized = reason.toLowerCase();
|
||||
if (normalized.includes("timed out")) {
|
||||
return "connect_timeout";
|
||||
}
|
||||
if (normalized.includes("disposed")) {
|
||||
return "disposed";
|
||||
}
|
||||
if (normalized.includes("client closed") || normalized.includes("client_closed")) {
|
||||
return "client_closed";
|
||||
}
|
||||
if (normalized.includes("transport")) {
|
||||
return "transport_error";
|
||||
}
|
||||
if (normalized.includes("failed to connect")) {
|
||||
return "connect_failed";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function hashForLog(value: string): string {
|
||||
let hash = 0;
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
@@ -915,35 +892,12 @@ export class HostRuntimeController {
|
||||
});
|
||||
}
|
||||
|
||||
private logConnectionTransition(input: {
|
||||
private logConnectionTransition(_input: {
|
||||
from: HostRuntimeConnectionMachineState["tag"];
|
||||
to: HostRuntimeConnectionMachineState["tag"];
|
||||
event: HostRuntimeConnectionMachineEvent;
|
||||
}): void {
|
||||
const { event } = input;
|
||||
const reason =
|
||||
event.type === "connect_failed"
|
||||
? event.message
|
||||
: event.type === "client_state"
|
||||
? event.state.status === "disconnected"
|
||||
? event.state.reason ?? event.lastError ?? null
|
||||
: null
|
||||
: null;
|
||||
const reasonCode =
|
||||
event.type === "connect_failed"
|
||||
? "connect_failed"
|
||||
: toReasonCode(reason);
|
||||
console.info("[HostRuntimeTransition]", {
|
||||
serverId: this.host.serverId,
|
||||
clientIdHash: this.clientIdHash,
|
||||
from: input.from,
|
||||
to: input.to,
|
||||
event: event.type,
|
||||
connectionPath: this.snapshot.activeConnection?.type ?? null,
|
||||
generation: this.snapshot.clientGeneration,
|
||||
reasonCode,
|
||||
reason,
|
||||
});
|
||||
// Intentionally empty - logging removed.
|
||||
}
|
||||
|
||||
private trackConnectionFirstSeen(): void {
|
||||
@@ -1822,6 +1776,15 @@ export function useHostRuntimeSession(serverId: string): {
|
||||
};
|
||||
}
|
||||
|
||||
export function useHostRuntimeIsConnected(serverId: string): boolean {
|
||||
const store = getHostRuntimeStore();
|
||||
return useSyncExternalStore(
|
||||
(onStoreChange) => store.subscribe(serverId, onStoreChange),
|
||||
() => isHostRuntimeConnected(store.getSnapshot(serverId)),
|
||||
() => isHostRuntimeConnected(store.getSnapshot(serverId))
|
||||
);
|
||||
}
|
||||
|
||||
export function useHosts(): HostProfile[] {
|
||||
const store = getHostRuntimeStore();
|
||||
return useSyncExternalStore(
|
||||
|
||||
@@ -60,20 +60,11 @@ import {
|
||||
} from "./agent-ready-screen-bottom-anchor";
|
||||
|
||||
const EMPTY_STREAM_ITEMS: StreamItem[] = [];
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
|
||||
|
||||
function logAgentExplorer(event: string, details: Record<string, unknown>): void {
|
||||
if (!IS_DEV) {
|
||||
return;
|
||||
}
|
||||
console.log(`[AgentExplorer] ${event}`, details);
|
||||
}
|
||||
|
||||
function logWebStickyBottom(event: string, details: Record<string, unknown>): void {
|
||||
if (!IS_DEV || Platform.OS !== "web") {
|
||||
return;
|
||||
}
|
||||
console.log("[WebStickyBottom]", event, details);
|
||||
function logWebStickyBottom(
|
||||
_event: string,
|
||||
_details: Record<string, unknown>
|
||||
): void {
|
||||
// Intentionally disabled: this path is too noisy during voice debugging.
|
||||
}
|
||||
|
||||
export function AgentReadyScreen({
|
||||
@@ -81,11 +72,13 @@ export function AgentReadyScreen({
|
||||
agentId,
|
||||
showExplorerSidebar = true,
|
||||
wrapWithExplorerSidebarProvider = true,
|
||||
onOpenWorkspaceFile,
|
||||
}: {
|
||||
serverId: string;
|
||||
agentId: string;
|
||||
showExplorerSidebar?: boolean;
|
||||
wrapWithExplorerSidebarProvider?: boolean;
|
||||
onOpenWorkspaceFile?: (input: { filePath: string }) => void;
|
||||
}) {
|
||||
const resolvedAgentId = agentId?.trim() || undefined;
|
||||
const resolvedServerId = serverId?.trim() || undefined;
|
||||
@@ -129,6 +122,7 @@ export function AgentReadyScreen({
|
||||
isConnected={runtimeIsConnected}
|
||||
connectionStatus={connectionStatus}
|
||||
showExplorerSidebar={showExplorerSidebar}
|
||||
onOpenWorkspaceFile={onOpenWorkspaceFile}
|
||||
/>
|
||||
);
|
||||
|
||||
@@ -146,6 +140,7 @@ type AgentScreenContentProps = {
|
||||
isConnected: boolean;
|
||||
connectionStatus: HostRuntimeConnectionStatus;
|
||||
showExplorerSidebar: boolean;
|
||||
onOpenWorkspaceFile?: (input: { filePath: string }) => void;
|
||||
};
|
||||
|
||||
function toErrorMessage(error: unknown): string {
|
||||
@@ -166,6 +161,7 @@ function AgentScreenContent({
|
||||
isConnected,
|
||||
connectionStatus,
|
||||
showExplorerSidebar,
|
||||
onOpenWorkspaceFile,
|
||||
}: AgentScreenContentProps) {
|
||||
const { theme } = useUnistyles();
|
||||
const toast = useToast();
|
||||
@@ -259,10 +255,6 @@ function AgentScreenContent({
|
||||
}, [resolveCachedCheckoutIsGit, resolvedAgentId, checkout?.isGit, serverId]);
|
||||
const openExplorerForActiveCheckout = useCallback(() => {
|
||||
const checkoutContext = resolveCurrentExplorerCheckout();
|
||||
logAgentExplorer("openExplorerForActiveCheckout", {
|
||||
hasCheckoutContext: Boolean(checkoutContext),
|
||||
checkoutContext,
|
||||
});
|
||||
if (checkoutContext) {
|
||||
activateExplorerTabForCheckout(checkoutContext);
|
||||
}
|
||||
@@ -904,6 +896,7 @@ function AgentScreenContent({
|
||||
pendingPermissions={pendingPermissions}
|
||||
routeBottomAnchorRequest={routeBottomAnchorRequest}
|
||||
isAuthoritativeHistoryReady={hasAppliedAuthoritativeHistory}
|
||||
onOpenWorkspaceFile={onOpenWorkspaceFile}
|
||||
/>
|
||||
</ReanimatedAnimated.View>
|
||||
</View>
|
||||
|
||||
@@ -156,8 +156,6 @@ export function WorkspaceDesktopTabsRow({
|
||||
const layoutItem = layout.items[index] ?? null;
|
||||
const resolvedTabWidth = layoutItem?.width ?? 150;
|
||||
const showLabel = layoutItem?.showLabel ?? true;
|
||||
const labelCharCap = layoutItem?.labelCharCap ?? tab.label.length;
|
||||
const renderedLabel = showLabel ? tab.label.slice(0, Math.max(1, labelCharCap)) : "";
|
||||
const presentation = deriveWorkspaceTabPresentation({ tab, agent: tabAgent });
|
||||
const tooltipLabel =
|
||||
tab.kind === "agent" && tab.titleState === "loading"
|
||||
@@ -172,11 +170,7 @@ export function WorkspaceDesktopTabsRow({
|
||||
return (
|
||||
<ContextMenu key={tab.key}>
|
||||
<Tooltip delayDuration={400} enabledOnDesktop enabledOnMobile={false}>
|
||||
<TooltipTrigger
|
||||
testID={`workspace-tab-tooltip-${tab.key}`}
|
||||
accessibilityRole="none"
|
||||
style={styles.tabTooltipTrigger}
|
||||
>
|
||||
<TooltipTrigger asChild triggerRefProp="triggerRef">
|
||||
<ContextMenuTrigger
|
||||
testID={`workspace-tab-${tab.key}`}
|
||||
enabledOnMobile={false}
|
||||
@@ -228,9 +222,11 @@ export function WorkspaceDesktopTabsRow({
|
||||
isActive && styles.tabLabelActive,
|
||||
shouldShowCloseButton && styles.tabLabelWithCloseButton,
|
||||
]}
|
||||
selectable={false}
|
||||
numberOfLines={1}
|
||||
ellipsizeMode="tail"
|
||||
>
|
||||
{renderedLabel}
|
||||
{tab.label}
|
||||
</Text>
|
||||
)
|
||||
) : null}
|
||||
@@ -411,9 +407,6 @@ const styles = StyleSheet.create((theme) => ({
|
||||
paddingRight: theme.spacing[2],
|
||||
paddingVertical: theme.spacing[1],
|
||||
},
|
||||
tabTooltipTrigger: {
|
||||
flexShrink: 0,
|
||||
},
|
||||
tab: {
|
||||
paddingHorizontal: theme.spacing[3],
|
||||
paddingVertical: theme.spacing[2],
|
||||
@@ -421,6 +414,7 @@ const styles = StyleSheet.create((theme) => ({
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
gap: theme.spacing[1],
|
||||
userSelect: "none",
|
||||
},
|
||||
tabHandle: {
|
||||
flexDirection: "row",
|
||||
@@ -428,6 +422,7 @@ const styles = StyleSheet.create((theme) => ({
|
||||
gap: theme.spacing[1],
|
||||
flex: 1,
|
||||
minWidth: 0,
|
||||
userSelect: "none",
|
||||
},
|
||||
tabIcon: {
|
||||
flexShrink: 0,
|
||||
@@ -444,6 +439,7 @@ const styles = StyleSheet.create((theme) => ({
|
||||
color: theme.colors.foregroundMuted,
|
||||
fontSize: theme.fontSize.sm,
|
||||
fontWeight: theme.fontWeight.normal,
|
||||
userSelect: "none",
|
||||
},
|
||||
tabLabelSkeleton: {
|
||||
width: 96,
|
||||
|
||||
@@ -29,6 +29,7 @@ type WorkspaceDraftAgentTabProps = {
|
||||
tabId: string
|
||||
draftId: string
|
||||
onCreated: (snapshot: AgentSnapshotPayload) => void
|
||||
onOpenWorkspaceFile: (input: { filePath: string }) => void
|
||||
}
|
||||
|
||||
export function WorkspaceDraftAgentTab({
|
||||
@@ -37,6 +38,7 @@ export function WorkspaceDraftAgentTab({
|
||||
tabId,
|
||||
draftId,
|
||||
onCreated,
|
||||
onOpenWorkspaceFile,
|
||||
}: WorkspaceDraftAgentTabProps) {
|
||||
const { client, isConnected } = useHostRuntimeSession(serverId)
|
||||
const addImagesRef = useRef<((images: ImageAttachment[]) => void) | null>(null)
|
||||
@@ -204,6 +206,7 @@ export function WorkspaceDraftAgentTab({
|
||||
agent={draftAgent}
|
||||
streamItems={optimisticStreamItems}
|
||||
pendingPermissions={EMPTY_PENDING_PERMISSIONS}
|
||||
onOpenWorkspaceFile={onOpenWorkspaceFile}
|
||||
/>
|
||||
</View>
|
||||
) : (
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { memo, useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import {
|
||||
ActivityIndicator,
|
||||
BackHandler,
|
||||
@@ -25,7 +25,7 @@ import { StyleSheet, UnistylesRuntime, useUnistyles } from "react-native-unistyl
|
||||
import { SidebarMenuToggle } from "@/components/headers/menu-header";
|
||||
import { HeaderToggleButton } from "@/components/headers/header-toggle-button";
|
||||
import { ScreenHeader } from "@/components/headers/screen-header";
|
||||
import { Combobox } from "@/components/ui/combobox";
|
||||
import { Combobox, type ComboboxOption } from "@/components/ui/combobox";
|
||||
import { Shortcut } from "@/components/ui/shortcut";
|
||||
import {
|
||||
DropdownMenu,
|
||||
@@ -80,6 +80,7 @@ import { WorkspaceDraftAgentTab } from "@/screens/workspace/workspace-draft-agen
|
||||
import { WorkspaceDesktopTabsRow } from "@/screens/workspace/workspace-desktop-tabs-row";
|
||||
import {
|
||||
deriveWorkspaceTabPresentation,
|
||||
type WorkspaceTabPresentation,
|
||||
WorkspaceTabIcon,
|
||||
WorkspaceTabOptionRow,
|
||||
} from "@/screens/workspace/workspace-tab-presentation";
|
||||
@@ -141,6 +142,120 @@ function buildOpenIntentKey(input: {
|
||||
return `${input.serverId}:${input.workspaceId}:${openParam}`;
|
||||
}
|
||||
|
||||
type MobileWorkspaceTabSwitcherProps = {
|
||||
activeTabKey: string;
|
||||
activeTabLabel: string;
|
||||
activeTabPresentation: WorkspaceTabPresentation | null;
|
||||
tabSwitcherOptions: ComboboxOption[];
|
||||
tabByKey: Map<string, WorkspaceTabDescriptor>;
|
||||
tabPresentationsByKey: Map<string, WorkspaceTabPresentation>;
|
||||
onSelectSwitcherTab: (key: string) => void;
|
||||
onSelectNewTabOption: (key: typeof NEW_TAB_AGENT_OPTION_ID) => void;
|
||||
};
|
||||
|
||||
const MobileWorkspaceTabSwitcher = memo(function MobileWorkspaceTabSwitcher({
|
||||
activeTabKey,
|
||||
activeTabLabel,
|
||||
activeTabPresentation,
|
||||
tabSwitcherOptions,
|
||||
tabByKey,
|
||||
tabPresentationsByKey,
|
||||
onSelectSwitcherTab,
|
||||
onSelectNewTabOption,
|
||||
}: MobileWorkspaceTabSwitcherProps) {
|
||||
const { theme } = useUnistyles();
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const anchorRef = useRef<View>(null);
|
||||
|
||||
return (
|
||||
<View style={styles.mobileTabsRow} testID="workspace-tabs-row">
|
||||
<Pressable
|
||||
ref={anchorRef}
|
||||
style={({ hovered, pressed }) => [
|
||||
styles.switcherTrigger,
|
||||
(hovered || pressed || isOpen) && styles.switcherTriggerActive,
|
||||
{ borderWidth: 0, borderColor: "transparent" },
|
||||
Platform.OS === "web"
|
||||
? {
|
||||
outlineStyle: "solid",
|
||||
outlineWidth: 0,
|
||||
outlineColor: "transparent",
|
||||
}
|
||||
: null,
|
||||
]}
|
||||
onPress={() => setIsOpen(true)}
|
||||
>
|
||||
<View style={styles.switcherTriggerLeft}>
|
||||
<View style={styles.switcherTriggerIcon} testID="workspace-active-tab-icon">
|
||||
{activeTabPresentation ? (
|
||||
<WorkspaceTabIcon presentation={activeTabPresentation} active />
|
||||
) : null}
|
||||
</View>
|
||||
|
||||
<Text style={styles.switcherTriggerText} numberOfLines={1}>
|
||||
{activeTabLabel}
|
||||
</Text>
|
||||
</View>
|
||||
|
||||
<ChevronDown size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
|
||||
</Pressable>
|
||||
|
||||
<View style={styles.mobileTabsActions}>
|
||||
<Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
|
||||
<TooltipTrigger
|
||||
testID="workspace-new-agent-tab"
|
||||
onPress={() => onSelectNewTabOption(NEW_TAB_AGENT_OPTION_ID)}
|
||||
accessibilityRole="button"
|
||||
accessibilityLabel="New agent tab"
|
||||
style={({ hovered, pressed }) => [
|
||||
styles.newTabActionButton,
|
||||
(hovered || pressed) && styles.newTabActionButtonHovered,
|
||||
]}
|
||||
>
|
||||
<Plus size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom" align="end" offset={8}>
|
||||
<View style={styles.newTabTooltipRow}>
|
||||
<Text style={styles.newTabTooltipText}>New agent tab</Text>
|
||||
<Shortcut keys={["mod", "T"]} style={styles.newTabTooltipShortcut} />
|
||||
</View>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
</View>
|
||||
|
||||
<Combobox
|
||||
options={tabSwitcherOptions}
|
||||
value={activeTabKey}
|
||||
onSelect={onSelectSwitcherTab}
|
||||
searchable={false}
|
||||
title="Switch tab"
|
||||
searchPlaceholder="Search tabs"
|
||||
open={isOpen}
|
||||
onOpenChange={setIsOpen}
|
||||
enableDismissOnClose={false}
|
||||
anchorRef={anchorRef}
|
||||
renderOption={({ option, selected, active, onPress }) => {
|
||||
const tab = tabByKey.get(option.id);
|
||||
if (!tab) {
|
||||
return <View />;
|
||||
}
|
||||
const presentation =
|
||||
tabPresentationsByKey.get(option.id) ??
|
||||
deriveWorkspaceTabPresentation({ tab });
|
||||
return (
|
||||
<WorkspaceTabOptionRow
|
||||
presentation={presentation}
|
||||
selected={selected}
|
||||
active={active}
|
||||
onPress={onPress}
|
||||
/>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</View>
|
||||
);
|
||||
});
|
||||
|
||||
export function WorkspaceScreen({
|
||||
serverId,
|
||||
workspaceId,
|
||||
@@ -344,6 +459,10 @@ function WorkspaceScreenContent({
|
||||
const isWorkspaceHeaderLoading = workspaceHeader === null;
|
||||
|
||||
const isGitCheckout = checkoutQuery.data?.isGit ?? false;
|
||||
const currentBranchName =
|
||||
checkoutQuery.data?.isGit && checkoutQuery.data.currentBranch !== "HEAD"
|
||||
? trimNonEmpty(checkoutQuery.data.currentBranch)
|
||||
: null;
|
||||
const mobileView = usePanelStore((state) => state.mobileView);
|
||||
const desktopFileExplorerOpen = usePanelStore(
|
||||
(state) => state.desktop.fileExplorerOpen
|
||||
@@ -752,12 +871,21 @@ function WorkspaceScreenContent({
|
||||
[closeToAgent, isMobile, navigateToTabId, normalizedServerId, normalizedWorkspaceId, openOrFocusTab]
|
||||
);
|
||||
|
||||
const [isTabSwitcherOpen, setIsTabSwitcherOpen] = useState(false);
|
||||
const handleOpenFileFromChat = useCallback(
|
||||
({ filePath }: { filePath: string }) => {
|
||||
const normalizedFilePath = filePath.trim();
|
||||
if (!normalizedFilePath) {
|
||||
return;
|
||||
}
|
||||
handleOpenFileFromExplorer(normalizedFilePath);
|
||||
},
|
||||
[handleOpenFileFromExplorer]
|
||||
);
|
||||
|
||||
const [hoveredTabKey, setHoveredTabKey] = useState<string | null>(null);
|
||||
const [hoveredCloseTabKey, setHoveredCloseTabKey] = useState<string | null>(
|
||||
null
|
||||
);
|
||||
const tabSwitcherAnchorRef = useRef<View>(null);
|
||||
|
||||
const tabByKey = useMemo(() => {
|
||||
const map = new Map<string, WorkspaceTabDescriptor>();
|
||||
@@ -822,7 +950,6 @@ function WorkspaceScreenContent({
|
||||
|
||||
const handleSelectSwitcherTab = useCallback(
|
||||
(key: string) => {
|
||||
setIsTabSwitcherOpen(false);
|
||||
navigateToTabId(key);
|
||||
},
|
||||
[navigateToTabId]
|
||||
@@ -1025,6 +1152,20 @@ function WorkspaceScreenContent({
|
||||
}
|
||||
}, [normalizedWorkspaceId, toast]);
|
||||
|
||||
const handleCopyBranchName = useCallback(async () => {
|
||||
if (!currentBranchName) {
|
||||
toast.error("Branch name not available");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await Clipboard.setStringAsync(currentBranchName);
|
||||
toast.copied("Branch name");
|
||||
} catch {
|
||||
toast.error("Copy failed");
|
||||
}
|
||||
}, [currentBranchName, toast]);
|
||||
|
||||
const handleBulkCloseTabs = useCallback(
|
||||
async (input: { tabsToClose: WorkspaceTabDescriptor[]; title: string; logLabel: string }) => {
|
||||
const { tabsToClose, title, logLabel } = input;
|
||||
@@ -1240,6 +1381,7 @@ function WorkspaceScreenContent({
|
||||
workspaceId={normalizedWorkspaceId}
|
||||
tabId={activeTabId ?? target.draftId}
|
||||
draftId={target.draftId}
|
||||
onOpenWorkspaceFile={handleOpenFileFromChat}
|
||||
onCreated={(agentSnapshot) => {
|
||||
const tabId = activeTabId ?? target.draftId;
|
||||
const normalized = normalizeAgentSnapshot(agentSnapshot, normalizedServerId);
|
||||
@@ -1274,6 +1416,7 @@ function WorkspaceScreenContent({
|
||||
agentId={target.agentId}
|
||||
showExplorerSidebar={false}
|
||||
wrapWithExplorerSidebarProvider={false}
|
||||
onOpenWorkspaceFile={handleOpenFileFromChat}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -1389,6 +1532,20 @@ function WorkspaceScreenContent({
|
||||
>
|
||||
Copy workspace path
|
||||
</DropdownMenuItem>
|
||||
{currentBranchName ? (
|
||||
<DropdownMenuItem
|
||||
testID="workspace-header-copy-branch-name"
|
||||
leading={
|
||||
<Copy
|
||||
size={16}
|
||||
color={theme.colors.foregroundMuted}
|
||||
/>
|
||||
}
|
||||
onSelect={handleCopyBranchName}
|
||||
>
|
||||
Copy branch name
|
||||
</DropdownMenuItem>
|
||||
) : null}
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</View>
|
||||
@@ -1477,91 +1634,16 @@ function WorkspaceScreenContent({
|
||||
/>
|
||||
|
||||
{isMobile ? (
|
||||
<View style={styles.mobileTabsRow} testID="workspace-tabs-row">
|
||||
<Pressable
|
||||
ref={tabSwitcherAnchorRef}
|
||||
style={({ hovered, pressed }) => [
|
||||
styles.switcherTrigger,
|
||||
(hovered || pressed || isTabSwitcherOpen) && styles.switcherTriggerActive,
|
||||
{ borderWidth: 0, borderColor: "transparent" },
|
||||
Platform.OS === "web"
|
||||
? {
|
||||
outlineStyle: "solid",
|
||||
outlineWidth: 0,
|
||||
outlineColor: "transparent",
|
||||
}
|
||||
: null,
|
||||
]}
|
||||
onPress={() => setIsTabSwitcherOpen(true)}
|
||||
>
|
||||
<View style={styles.switcherTriggerLeft}>
|
||||
<View style={styles.switcherTriggerIcon} testID="workspace-active-tab-icon">
|
||||
{activeTabPresentation ? (
|
||||
<WorkspaceTabIcon presentation={activeTabPresentation} active />
|
||||
) : null}
|
||||
</View>
|
||||
|
||||
<Text style={styles.switcherTriggerText} numberOfLines={1}>
|
||||
{activeTabLabel}
|
||||
</Text>
|
||||
</View>
|
||||
|
||||
<ChevronDown size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
|
||||
</Pressable>
|
||||
|
||||
<View style={styles.mobileTabsActions}>
|
||||
<Tooltip delayDuration={0} enabledOnDesktop enabledOnMobile={false}>
|
||||
<TooltipTrigger
|
||||
testID="workspace-new-agent-tab"
|
||||
onPress={() => handleSelectNewTabOption(NEW_TAB_AGENT_OPTION_ID)}
|
||||
accessibilityRole="button"
|
||||
accessibilityLabel="New agent tab"
|
||||
style={({ hovered, pressed }) => [
|
||||
styles.newTabActionButton,
|
||||
(hovered || pressed) && styles.newTabActionButtonHovered,
|
||||
]}
|
||||
>
|
||||
<Plus size={theme.iconSize.sm} color={theme.colors.foregroundMuted} />
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom" align="end" offset={8}>
|
||||
<View style={styles.newTabTooltipRow}>
|
||||
<Text style={styles.newTabTooltipText}>New agent tab</Text>
|
||||
<Shortcut keys={["mod", "T"]} style={styles.newTabTooltipShortcut} />
|
||||
</View>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
|
||||
</View>
|
||||
|
||||
<Combobox
|
||||
options={tabSwitcherOptions}
|
||||
value={activeTabKey}
|
||||
onSelect={handleSelectSwitcherTab}
|
||||
searchable={false}
|
||||
title="Switch tab"
|
||||
searchPlaceholder="Search tabs"
|
||||
open={isTabSwitcherOpen}
|
||||
onOpenChange={setIsTabSwitcherOpen}
|
||||
anchorRef={tabSwitcherAnchorRef}
|
||||
renderOption={({ option, selected, active, onPress }) => {
|
||||
const tab = tabByKey.get(option.id);
|
||||
if (!tab) {
|
||||
return <View />;
|
||||
}
|
||||
const presentation =
|
||||
tabPresentationsByKey.get(option.id) ??
|
||||
deriveWorkspaceTabPresentation({ tab });
|
||||
return (
|
||||
<WorkspaceTabOptionRow
|
||||
presentation={presentation}
|
||||
selected={selected}
|
||||
active={active}
|
||||
onPress={onPress}
|
||||
/>
|
||||
);
|
||||
}}
|
||||
/>
|
||||
</View>
|
||||
<MobileWorkspaceTabSwitcher
|
||||
activeTabKey={activeTabKey}
|
||||
activeTabLabel={activeTabLabel}
|
||||
activeTabPresentation={activeTabPresentation}
|
||||
tabSwitcherOptions={tabSwitcherOptions}
|
||||
tabByKey={tabByKey}
|
||||
tabPresentationsByKey={tabPresentationsByKey}
|
||||
onSelectSwitcherTab={handleSelectSwitcherTab}
|
||||
onSelectNewTabOption={handleSelectNewTabOption}
|
||||
/>
|
||||
) : (
|
||||
<WorkspaceDesktopTabsRow
|
||||
tabs={tabs}
|
||||
|
||||
@@ -51,23 +51,6 @@ export const DEFAULT_EXPLORER_FILES_SPLIT_RATIO = 0.38;
|
||||
export const MIN_EXPLORER_FILES_SPLIT_RATIO = 0.2;
|
||||
export const MAX_EXPLORER_FILES_SPLIT_RATIO = 0.8;
|
||||
|
||||
const IS_DEV = Boolean((globalThis as { __DEV__?: boolean }).__DEV__);
|
||||
|
||||
function logPanelTransition(
|
||||
action: string,
|
||||
details: Record<string, unknown>
|
||||
): void {
|
||||
if (!IS_DEV) {
|
||||
return;
|
||||
}
|
||||
const stack =
|
||||
new Error().stack
|
||||
?.split("\n")
|
||||
.slice(2, 8)
|
||||
.join("\n") ?? "stack unavailable";
|
||||
console.log(`[PanelStore] ${action}`, details, stack);
|
||||
}
|
||||
|
||||
interface PanelState {
|
||||
// Mobile: which panel is currently shown
|
||||
mobileView: MobilePanelView;
|
||||
@@ -151,103 +134,52 @@ export const usePanelStore = create<PanelState>()(
|
||||
explorerFilesSplitRatio: DEFAULT_EXPLORER_FILES_SPLIT_RATIO,
|
||||
|
||||
openAgentList: () =>
|
||||
set((state) => {
|
||||
const nextState = {
|
||||
mobileView: "agent-list" as const,
|
||||
desktop: { ...state.desktop, agentListOpen: true },
|
||||
};
|
||||
logPanelTransition("openAgentList", {
|
||||
fromMobileView: state.mobileView,
|
||||
toMobileView: nextState.mobileView,
|
||||
fromDesktopAgentListOpen: state.desktop.agentListOpen,
|
||||
toDesktopAgentListOpen: nextState.desktop.agentListOpen,
|
||||
});
|
||||
return nextState;
|
||||
}),
|
||||
set((state) => ({
|
||||
mobileView: "agent-list" as const,
|
||||
desktop: { ...state.desktop, agentListOpen: true },
|
||||
})),
|
||||
|
||||
openFileExplorer: () =>
|
||||
set((state) => {
|
||||
const resolvedTab = resolveExplorerTabFromActiveCheckout(state);
|
||||
const nextMobileView: MobilePanelView = "file-explorer";
|
||||
const nextDesktop = { ...state.desktop, fileExplorerOpen: true };
|
||||
const nextState = {
|
||||
mobileView: nextMobileView,
|
||||
desktop: nextDesktop,
|
||||
return {
|
||||
mobileView: "file-explorer" as MobilePanelView,
|
||||
desktop: { ...state.desktop, fileExplorerOpen: true },
|
||||
...(resolvedTab ? { explorerTab: resolvedTab } : {}),
|
||||
};
|
||||
logPanelTransition("openFileExplorer", {
|
||||
fromMobileView: state.mobileView,
|
||||
toMobileView: nextMobileView,
|
||||
fromDesktopFileExplorerOpen: state.desktop.fileExplorerOpen,
|
||||
toDesktopFileExplorerOpen: nextDesktop.fileExplorerOpen,
|
||||
resolvedTab: resolvedTab ?? null,
|
||||
activeCheckout: state.activeExplorerCheckout,
|
||||
});
|
||||
return nextState;
|
||||
}),
|
||||
closeFileExplorer: () =>
|
||||
set((state) => {
|
||||
const nextState = {
|
||||
mobileView:
|
||||
state.mobileView === "file-explorer" ? ("agent" as const) : state.mobileView,
|
||||
desktop: {
|
||||
...state.desktop,
|
||||
fileExplorerOpen: false,
|
||||
},
|
||||
};
|
||||
logPanelTransition("closeFileExplorer", {
|
||||
fromMobileView: state.mobileView,
|
||||
toMobileView: nextState.mobileView,
|
||||
fromDesktopFileExplorerOpen: state.desktop.fileExplorerOpen,
|
||||
toDesktopFileExplorerOpen: nextState.desktop.fileExplorerOpen,
|
||||
});
|
||||
return nextState;
|
||||
}),
|
||||
set((state) => ({
|
||||
mobileView:
|
||||
state.mobileView === "file-explorer" ? ("agent" as const) : state.mobileView,
|
||||
desktop: {
|
||||
...state.desktop,
|
||||
fileExplorerOpen: false,
|
||||
},
|
||||
})),
|
||||
|
||||
closeToAgent: () =>
|
||||
set((state) => {
|
||||
const nextState = {
|
||||
mobileView: "agent" as const,
|
||||
// On desktop, closing depends on which panel triggered it
|
||||
// This is called when closing via gesture/backdrop, so we close the currently active mobile panel
|
||||
desktop: {
|
||||
agentListOpen:
|
||||
state.mobileView === "agent-list" ? false : state.desktop.agentListOpen,
|
||||
fileExplorerOpen:
|
||||
state.mobileView === "file-explorer" ? false : state.desktop.fileExplorerOpen,
|
||||
},
|
||||
};
|
||||
logPanelTransition("closeToAgent", {
|
||||
fromMobileView: state.mobileView,
|
||||
toMobileView: nextState.mobileView,
|
||||
fromDesktopAgentListOpen: state.desktop.agentListOpen,
|
||||
toDesktopAgentListOpen: nextState.desktop.agentListOpen,
|
||||
fromDesktopFileExplorerOpen: state.desktop.fileExplorerOpen,
|
||||
toDesktopFileExplorerOpen: nextState.desktop.fileExplorerOpen,
|
||||
});
|
||||
return nextState;
|
||||
}),
|
||||
set((state) => ({
|
||||
mobileView: "agent" as const,
|
||||
// On desktop, closing depends on which panel triggered it
|
||||
// This is called when closing via gesture/backdrop, so we close the currently active mobile panel
|
||||
desktop: {
|
||||
agentListOpen:
|
||||
state.mobileView === "agent-list" ? false : state.desktop.agentListOpen,
|
||||
fileExplorerOpen:
|
||||
state.mobileView === "file-explorer" ? false : state.desktop.fileExplorerOpen,
|
||||
},
|
||||
})),
|
||||
|
||||
toggleAgentList: () =>
|
||||
set((state) => {
|
||||
set((state) => ({
|
||||
// Mobile: toggle between agent and agent-list
|
||||
const newMobileView: MobilePanelView =
|
||||
state.mobileView === "agent-list" ? "agent" : "agent-list";
|
||||
const nextState = {
|
||||
mobileView: newMobileView,
|
||||
desktop: {
|
||||
...state.desktop,
|
||||
agentListOpen: !state.desktop.agentListOpen,
|
||||
},
|
||||
};
|
||||
logPanelTransition("toggleAgentList", {
|
||||
fromMobileView: state.mobileView,
|
||||
toMobileView: nextState.mobileView,
|
||||
fromDesktopAgentListOpen: state.desktop.agentListOpen,
|
||||
toDesktopAgentListOpen: nextState.desktop.agentListOpen,
|
||||
});
|
||||
return nextState;
|
||||
}),
|
||||
mobileView: (state.mobileView === "agent-list" ? "agent" : "agent-list") as MobilePanelView,
|
||||
desktop: {
|
||||
...state.desktop,
|
||||
agentListOpen: !state.desktop.agentListOpen,
|
||||
},
|
||||
})),
|
||||
|
||||
toggleFileExplorer: () =>
|
||||
set((state) => {
|
||||
@@ -266,23 +198,12 @@ export const usePanelStore = create<PanelState>()(
|
||||
mobileView: nextMobileView,
|
||||
desktop: nextDesktop,
|
||||
};
|
||||
let resolvedTab: ExplorerTab | null = null;
|
||||
if (willOpenMobile || willOpenDesktop) {
|
||||
resolvedTab = resolveExplorerTabFromActiveCheckout(state);
|
||||
const resolvedTab = resolveExplorerTabFromActiveCheckout(state);
|
||||
if (resolvedTab) {
|
||||
nextState.explorerTab = resolvedTab;
|
||||
}
|
||||
}
|
||||
logPanelTransition("toggleFileExplorer", {
|
||||
fromMobileView: state.mobileView,
|
||||
toMobileView: nextMobileView,
|
||||
fromDesktopFileExplorerOpen: state.desktop.fileExplorerOpen,
|
||||
toDesktopFileExplorerOpen: nextDesktop.fileExplorerOpen,
|
||||
willOpenMobile,
|
||||
willOpenDesktop,
|
||||
resolvedTab: resolvedTab ?? null,
|
||||
activeCheckout: state.activeExplorerCheckout,
|
||||
});
|
||||
return nextState;
|
||||
}),
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { create } from "zustand";
|
||||
import { subscribeWithSelector } from "zustand/middleware";
|
||||
import type { DaemonClient } from "@server/client/daemon-client";
|
||||
import type { useAudioPlayer } from "@/hooks/use-audio-player";
|
||||
import type { AgentDirectoryEntry } from "@/types/agent-directory";
|
||||
import type { StreamItem } from "@/types/stream";
|
||||
import type { PendingPermission } from "@/types/shared";
|
||||
@@ -208,9 +207,6 @@ export interface SessionState {
|
||||
// Server metadata (from server_info handshake)
|
||||
serverInfo: DaemonServerInfo | null;
|
||||
|
||||
// Audio player (immutable reference)
|
||||
audioPlayer: ReturnType<typeof useAudioPlayer> | null;
|
||||
|
||||
// Hydration status
|
||||
hasHydratedAgents: boolean;
|
||||
hasHydratedWorkspaces: boolean;
|
||||
@@ -264,7 +260,7 @@ interface SessionStoreState {
|
||||
// Action types
|
||||
interface SessionStoreActions {
|
||||
// Session management
|
||||
initializeSession: (serverId: string, client: DaemonClient, audioPlayer: ReturnType<typeof useAudioPlayer>) => void;
|
||||
initializeSession: (serverId: string, client: DaemonClient) => void;
|
||||
clearSession: (serverId: string) => void;
|
||||
getSession: (serverId: string) => SessionState | undefined;
|
||||
updateSessionClient: (serverId: string, client: DaemonClient) => void;
|
||||
@@ -359,12 +355,14 @@ type SessionStore = SessionStoreState & SessionStoreActions;
|
||||
const agentLastActivityCoalescer = createAgentLastActivityCoalescer();
|
||||
|
||||
// Helper to create initial session state
|
||||
function createInitialSessionState(serverId: string, client: DaemonClient, audioPlayer: ReturnType<typeof useAudioPlayer>): SessionState {
|
||||
function createInitialSessionState(
|
||||
serverId: string,
|
||||
client: DaemonClient
|
||||
): SessionState {
|
||||
return {
|
||||
serverId,
|
||||
client,
|
||||
serverInfo: null,
|
||||
audioPlayer,
|
||||
hasHydratedAgents: false,
|
||||
hasHydratedWorkspaces: false,
|
||||
isPlayingAudio: false,
|
||||
@@ -424,7 +422,7 @@ export const useSessionStore = create<SessionStore>()(
|
||||
agentLastActivity: new Map(),
|
||||
|
||||
// Session management
|
||||
initializeSession: (serverId, client, audioPlayer) => {
|
||||
initializeSession: (serverId, client) => {
|
||||
set((prev) => {
|
||||
if (prev.sessions[serverId]) {
|
||||
return prev;
|
||||
@@ -433,7 +431,7 @@ export const useSessionStore = create<SessionStore>()(
|
||||
...prev,
|
||||
sessions: {
|
||||
...prev.sessions,
|
||||
[serverId]: createInitialSessionState(serverId, client, audioPlayer),
|
||||
[serverId]: createInitialSessionState(serverId, client),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
@@ -136,4 +136,38 @@ describe("workspace-tabs-store retargetTab", () => {
|
||||
expect(matchingTabs).toHaveLength(1);
|
||||
expect(order).toEqual([draftTabId]);
|
||||
});
|
||||
|
||||
it("openOrFocusTab re-focuses an existing file tab after the workspace focus changed", () => {
|
||||
const key = buildWorkspaceTabPersistenceKey({ serverId: SERVER_ID, workspaceId: WORKSPACE_ID });
|
||||
expect(key).toBeTruthy();
|
||||
const workspaceKey = key as string;
|
||||
|
||||
const fileTabId = useWorkspaceTabsStore.getState().openOrFocusTab({
|
||||
serverId: SERVER_ID,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
target: { kind: "file", path: "/repo/worktree/src/index.ts" },
|
||||
});
|
||||
const terminalTabId = useWorkspaceTabsStore.getState().openOrFocusTab({
|
||||
serverId: SERVER_ID,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
target: { kind: "terminal", terminalId: "term-1" },
|
||||
});
|
||||
|
||||
expect(fileTabId).toBe("file_/repo/worktree/src/index.ts");
|
||||
expect(terminalTabId).toBe("terminal_term-1");
|
||||
expect(useWorkspaceTabsStore.getState().focusedTabIdByWorkspace[workspaceKey]).toBe(
|
||||
terminalTabId
|
||||
);
|
||||
|
||||
const reopenedFileTabId = useWorkspaceTabsStore.getState().openOrFocusTab({
|
||||
serverId: SERVER_ID,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
target: { kind: "file", path: "/repo/worktree/src/index.ts" },
|
||||
});
|
||||
|
||||
expect(reopenedFileTabId).toBe(fileTabId);
|
||||
expect(useWorkspaceTabsStore.getState().focusedTabIdByWorkspace[workspaceKey]).toBe(
|
||||
fileTabId
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
40
packages/app/src/styles/markdown-styles.test.ts
Normal file
40
packages/app/src/styles/markdown-styles.test.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createMarkdownStyles } from "./markdown-styles";
|
||||
import { darkTheme } from "./theme";
|
||||
|
||||
describe("createMarkdownStyles", () => {
|
||||
it("applies shrink-and-wrap constraints to long markdown text and links", () => {
|
||||
const styles = createMarkdownStyles(darkTheme);
|
||||
|
||||
expect(styles.body).toMatchObject({
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
width: "100%",
|
||||
});
|
||||
|
||||
expect(styles.paragraph).toMatchObject({
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
width: "100%",
|
||||
flexWrap: "wrap",
|
||||
});
|
||||
|
||||
expect(styles.text).toMatchObject({
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
overflowWrap: "anywhere",
|
||||
});
|
||||
|
||||
expect(styles.link).toMatchObject({
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
overflowWrap: "anywhere",
|
||||
});
|
||||
|
||||
expect(styles.blocklink).toMatchObject({
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
overflowWrap: "anywhere",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -18,10 +18,16 @@ export function createMarkdownStyles(theme: Theme) {
|
||||
color: theme.colors.foreground,
|
||||
fontSize: theme.fontSize.base,
|
||||
lineHeight: 22,
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
width: "100%" as const,
|
||||
},
|
||||
|
||||
text: {
|
||||
color: theme.colors.foreground,
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
overflowWrap: "anywhere" as const,
|
||||
},
|
||||
|
||||
paragraph: {
|
||||
@@ -31,6 +37,9 @@ export function createMarkdownStyles(theme: Theme) {
|
||||
flexDirection: "row" as const,
|
||||
alignItems: "flex-start" as const,
|
||||
justifyContent: "flex-start" as const,
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
width: "100%" as const,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
@@ -117,13 +126,19 @@ export function createMarkdownStyles(theme: Theme) {
|
||||
},
|
||||
|
||||
link: {
|
||||
color: theme.colors.primary,
|
||||
textDecorationLine: "underline" as const,
|
||||
color: theme.colors.accentBright,
|
||||
textDecorationLine: "none" as const,
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
overflowWrap: "anywhere" as const,
|
||||
},
|
||||
|
||||
blocklink: {
|
||||
color: theme.colors.primary,
|
||||
textDecorationLine: "underline" as const,
|
||||
color: theme.colors.accentBright,
|
||||
textDecorationLine: "none" as const,
|
||||
flexShrink: 1,
|
||||
minWidth: 0,
|
||||
overflowWrap: "anywhere" as const,
|
||||
},
|
||||
|
||||
// =========================================================================
|
||||
@@ -135,7 +150,8 @@ export function createMarkdownStyles(theme: Theme) {
|
||||
color: theme.colors.foreground,
|
||||
paddingHorizontal: theme.spacing[1],
|
||||
paddingVertical: 2,
|
||||
borderRadius: theme.borderRadius.sm,
|
||||
borderRadius: theme.borderRadius.md,
|
||||
borderWidth: 0,
|
||||
fontFamily: Fonts.mono,
|
||||
fontSize: theme.fontSize.sm,
|
||||
},
|
||||
|
||||
@@ -123,6 +123,7 @@ const lightSemanticColors = {
|
||||
|
||||
// Brand
|
||||
accent: "#20744A",
|
||||
accentBright: "#239956",
|
||||
accentForeground: "#ffffff",
|
||||
|
||||
// Semantic
|
||||
@@ -192,6 +193,7 @@ const darkSemanticColors = {
|
||||
|
||||
// Brand
|
||||
accent: "#20744A",
|
||||
accentBright: "#7ccba0",
|
||||
accentForeground: "#ffffff",
|
||||
|
||||
// Semantic
|
||||
|
||||
@@ -2,9 +2,6 @@ import { StyleSheet } from "react-native-unistyles";
|
||||
// import { UnistylesRuntime } from "react-native-unistyles";
|
||||
import { lightTheme, darkTheme } from "./theme";
|
||||
|
||||
console.log("[Unistyles] Configuring...");
|
||||
|
||||
// // Configure Unistyles with adaptive themes
|
||||
StyleSheet.configure({
|
||||
themes: {
|
||||
light: lightTheme,
|
||||
@@ -22,8 +19,6 @@ StyleSheet.configure({
|
||||
},
|
||||
});
|
||||
|
||||
console.log("[Unistyles] Configuration complete!");
|
||||
|
||||
// Type augmentation for TypeScript
|
||||
type AppThemes = {
|
||||
light: typeof lightTheme;
|
||||
@@ -43,6 +38,4 @@ declare module "react-native-unistyles" {
|
||||
export interface UnistylesBreakpoints extends AppBreakpoints {}
|
||||
}
|
||||
|
||||
console.log(lightTheme.colors.background);
|
||||
|
||||
// UnistylesRuntime.setRootViewBackgroundColor(lightTheme.colors.background);
|
||||
|
||||
22
packages/app/src/utils/app-visibility.ts
Normal file
22
packages/app/src/utils/app-visibility.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import { AppState, Platform } from "react-native";
|
||||
|
||||
export function getIsAppActivelyVisible(
|
||||
appState: string = AppState.currentState
|
||||
): boolean {
|
||||
if (appState !== "active") {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (Platform.OS !== "web") {
|
||||
return true;
|
||||
}
|
||||
|
||||
const documentVisible =
|
||||
typeof document === "undefined" || document.visibilityState === "visible";
|
||||
const windowFocused =
|
||||
typeof document === "undefined" ||
|
||||
typeof document.hasFocus !== "function" ||
|
||||
document.hasFocus();
|
||||
|
||||
return documentVisible && windowFocused;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseInlinePathToken } from "./inline-path";
|
||||
import { parseFileProtocolUrl, parseInlinePathToken } from "./inline-path";
|
||||
|
||||
describe("parseInlinePathToken", () => {
|
||||
it("returns null for plain paths (no line)", () => {
|
||||
@@ -31,3 +31,42 @@ describe("parseInlinePathToken", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseFileProtocolUrl", () => {
|
||||
it("parses file URLs with line fragments", () => {
|
||||
expect(
|
||||
parseFileProtocolUrl("file:///Users/test/project/src/app.tsx#L81")
|
||||
).toEqual({
|
||||
raw: "file:///Users/test/project/src/app.tsx#L81",
|
||||
path: "/Users/test/project/src/app.tsx",
|
||||
lineStart: 81,
|
||||
lineEnd: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("parses file URLs without line fragments", () => {
|
||||
expect(
|
||||
parseFileProtocolUrl("file:///Users/test/project/src/app.tsx")
|
||||
).toEqual({
|
||||
raw: "file:///Users/test/project/src/app.tsx",
|
||||
path: "/Users/test/project/src/app.tsx",
|
||||
lineStart: undefined,
|
||||
lineEnd: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("parses windows file URLs and line ranges", () => {
|
||||
expect(
|
||||
parseFileProtocolUrl("file:///C:/Users/test/project/src/app.tsx#L12-L20")
|
||||
).toEqual({
|
||||
raw: "file:///C:/Users/test/project/src/app.tsx#L12-L20",
|
||||
path: "C:/Users/test/project/src/app.tsx",
|
||||
lineStart: 12,
|
||||
lineEnd: 20,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects non-file URLs and invalid ranges", () => {
|
||||
expect(parseFileProtocolUrl("https://example.com/test.ts#L10")).toBeNull();
|
||||
expect(parseFileProtocolUrl("file:///Users/test/project/src/app.tsx#L20-L12")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,8 @@ export interface InlinePathTarget {
|
||||
lineEnd?: number;
|
||||
}
|
||||
|
||||
const FILE_PROTOCOL = "file:";
|
||||
|
||||
function normalizePathToken(value: string): string | null {
|
||||
const trimmed = value
|
||||
.trim()
|
||||
@@ -79,3 +81,64 @@ export function parseInlinePathToken(value: string): InlinePathTarget | null {
|
||||
};
|
||||
}
|
||||
|
||||
export function parseFileProtocolUrl(value: string): InlinePathTarget | null {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let parsedUrl: URL;
|
||||
try {
|
||||
parsedUrl = new URL(trimmed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (parsedUrl.protocol !== FILE_PROTOCOL) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalizedPath = normalizeFileUrlPath(parsedUrl.pathname);
|
||||
if (!normalizedPath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const rawFragment = parsedUrl.hash.startsWith("#")
|
||||
? parsedUrl.hash.slice(1)
|
||||
: parsedUrl.hash;
|
||||
const lineMatch = rawFragment.match(/^L([0-9]+)(?:-L?([0-9]+))?$/i);
|
||||
const lineStart = lineMatch?.[1] ? parseInt(lineMatch[1], 10) : undefined;
|
||||
const lineEnd = lineMatch?.[2] ? parseInt(lineMatch[2], 10) : undefined;
|
||||
|
||||
if (
|
||||
(lineStart !== undefined && (!Number.isFinite(lineStart) || lineStart <= 0)) ||
|
||||
(lineEnd !== undefined && (!Number.isFinite(lineEnd) || lineEnd <= 0)) ||
|
||||
(lineStart !== undefined && lineEnd !== undefined && lineEnd < lineStart)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
raw: value,
|
||||
path: normalizedPath,
|
||||
lineStart,
|
||||
lineEnd,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeFileUrlPath(pathname: string): string | null {
|
||||
if (!pathname) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const decoded = decodeURIComponent(pathname).replace(/\\/g, "/");
|
||||
if (!decoded) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (/^\/[A-Za-z]:\//.test(decoded)) {
|
||||
return decoded.slice(1);
|
||||
}
|
||||
|
||||
return decoded;
|
||||
}
|
||||
|
||||
@@ -71,6 +71,7 @@ describe("sendOsNotification", () => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.doUnmock("@/desktop/notifications/desktop-notifications");
|
||||
vi.doUnmock("react-native");
|
||||
vi.restoreAllMocks();
|
||||
vi.resetModules();
|
||||
@@ -167,18 +168,13 @@ describe("sendOsNotification", () => {
|
||||
expect(assign).toHaveBeenCalledWith("/h/srv%20with%20space/agent/agent%2F1");
|
||||
});
|
||||
|
||||
it("uses Tauri notification module when available", async () => {
|
||||
const isPermissionGranted = vi.fn(async () => false);
|
||||
const requestPermission = vi.fn(async () => "granted");
|
||||
const sendNotification = vi.fn(async () => undefined);
|
||||
it("uses the desktop notification bridge when Tauri is available", async () => {
|
||||
const sendDesktopNotification = vi.fn(async () => true);
|
||||
vi.doMock("@/desktop/notifications/desktop-notifications", () => ({
|
||||
sendDesktopNotification,
|
||||
}));
|
||||
|
||||
(globalThis as { __TAURI__?: unknown }).__TAURI__ = {
|
||||
notification: {
|
||||
isPermissionGranted,
|
||||
requestPermission,
|
||||
sendNotification,
|
||||
},
|
||||
};
|
||||
(globalThis as { __TAURI__?: unknown }).__TAURI__ = {};
|
||||
|
||||
const { sendOsNotification } = await loadModuleForPlatform("web");
|
||||
const sent = await sendOsNotification({
|
||||
@@ -188,11 +184,10 @@ describe("sendOsNotification", () => {
|
||||
});
|
||||
|
||||
expect(sent).toBe(true);
|
||||
expect(isPermissionGranted).toHaveBeenCalledTimes(1);
|
||||
expect(requestPermission).toHaveBeenCalledTimes(1);
|
||||
expect(sendNotification).toHaveBeenCalledWith({
|
||||
expect(sendDesktopNotification).toHaveBeenCalledWith({
|
||||
title: "Agent finished",
|
||||
body: "Done",
|
||||
data: { serverId: "srv-1", agentId: "agent-1" },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { Platform } from "react-native";
|
||||
import { sendDesktopNotification } from "@/desktop/notifications/desktop-notifications";
|
||||
import { buildNotificationRoute } from "./notification-routing";
|
||||
import { getTauri, type TauriNotificationApi } from "@/utils/tauri";
|
||||
|
||||
@@ -14,13 +15,16 @@ export type WebNotificationClickDetail = {
|
||||
|
||||
type WebNotificationInstance = {
|
||||
onclick?: ((event: Event) => void) | null;
|
||||
onclose?: ((event: Event) => void) | null;
|
||||
addEventListener?: (type: string, listener: (event: Event) => void) => void;
|
||||
removeEventListener?: (type: string, listener: (event: Event) => void) => void;
|
||||
close?: () => void;
|
||||
};
|
||||
|
||||
export const WEB_NOTIFICATION_CLICK_EVENT = "paseo:web-notification-click";
|
||||
|
||||
let permissionRequest: Promise<boolean> | null = null;
|
||||
const activeWebNotifications = new Set<WebNotificationInstance>();
|
||||
|
||||
function getTauriNotificationModule(): TauriNotificationApi | null {
|
||||
if (Platform.OS !== "web") {
|
||||
@@ -82,7 +86,6 @@ async function ensureTauriNotificationPermission(
|
||||
try {
|
||||
const granted = await notificationModule.isPermissionGranted();
|
||||
if (granted) {
|
||||
console.log("[OSNotifications][Tauri] Permission already granted");
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
@@ -102,7 +105,6 @@ async function ensureTauriNotificationPermission(
|
||||
|
||||
try {
|
||||
const result = await notificationModule.requestPermission();
|
||||
console.log("[OSNotifications][Tauri] requestPermission result:", result);
|
||||
return result === "granted";
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
@@ -113,35 +115,6 @@ async function ensureTauriNotificationPermission(
|
||||
}
|
||||
}
|
||||
|
||||
async function sendTauriNotification(
|
||||
payload: OsNotificationPayload,
|
||||
notificationModule: TauriNotificationApi
|
||||
): Promise<boolean> {
|
||||
if (typeof notificationModule.sendNotification !== "function") {
|
||||
console.warn(
|
||||
"[OSNotifications][Tauri] notification.sendNotification is unavailable"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
const granted = await ensureTauriNotificationPermission(notificationModule);
|
||||
if (!granted) {
|
||||
console.log("[OSNotifications][Tauri] Permission not granted");
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
await notificationModule.sendNotification({
|
||||
title: payload.title,
|
||||
body: payload.body,
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn("[OSNotifications][Tauri] Failed to send notification", error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function dispatchWebNotificationClick(detail: WebNotificationClickDetail): boolean {
|
||||
const dispatch = (globalThis as { dispatchEvent?: (event: Event) => boolean }).dispatchEvent;
|
||||
const CustomEventConstructor = (globalThis as { CustomEvent?: typeof CustomEvent })
|
||||
@@ -183,6 +156,12 @@ function attachWebClickHandler(
|
||||
notification: WebNotificationInstance,
|
||||
data: Record<string, unknown> | undefined
|
||||
): void {
|
||||
activeWebNotifications.add(notification);
|
||||
|
||||
const cleanup = () => {
|
||||
activeWebNotifications.delete(notification);
|
||||
};
|
||||
|
||||
const onClick = () => {
|
||||
const focus = (globalThis as { focus?: () => void }).focus;
|
||||
if (typeof focus === "function") {
|
||||
@@ -194,17 +173,24 @@ function attachWebClickHandler(
|
||||
fallbackNavigateToNotificationTarget(data);
|
||||
}
|
||||
|
||||
cleanup();
|
||||
if (typeof notification.close === "function") {
|
||||
notification.close();
|
||||
}
|
||||
};
|
||||
|
||||
const onClose = () => {
|
||||
cleanup();
|
||||
};
|
||||
|
||||
if (typeof notification.addEventListener === "function") {
|
||||
notification.addEventListener("click", onClick);
|
||||
notification.addEventListener("close", onClose);
|
||||
return;
|
||||
}
|
||||
|
||||
notification.onclick = onClick;
|
||||
notification.onclose = onClose;
|
||||
}
|
||||
|
||||
export async function sendOsNotification(
|
||||
@@ -215,34 +201,22 @@ export async function sendOsNotification(
|
||||
return false;
|
||||
}
|
||||
|
||||
const tauriNotification = getTauriNotificationModule();
|
||||
if (tauriNotification) {
|
||||
console.log("[OSNotifications] Using Tauri notification module");
|
||||
return await sendTauriNotification(payload, tauriNotification);
|
||||
if (getTauri()) {
|
||||
return await sendDesktopNotification(payload);
|
||||
}
|
||||
|
||||
const NotificationConstructor = getWebNotificationConstructor();
|
||||
if (!NotificationConstructor) {
|
||||
console.log(
|
||||
"[OSNotifications][Web] Notification constructor unavailable",
|
||||
typeof window !== "undefined" && window.location
|
||||
? window.location.origin
|
||||
: "unknown-origin"
|
||||
);
|
||||
return false;
|
||||
if (NotificationConstructor) {
|
||||
const granted = await ensureNotificationPermission();
|
||||
if (granted) {
|
||||
const notification = new NotificationConstructor(payload.title, {
|
||||
body: payload.body,
|
||||
data: payload.data,
|
||||
}) as WebNotificationInstance;
|
||||
attachWebClickHandler(notification, payload.data);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
const granted = await ensureOsNotificationPermission();
|
||||
if (!granted) {
|
||||
console.log(
|
||||
"[OSNotifications][Web] Permission not granted:",
|
||||
NotificationConstructor.permission
|
||||
);
|
||||
return false;
|
||||
}
|
||||
const notification = new NotificationConstructor(payload.title, {
|
||||
body: payload.body,
|
||||
data: payload.data,
|
||||
}) as WebNotificationInstance;
|
||||
attachWebClickHandler(notification, payload.data);
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
88
packages/app/src/utils/pcm16-wav.ts
Normal file
88
packages/app/src/utils/pcm16-wav.ts
Normal file
@@ -0,0 +1,88 @@
|
||||
export interface Pcm16Wav {
|
||||
sampleRate: number;
|
||||
samples: Int16Array;
|
||||
}
|
||||
|
||||
export function parsePcm16Wav(buffer: ArrayBuffer): Pcm16Wav | null {
|
||||
if (buffer.byteLength < 44) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const view = new DataView(buffer);
|
||||
|
||||
function readAscii(offset: number, length: number): string {
|
||||
let out = "";
|
||||
for (let i = 0; i < length; i += 1) {
|
||||
out += String.fromCharCode(view.getUint8(offset + i));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
if (readAscii(0, 4) !== "RIFF" || readAscii(8, 4) !== "WAVE") {
|
||||
return null;
|
||||
}
|
||||
|
||||
let offset = 12;
|
||||
let channels = 0;
|
||||
let sampleRate = 0;
|
||||
let bitsPerSample = 0;
|
||||
let dataOffset = 0;
|
||||
let dataSize = 0;
|
||||
|
||||
while (offset + 8 <= buffer.byteLength) {
|
||||
const chunkId = readAscii(offset, 4);
|
||||
const chunkSize = view.getUint32(offset + 4, true);
|
||||
const chunkDataOffset = offset + 8;
|
||||
|
||||
if (chunkDataOffset + chunkSize > buffer.byteLength) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (chunkId === "fmt " && chunkSize >= 16) {
|
||||
const audioFormat = view.getUint16(chunkDataOffset, true);
|
||||
channels = view.getUint16(chunkDataOffset + 2, true);
|
||||
sampleRate = view.getUint32(chunkDataOffset + 4, true);
|
||||
bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
|
||||
if (audioFormat !== 1) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (chunkId === "data") {
|
||||
dataOffset = chunkDataOffset;
|
||||
dataSize = chunkSize;
|
||||
break;
|
||||
}
|
||||
|
||||
offset = chunkDataOffset + chunkSize + (chunkSize % 2);
|
||||
}
|
||||
|
||||
if (!dataOffset || dataSize <= 0 || sampleRate <= 0 || bitsPerSample !== 16 || channels <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sampleCount = Math.floor(dataSize / 2);
|
||||
const interleaved = new Int16Array(buffer, dataOffset, sampleCount);
|
||||
|
||||
if (channels === 1) {
|
||||
return {
|
||||
sampleRate,
|
||||
samples: new Int16Array(interleaved),
|
||||
};
|
||||
}
|
||||
|
||||
const frameCount = Math.floor(interleaved.length / channels);
|
||||
const mono = new Int16Array(frameCount);
|
||||
for (let frame = 0; frame < frameCount; frame += 1) {
|
||||
let sum = 0;
|
||||
for (let channel = 0; channel < channels; channel += 1) {
|
||||
sum += interleaved[frame * channels + channel] ?? 0;
|
||||
}
|
||||
mono[frame] = Math.round(sum / channels);
|
||||
}
|
||||
|
||||
return {
|
||||
sampleRate,
|
||||
samples: mono,
|
||||
};
|
||||
}
|
||||
@@ -34,7 +34,7 @@ export function attachConsole(): void {
|
||||
|
||||
console.log = (...args: unknown[]) => {
|
||||
originalLog.apply(console, args);
|
||||
forwardToRust(0, args); // debug
|
||||
forwardToRust(1, args); // info
|
||||
};
|
||||
console.info = (...args: unknown[]) => {
|
||||
originalInfo.apply(console, args);
|
||||
|
||||
59
packages/app/src/utils/tauri-daemon-transport.test.ts
Normal file
59
packages/app/src/utils/tauri-daemon-transport.test.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const tauriMock = vi.hoisted(() => {
|
||||
let listenerCleanup: (() => void) | null = null;
|
||||
|
||||
const connect = vi.fn(async () => {
|
||||
let listenerActive = false;
|
||||
|
||||
return {
|
||||
addListener: vi.fn(() => {
|
||||
listenerActive = true;
|
||||
listenerCleanup = vi.fn(() => {
|
||||
if (!listenerActive) {
|
||||
throw new TypeError(
|
||||
"undefined is not an object (evaluating 'listeners[eventId].handlerId')"
|
||||
);
|
||||
}
|
||||
listenerActive = false;
|
||||
});
|
||||
return listenerCleanup;
|
||||
}),
|
||||
send: vi.fn(async () => undefined),
|
||||
disconnect: vi.fn(async () => {
|
||||
listenerCleanup?.();
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
connect,
|
||||
getListenerCleanup: () => listenerCleanup,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./tauri", () => ({
|
||||
getTauri: () => ({
|
||||
websocket: {
|
||||
connect: tauriMock.connect,
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
describe("tauri-daemon-transport", () => {
|
||||
beforeEach(() => {
|
||||
tauriMock.connect.mockClear();
|
||||
});
|
||||
|
||||
it("does not unregister the websocket listener twice during close", async () => {
|
||||
const mod = await import("./tauri-daemon-transport");
|
||||
const transportFactory = mod.createTauriWebSocketTransportFactory();
|
||||
expect(transportFactory).not.toBeNull();
|
||||
|
||||
const transport = transportFactory!({ url: "ws://localhost:6767/ws" });
|
||||
await Promise.resolve();
|
||||
|
||||
expect(() => transport.close()).not.toThrow();
|
||||
expect(tauriMock.getListenerCleanup()).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -229,14 +229,10 @@ export function createTauriWebSocketTransportFactory(): DaemonTransportFactory |
|
||||
return {
|
||||
...transport,
|
||||
close: (code?: number, reason?: string) => {
|
||||
wsListenerCleanup = null;
|
||||
transport.close(code, reason);
|
||||
disposed = true;
|
||||
try {
|
||||
wsListenerCleanup?.();
|
||||
} catch {
|
||||
// no-op
|
||||
}
|
||||
wsListenerCleanup = null;
|
||||
ws = null;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
@@ -7,8 +7,22 @@ export interface TauriDialogAskOptions {
|
||||
kind?: "info" | "warning" | "error";
|
||||
}
|
||||
|
||||
export interface TauriDialogOpenOptions {
|
||||
title?: string;
|
||||
defaultPath?: string;
|
||||
directory?: boolean;
|
||||
multiple?: boolean;
|
||||
filters?: Array<{
|
||||
name: string;
|
||||
extensions: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface TauriDialogApi {
|
||||
ask?: (message: string, options?: TauriDialogAskOptions) => Promise<boolean>;
|
||||
open?: (
|
||||
options?: TauriDialogOpenOptions
|
||||
) => Promise<string | string[] | null>;
|
||||
}
|
||||
|
||||
export interface TauriCoreApi {
|
||||
|
||||
2
packages/app/src/utils/thinking-tone.native-pcm.ts
Normal file
2
packages/app/src/utils/thinking-tone.native-pcm.ts
Normal file
File diff suppressed because one or more lines are too long
76
packages/app/src/utils/thinking-tone.test.ts
Normal file
76
packages/app/src/utils/thinking-tone.test.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parsePcm16Wav } from "@/utils/pcm16-wav";
|
||||
|
||||
interface BuildWavInput {
|
||||
channels?: number;
|
||||
sampleRate?: number;
|
||||
samples: number[];
|
||||
}
|
||||
|
||||
function buildPcm16Wav(input: BuildWavInput): ArrayBuffer {
|
||||
const channels = input.channels ?? 1;
|
||||
const sampleRate = input.sampleRate ?? 24000;
|
||||
const bytesPerSample = 2;
|
||||
const dataSize = input.samples.length * bytesPerSample;
|
||||
const buffer = new ArrayBuffer(44 + dataSize);
|
||||
const view = new DataView(buffer);
|
||||
|
||||
function writeAscii(offset: number, value: string): void {
|
||||
for (let i = 0; i < value.length; i += 1) {
|
||||
view.setUint8(offset + i, value.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
writeAscii(0, "RIFF");
|
||||
view.setUint32(4, 36 + dataSize, true);
|
||||
writeAscii(8, "WAVE");
|
||||
writeAscii(12, "fmt ");
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, channels, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * channels * bytesPerSample, true);
|
||||
view.setUint16(32, channels * bytesPerSample, true);
|
||||
view.setUint16(34, 16, true);
|
||||
writeAscii(36, "data");
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
input.samples.forEach((sample, index) => {
|
||||
view.setInt16(44 + index * bytesPerSample, sample, true);
|
||||
});
|
||||
|
||||
return buffer;
|
||||
}
|
||||
|
||||
describe("parsePcm16Wav", () => {
|
||||
it("parses mono PCM16 wav data", () => {
|
||||
const buffer = buildPcm16Wav({
|
||||
sampleRate: 16000,
|
||||
samples: [100, -200, 300],
|
||||
});
|
||||
|
||||
const parsed = parsePcm16Wav(buffer);
|
||||
|
||||
expect(parsed).not.toBeNull();
|
||||
expect(parsed?.sampleRate).toBe(16000);
|
||||
expect(Array.from(parsed?.samples ?? [])).toEqual([100, -200, 300]);
|
||||
});
|
||||
|
||||
it("downmixes multichannel PCM16 wav data to mono", () => {
|
||||
const buffer = buildPcm16Wav({
|
||||
channels: 2,
|
||||
samples: [1000, -1000, 3000, 1000],
|
||||
});
|
||||
|
||||
const parsed = parsePcm16Wav(buffer);
|
||||
|
||||
expect(parsed).not.toBeNull();
|
||||
expect(Array.from(parsed?.samples ?? [])).toEqual([0, 2000]);
|
||||
});
|
||||
|
||||
it("rejects non-wav payloads", () => {
|
||||
const buffer = new TextEncoder().encode("not-a-wav").buffer;
|
||||
|
||||
expect(parsePcm16Wav(buffer)).toBeNull();
|
||||
});
|
||||
});
|
||||
33
packages/app/src/utils/thinking-tone.ts
Normal file
33
packages/app/src/utils/thinking-tone.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import { Asset } from "expo-asset";
|
||||
import { File } from "expo-file-system";
|
||||
import { Platform } from "react-native";
|
||||
export { parsePcm16Wav, type Pcm16Wav } from "@/utils/pcm16-wav";
|
||||
|
||||
export const THINKING_TONE_REPEAT_GAP_MS = 350;
|
||||
|
||||
let thinkingToneArrayBufferPromise: Promise<ArrayBuffer> | null = null;
|
||||
|
||||
async function readThinkingToneArrayBuffer(): Promise<ArrayBuffer> {
|
||||
const toneModule = require("../../assets/audio/thinking-tone.wav");
|
||||
const asset = Asset.fromModule(toneModule);
|
||||
|
||||
if (Platform.OS === "web") {
|
||||
const response = await fetch(asset.uri);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch thinking tone asset: ${response.status}`);
|
||||
}
|
||||
return await response.arrayBuffer();
|
||||
}
|
||||
|
||||
const resolvedAsset = asset.localUri ? asset : await asset.downloadAsync();
|
||||
const fileUri = resolvedAsset.localUri ?? resolvedAsset.uri;
|
||||
const file = new File(fileUri);
|
||||
return await file.arrayBuffer();
|
||||
}
|
||||
|
||||
export async function loadThinkingToneArrayBuffer(): Promise<ArrayBuffer> {
|
||||
if (!thinkingToneArrayBufferPromise) {
|
||||
thinkingToneArrayBufferPromise = readThinkingToneArrayBuffer();
|
||||
}
|
||||
return await thinkingToneArrayBufferPromise;
|
||||
}
|
||||
@@ -140,4 +140,38 @@ describe("tool-call-display", () => {
|
||||
|
||||
expect(display.errorText).toBe('{\n "message": "boom"\n}');
|
||||
});
|
||||
|
||||
it("shows terminal interaction with only the fixed label when no command is available", () => {
|
||||
const display = buildToolCallDisplayModel({
|
||||
name: "terminal",
|
||||
status: "completed",
|
||||
error: null,
|
||||
detail: {
|
||||
type: "plain_text",
|
||||
icon: "square_terminal",
|
||||
},
|
||||
});
|
||||
|
||||
expect(display).toEqual({
|
||||
displayName: "Interacted with terminal",
|
||||
});
|
||||
});
|
||||
|
||||
it("shows terminal interaction command as the summary when available", () => {
|
||||
const display = buildToolCallDisplayModel({
|
||||
name: "terminal",
|
||||
status: "completed",
|
||||
error: null,
|
||||
detail: {
|
||||
type: "plain_text",
|
||||
label: "npm run test",
|
||||
icon: "square_terminal",
|
||||
},
|
||||
});
|
||||
|
||||
expect(display).toEqual({
|
||||
displayName: "Interacted with terminal",
|
||||
summary: "npm run test",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
29
packages/app/src/voice/audio-engine-types.ts
Normal file
29
packages/app/src/voice/audio-engine-types.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
export interface AudioEngineCallbacks {
|
||||
onCaptureData(pcm: Uint8Array): void;
|
||||
onVolumeLevel(level: number): void;
|
||||
onError?(error: Error): void;
|
||||
}
|
||||
|
||||
export interface AudioPlaybackSource {
|
||||
arrayBuffer(): Promise<ArrayBuffer>;
|
||||
size: number;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface AudioEngine {
|
||||
initialize(): Promise<void>;
|
||||
destroy(): Promise<void>;
|
||||
|
||||
startCapture(): Promise<void>;
|
||||
stopCapture(): Promise<void>;
|
||||
toggleMute(): boolean;
|
||||
isMuted(): boolean;
|
||||
|
||||
play(audio: AudioPlaybackSource): Promise<number>;
|
||||
stop(): void;
|
||||
clearQueue(): void;
|
||||
isPlaying(): boolean;
|
||||
|
||||
playLooping(pcm: Uint8Array, gapMs: number): void;
|
||||
stopLooping(): void;
|
||||
}
|
||||
1
packages/app/src/voice/audio-engine.d.ts
vendored
Normal file
1
packages/app/src/voice/audio-engine.d.ts
vendored
Normal file
@@ -0,0 +1 @@
|
||||
export * from "./audio-engine.native";
|
||||
402
packages/app/src/voice/audio-engine.native.ts
Normal file
402
packages/app/src/voice/audio-engine.native.ts
Normal file
@@ -0,0 +1,402 @@
|
||||
import {
|
||||
addExpoTwoWayAudioEventListener,
|
||||
getMicrophonePermissionsAsync,
|
||||
initialize,
|
||||
playPCMData,
|
||||
requestMicrophonePermissionsAsync,
|
||||
resumePlayback,
|
||||
stopPlayback,
|
||||
tearDown,
|
||||
toggleRecording,
|
||||
} from "@boudra/expo-two-way-audio";
|
||||
import {
|
||||
THINKING_TONE_REPEAT_GAP_MS,
|
||||
} from "@/utils/thinking-tone";
|
||||
import {
|
||||
THINKING_TONE_NATIVE_PCM_BASE64,
|
||||
THINKING_TONE_NATIVE_PCM_DURATION_MS,
|
||||
} from "@/utils/thinking-tone.native-pcm";
|
||||
import type {
|
||||
AudioEngine,
|
||||
AudioEngineCallbacks,
|
||||
AudioPlaybackSource,
|
||||
} from "@/voice/audio-engine-types";
|
||||
import { Buffer } from "buffer";
|
||||
|
||||
interface QueuedAudio {
|
||||
audio: AudioPlaybackSource;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
interface CuePcm {
|
||||
pcm16k: Uint8Array;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
interface AudioEngineTraceOptions {
|
||||
traceLabel?: string;
|
||||
}
|
||||
|
||||
function resamplePcm16(
|
||||
pcm: Uint8Array,
|
||||
fromRate: number,
|
||||
toRate: number
|
||||
): Uint8Array {
|
||||
if (fromRate === toRate) {
|
||||
return pcm;
|
||||
}
|
||||
|
||||
const inputSamples = Math.floor(pcm.length / 2);
|
||||
const outputSamples = Math.floor((inputSamples * toRate) / fromRate);
|
||||
const output = new Uint8Array(outputSamples * 2);
|
||||
const ratio = fromRate / toRate;
|
||||
|
||||
const readInt16 = (sampleIndex: number): number => {
|
||||
const offset = sampleIndex * 2;
|
||||
if (offset + 1 >= pcm.length) {
|
||||
return 0;
|
||||
}
|
||||
const lo = pcm[offset]!;
|
||||
const hi = pcm[offset + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value -= 0x10000;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const writeInt16 = (sampleIndex: number, value: number): void => {
|
||||
const clamped = Math.max(-32768, Math.min(32767, Math.round(value)));
|
||||
const offset = sampleIndex * 2;
|
||||
output[offset] = clamped & 0xff;
|
||||
output[offset + 1] = (clamped >> 8) & 0xff;
|
||||
};
|
||||
|
||||
for (let i = 0; i < outputSamples; i += 1) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const frac = sourceIndex - i0;
|
||||
const s0 = readInt16(i0);
|
||||
const s1 = readInt16(Math.min(inputSamples - 1, i0 + 1));
|
||||
writeInt16(i, s0 + (s1 - s0) * frac);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
}
|
||||
|
||||
export function createAudioEngine(
|
||||
callbacks: AudioEngineCallbacks,
|
||||
_options?: AudioEngineTraceOptions
|
||||
): AudioEngine {
|
||||
const refs: {
|
||||
initialized: boolean;
|
||||
captureActive: boolean;
|
||||
muted: boolean;
|
||||
queue: QueuedAudio[];
|
||||
processingQueue: boolean;
|
||||
playbackTimeout: ReturnType<typeof setTimeout> | null;
|
||||
activePlayback: {
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
settled: boolean;
|
||||
} | null;
|
||||
looping: {
|
||||
active: boolean;
|
||||
token: number;
|
||||
timeout: ReturnType<typeof setTimeout> | null;
|
||||
};
|
||||
thinkingTone: CuePcm | null;
|
||||
destroyed: boolean;
|
||||
} = {
|
||||
initialized: false,
|
||||
captureActive: false,
|
||||
muted: false,
|
||||
queue: [],
|
||||
processingQueue: false,
|
||||
playbackTimeout: null,
|
||||
activePlayback: null,
|
||||
looping: {
|
||||
active: false,
|
||||
token: 0,
|
||||
timeout: null,
|
||||
},
|
||||
thinkingTone: null,
|
||||
destroyed: false,
|
||||
};
|
||||
|
||||
const microphoneSubscription = addExpoTwoWayAudioEventListener(
|
||||
"onMicrophoneData",
|
||||
(event) => {
|
||||
if (!refs.captureActive || refs.muted) {
|
||||
return;
|
||||
}
|
||||
callbacks.onCaptureData(event.data);
|
||||
}
|
||||
);
|
||||
|
||||
const volumeSubscription = addExpoTwoWayAudioEventListener(
|
||||
"onInputVolumeLevelData",
|
||||
(event) => {
|
||||
if (!refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
const level = refs.muted ? 0 : event.data;
|
||||
callbacks.onVolumeLevel(level);
|
||||
}
|
||||
);
|
||||
|
||||
async function ensureInitialized(): Promise<void> {
|
||||
if (refs.initialized) {
|
||||
return;
|
||||
}
|
||||
await initialize();
|
||||
refs.initialized = true;
|
||||
}
|
||||
|
||||
async function ensureMicrophonePermission(): Promise<void> {
|
||||
let permission = await getMicrophonePermissionsAsync().catch(() => null);
|
||||
if (!permission?.granted) {
|
||||
permission = await requestMicrophonePermissionsAsync().catch(() => null);
|
||||
}
|
||||
if (!permission?.granted) {
|
||||
throw new Error(
|
||||
"Microphone permission is required to capture audio. Please enable microphone access in system settings."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureThinkingTone(): Promise<CuePcm> {
|
||||
if (refs.thinkingTone) {
|
||||
return refs.thinkingTone;
|
||||
}
|
||||
const pcm16k = Buffer.from(THINKING_TONE_NATIVE_PCM_BASE64, "base64");
|
||||
const durationMs = THINKING_TONE_NATIVE_PCM_DURATION_MS;
|
||||
refs.thinkingTone = { pcm16k, durationMs };
|
||||
return refs.thinkingTone;
|
||||
}
|
||||
|
||||
function clearPlaybackTimeout(): void {
|
||||
if (refs.playbackTimeout) {
|
||||
clearTimeout(refs.playbackTimeout);
|
||||
refs.playbackTimeout = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
|
||||
await ensureInitialized();
|
||||
resumePlayback();
|
||||
|
||||
return await new Promise<number>(async (resolve, reject) => {
|
||||
refs.activePlayback = { resolve, reject, settled: false };
|
||||
|
||||
try {
|
||||
const arrayBuffer = await audio.arrayBuffer();
|
||||
const pcm = new Uint8Array(arrayBuffer);
|
||||
const inputRate = parsePcmSampleRate(audio.type || "") ?? 24000;
|
||||
const pcm16k = resamplePcm16(pcm, inputRate, 16000);
|
||||
const durationSec = pcm16k.length / 2 / 16000;
|
||||
|
||||
playPCMData(pcm16k);
|
||||
clearPlaybackTimeout();
|
||||
refs.playbackTimeout = setTimeout(() => {
|
||||
clearPlaybackTimeout();
|
||||
const active = refs.activePlayback;
|
||||
if (!active || active.settled) {
|
||||
return;
|
||||
}
|
||||
active.settled = true;
|
||||
refs.activePlayback = null;
|
||||
resolve(durationSec);
|
||||
}, durationSec * 1000);
|
||||
} catch (error) {
|
||||
clearPlaybackTimeout();
|
||||
const active = refs.activePlayback;
|
||||
if (active && !active.settled) {
|
||||
active.settled = true;
|
||||
refs.activePlayback = null;
|
||||
reject(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (refs.processingQueue || refs.queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
refs.processingQueue = true;
|
||||
while (refs.queue.length > 0) {
|
||||
const item = refs.queue.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audio);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
}
|
||||
|
||||
function stopLooping(): void {
|
||||
refs.looping.active = false;
|
||||
refs.looping.token += 1;
|
||||
if (refs.looping.timeout) {
|
||||
clearTimeout(refs.looping.timeout);
|
||||
refs.looping.timeout = null;
|
||||
}
|
||||
stopPlayback();
|
||||
}
|
||||
|
||||
return {
|
||||
async initialize() {
|
||||
await ensureInitialized();
|
||||
await ensureThinkingTone();
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
if (refs.destroyed) {
|
||||
return;
|
||||
}
|
||||
refs.destroyed = true;
|
||||
stopLooping();
|
||||
this.stop();
|
||||
this.clearQueue();
|
||||
if (refs.captureActive) {
|
||||
toggleRecording(false);
|
||||
refs.captureActive = false;
|
||||
}
|
||||
clearPlaybackTimeout();
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
if (refs.initialized) {
|
||||
tearDown();
|
||||
refs.initialized = false;
|
||||
}
|
||||
microphoneSubscription.remove();
|
||||
volumeSubscription.remove();
|
||||
},
|
||||
|
||||
async startCapture() {
|
||||
if (refs.captureActive) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureMicrophonePermission();
|
||||
await ensureInitialized();
|
||||
toggleRecording(true);
|
||||
refs.captureActive = true;
|
||||
} catch (error) {
|
||||
const wrapped = error instanceof Error ? error : new Error(String(error));
|
||||
callbacks.onError?.(wrapped);
|
||||
throw wrapped;
|
||||
}
|
||||
},
|
||||
|
||||
async stopCapture() {
|
||||
if (refs.captureActive) {
|
||||
toggleRecording(false);
|
||||
}
|
||||
refs.captureActive = false;
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
refs.muted = !refs.muted;
|
||||
if (refs.muted) {
|
||||
callbacks.onVolumeLevel(0);
|
||||
}
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
isMuted() {
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
async play(audio: AudioPlaybackSource) {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
refs.queue.push({ audio, resolve, reject });
|
||||
if (!refs.processingQueue) {
|
||||
void processQueue();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
stop() {
|
||||
stopPlayback();
|
||||
clearPlaybackTimeout();
|
||||
const active = refs.activePlayback;
|
||||
refs.activePlayback = null;
|
||||
if (active && !active.settled) {
|
||||
active.settled = true;
|
||||
active.reject(new Error("Playback stopped"));
|
||||
}
|
||||
},
|
||||
|
||||
clearQueue() {
|
||||
while (refs.queue.length > 0) {
|
||||
refs.queue.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
},
|
||||
|
||||
isPlaying() {
|
||||
return refs.activePlayback !== null;
|
||||
},
|
||||
|
||||
playLooping(audio, gapMs) {
|
||||
if (refs.looping.active) {
|
||||
return;
|
||||
}
|
||||
|
||||
refs.looping.active = true;
|
||||
const token = refs.looping.token + 1;
|
||||
refs.looping.token = token;
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
await ensureInitialized();
|
||||
const cue =
|
||||
audio.byteLength > 0
|
||||
? {
|
||||
pcm16k: audio,
|
||||
durationMs: (audio.byteLength / 2 / 16000) * 1000,
|
||||
}
|
||||
: await ensureThinkingTone();
|
||||
|
||||
const loop = () => {
|
||||
if (!refs.looping.active || refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
resumePlayback();
|
||||
playPCMData(cue.pcm16k);
|
||||
refs.looping.timeout = setTimeout(
|
||||
loop,
|
||||
cue.durationMs + (gapMs || THINKING_TONE_REPEAT_GAP_MS)
|
||||
);
|
||||
};
|
||||
|
||||
loop();
|
||||
} catch (error) {
|
||||
callbacks.onError?.(
|
||||
error instanceof Error ? error : new Error(String(error))
|
||||
);
|
||||
}
|
||||
})();
|
||||
},
|
||||
|
||||
stopLooping,
|
||||
};
|
||||
}
|
||||
548
packages/app/src/voice/audio-engine.web.ts
Normal file
548
packages/app/src/voice/audio-engine.web.ts
Normal file
@@ -0,0 +1,548 @@
|
||||
import { getTauri } from "@/utils/tauri";
|
||||
import {
|
||||
loadThinkingToneArrayBuffer,
|
||||
THINKING_TONE_REPEAT_GAP_MS,
|
||||
} from "@/utils/thinking-tone";
|
||||
import type {
|
||||
AudioEngine,
|
||||
AudioEngineCallbacks,
|
||||
AudioPlaybackSource,
|
||||
} from "@/voice/audio-engine-types";
|
||||
|
||||
interface QueuedAudio {
|
||||
audio: AudioPlaybackSource;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
function getAudioContextCtor(): (typeof AudioContext) | null {
|
||||
if (typeof window === "undefined") {
|
||||
return null;
|
||||
}
|
||||
const browserWindow = window as typeof window & {
|
||||
webkitAudioContext?: typeof AudioContext;
|
||||
};
|
||||
return browserWindow.AudioContext ?? browserWindow.webkitAudioContext ?? null;
|
||||
}
|
||||
|
||||
function floatToInt16(sample: number): number {
|
||||
const clamped = Math.max(-1, Math.min(1, sample));
|
||||
return clamped < 0 ? Math.round(clamped * 0x8000) : Math.round(clamped * 0x7fff);
|
||||
}
|
||||
|
||||
function resampleToPcm16(
|
||||
input: Float32Array,
|
||||
inputRate: number,
|
||||
outputRate: number
|
||||
): Uint8Array {
|
||||
if (input.length === 0) {
|
||||
return new Uint8Array(0);
|
||||
}
|
||||
|
||||
const ratio = inputRate / outputRate;
|
||||
const outputLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const output = new Int16Array(outputLength);
|
||||
for (let i = 0; i < outputLength; i += 1) {
|
||||
const sourceIndex = i * ratio;
|
||||
const i0 = Math.floor(sourceIndex);
|
||||
const i1 = Math.min(input.length - 1, i0 + 1);
|
||||
const frac = sourceIndex - i0;
|
||||
const sample = input[i0]! * (1 - frac) + input[i1]! * frac;
|
||||
output[i] = floatToInt16(sample);
|
||||
}
|
||||
|
||||
return new Uint8Array(output.buffer, output.byteOffset, output.byteLength);
|
||||
}
|
||||
|
||||
function parsePcmSampleRate(mimeType: string): number | null {
|
||||
const match = /rate=(\d+)/i.exec(mimeType);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const rate = Number(match[1]);
|
||||
return Number.isFinite(rate) && rate > 0 ? rate : null;
|
||||
}
|
||||
|
||||
function pcm16LeToAudioBuffer(
|
||||
context: AudioContext,
|
||||
bytes: Uint8Array,
|
||||
sampleRate: number
|
||||
): AudioBuffer {
|
||||
const sampleCount = Math.floor(bytes.length / 2);
|
||||
const audioBuffer = context.createBuffer(1, sampleCount, sampleRate);
|
||||
const channel = audioBuffer.getChannelData(0);
|
||||
for (let i = 0; i < sampleCount; i += 1) {
|
||||
const lo = bytes[i * 2]!;
|
||||
const hi = bytes[i * 2 + 1]!;
|
||||
let value = (hi << 8) | lo;
|
||||
if (value & 0x8000) {
|
||||
value -= 0x10000;
|
||||
}
|
||||
channel[i] = value / 0x8000;
|
||||
}
|
||||
return audioBuffer;
|
||||
}
|
||||
|
||||
async function decodeAudioData(
|
||||
context: AudioContext,
|
||||
buffer: ArrayBuffer
|
||||
): Promise<AudioBuffer> {
|
||||
const maybePromise = context.decodeAudioData(buffer.slice(0));
|
||||
if (maybePromise && typeof (maybePromise as Promise<AudioBuffer>).then === "function") {
|
||||
return maybePromise as Promise<AudioBuffer>;
|
||||
}
|
||||
return await new Promise<AudioBuffer>((resolve, reject) => {
|
||||
context.decodeAudioData(buffer.slice(0), resolve, reject);
|
||||
});
|
||||
}
|
||||
|
||||
export function createAudioEngine(
|
||||
callbacks: AudioEngineCallbacks,
|
||||
_options?: { traceLabel?: string }
|
||||
): AudioEngine {
|
||||
const refs: {
|
||||
playbackContext: AudioContext | null;
|
||||
captureContext: AudioContext | null;
|
||||
stream: MediaStream | null;
|
||||
source: MediaStreamAudioSourceNode | null;
|
||||
processor: ScriptProcessorNode | null;
|
||||
gain: GainNode | null;
|
||||
started: boolean;
|
||||
muted: boolean;
|
||||
queue: QueuedAudio[];
|
||||
processingQueue: boolean;
|
||||
activePlayback: {
|
||||
source: AudioBufferSourceNode;
|
||||
resolve: (duration: number) => void;
|
||||
reject: (error: Error) => void;
|
||||
settled: boolean;
|
||||
} | null;
|
||||
looping: {
|
||||
token: number;
|
||||
source: AudioBufferSourceNode | null;
|
||||
gain: GainNode | null;
|
||||
timeout: ReturnType<typeof setTimeout> | null;
|
||||
};
|
||||
thinkingTone: {
|
||||
bytes: Uint8Array | null;
|
||||
buffer: AudioBuffer | null;
|
||||
};
|
||||
} = {
|
||||
playbackContext: null,
|
||||
captureContext: null,
|
||||
stream: null,
|
||||
source: null,
|
||||
processor: null,
|
||||
gain: null,
|
||||
started: false,
|
||||
muted: false,
|
||||
queue: [],
|
||||
processingQueue: false,
|
||||
activePlayback: null,
|
||||
looping: {
|
||||
token: 0,
|
||||
source: null,
|
||||
gain: null,
|
||||
timeout: null,
|
||||
},
|
||||
thinkingTone: {
|
||||
bytes: null,
|
||||
buffer: null,
|
||||
},
|
||||
};
|
||||
|
||||
async function ensurePlaybackContext(): Promise<AudioContext> {
|
||||
if (refs.playbackContext) {
|
||||
if (refs.playbackContext.state === "suspended") {
|
||||
await refs.playbackContext.resume().catch(() => undefined);
|
||||
}
|
||||
return refs.playbackContext;
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
if (context.state === "suspended") {
|
||||
await context.resume().catch(() => undefined);
|
||||
}
|
||||
refs.playbackContext = context;
|
||||
return context;
|
||||
}
|
||||
|
||||
async function ensureCaptureContext(): Promise<AudioContext> {
|
||||
if (refs.captureContext) {
|
||||
if (refs.captureContext.state === "suspended") {
|
||||
await refs.captureContext.resume().catch(() => undefined);
|
||||
}
|
||||
return refs.captureContext;
|
||||
}
|
||||
|
||||
const AudioContextCtor = getAudioContextCtor();
|
||||
if (!AudioContextCtor) {
|
||||
throw new Error("AudioContext unavailable");
|
||||
}
|
||||
|
||||
const context = new AudioContextCtor();
|
||||
if (context.state === "suspended") {
|
||||
await context.resume().catch(() => undefined);
|
||||
}
|
||||
refs.captureContext = context;
|
||||
return context;
|
||||
}
|
||||
|
||||
async function ensureThinkingToneBytes(): Promise<Uint8Array> {
|
||||
if (refs.thinkingTone.bytes) {
|
||||
return refs.thinkingTone.bytes;
|
||||
}
|
||||
const wav = await loadThinkingToneArrayBuffer();
|
||||
refs.thinkingTone.bytes = new Uint8Array(wav);
|
||||
return refs.thinkingTone.bytes;
|
||||
}
|
||||
|
||||
async function ensureThinkingToneBuffer(): Promise<AudioBuffer> {
|
||||
if (refs.thinkingTone.buffer) {
|
||||
return refs.thinkingTone.buffer;
|
||||
}
|
||||
const context = await ensurePlaybackContext();
|
||||
const wav = await loadThinkingToneArrayBuffer();
|
||||
const audioBuffer = await decodeAudioData(context, wav);
|
||||
refs.thinkingTone.buffer = audioBuffer;
|
||||
return audioBuffer;
|
||||
}
|
||||
|
||||
function stopLooping(): void {
|
||||
refs.looping.token += 1;
|
||||
if (refs.looping.timeout) {
|
||||
clearTimeout(refs.looping.timeout);
|
||||
refs.looping.timeout = null;
|
||||
}
|
||||
if (refs.looping.source) {
|
||||
try {
|
||||
refs.looping.source.onended = null;
|
||||
refs.looping.source.stop();
|
||||
} catch {
|
||||
// Ignore best-effort stop errors.
|
||||
}
|
||||
refs.looping.source.disconnect();
|
||||
refs.looping.source = null;
|
||||
}
|
||||
if (refs.looping.gain) {
|
||||
refs.looping.gain.disconnect();
|
||||
refs.looping.gain = null;
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(audio: AudioPlaybackSource): Promise<number> {
|
||||
const context = await ensurePlaybackContext();
|
||||
const arrayBuffer = await audio.arrayBuffer();
|
||||
const type = (audio.type || "").toLowerCase();
|
||||
const audioBuffer = type.startsWith("audio/pcm")
|
||||
? pcm16LeToAudioBuffer(
|
||||
context,
|
||||
new Uint8Array(arrayBuffer),
|
||||
parsePcmSampleRate(type) ?? 24000
|
||||
)
|
||||
: await decodeAudioData(context, arrayBuffer);
|
||||
|
||||
const durationSec = audioBuffer.duration;
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(context.destination);
|
||||
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
refs.activePlayback = { source, resolve, reject, settled: false };
|
||||
|
||||
const settle = (fn: () => void) => {
|
||||
const active = refs.activePlayback;
|
||||
if (!active || active.source !== source || active.settled) {
|
||||
return;
|
||||
}
|
||||
active.settled = true;
|
||||
refs.activePlayback = null;
|
||||
fn();
|
||||
};
|
||||
|
||||
source.onended = () => {
|
||||
settle(() => resolve(durationSec));
|
||||
};
|
||||
|
||||
try {
|
||||
source.start();
|
||||
} catch (error) {
|
||||
settle(() =>
|
||||
reject(error instanceof Error ? error : new Error(String(error)))
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function processQueue(): Promise<void> {
|
||||
if (refs.processingQueue || refs.queue.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
refs.processingQueue = true;
|
||||
while (refs.queue.length > 0) {
|
||||
const item = refs.queue.shift()!;
|
||||
try {
|
||||
const duration = await playAudio(item.audio);
|
||||
item.resolve(duration);
|
||||
} catch (error) {
|
||||
item.reject(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
}
|
||||
|
||||
async function stopCapture(): Promise<void> {
|
||||
refs.started = false;
|
||||
|
||||
try {
|
||||
refs.processor?.disconnect();
|
||||
refs.source?.disconnect();
|
||||
refs.gain?.disconnect();
|
||||
} catch {
|
||||
// Ignore best-effort teardown errors.
|
||||
}
|
||||
|
||||
if (refs.stream) {
|
||||
for (const track of refs.stream.getTracks()) {
|
||||
try {
|
||||
track.stop();
|
||||
} catch {
|
||||
// Ignore best-effort teardown errors.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
refs.stream = null;
|
||||
refs.source = null;
|
||||
refs.processor = null;
|
||||
refs.gain = null;
|
||||
refs.muted = false;
|
||||
callbacks.onVolumeLevel(0);
|
||||
|
||||
const captureContext = refs.captureContext;
|
||||
refs.captureContext = null;
|
||||
if (captureContext && captureContext.state !== "closed") {
|
||||
await captureContext.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async initialize() {
|
||||
await ensurePlaybackContext();
|
||||
await Promise.all([ensureThinkingToneBytes(), ensureThinkingToneBuffer()]);
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
stopLooping();
|
||||
this.stop();
|
||||
this.clearQueue();
|
||||
await stopCapture();
|
||||
|
||||
const playbackContext = refs.playbackContext;
|
||||
refs.playbackContext = null;
|
||||
refs.thinkingTone.buffer = null;
|
||||
if (playbackContext && playbackContext.state !== "closed") {
|
||||
await playbackContext.close().catch(() => undefined);
|
||||
}
|
||||
},
|
||||
|
||||
async startCapture() {
|
||||
if (refs.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const missingNavigator =
|
||||
typeof navigator === "undefined" ||
|
||||
!navigator.mediaDevices ||
|
||||
typeof navigator.mediaDevices.getUserMedia !== "function";
|
||||
const secureContext =
|
||||
typeof window !== "undefined" && typeof window.isSecureContext === "boolean"
|
||||
? window.isSecureContext
|
||||
: true;
|
||||
const currentOrigin =
|
||||
typeof window !== "undefined" && window.location
|
||||
? window.location.origin
|
||||
: "unknown";
|
||||
const isTauri = getTauri() !== null;
|
||||
|
||||
if (missingNavigator) {
|
||||
throw new Error("Microphone capture is not supported in this environment");
|
||||
}
|
||||
if (!secureContext && !isTauri) {
|
||||
throw new Error(
|
||||
`Microphone access requires HTTPS or localhost. Current origin: ${currentOrigin}`
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const context = await ensureCaptureContext();
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
noiseSuppression: true,
|
||||
echoCancellation: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
});
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
const processor = context.createScriptProcessor(4096, 1, 1);
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 0;
|
||||
|
||||
processor.onaudioprocess = (event) => {
|
||||
if (!refs.started) {
|
||||
return;
|
||||
}
|
||||
|
||||
const input = event.inputBuffer.getChannelData(0);
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < input.length; i += 1) {
|
||||
const sample = input[i]!;
|
||||
sumSquares += sample * sample;
|
||||
}
|
||||
const rms = Math.sqrt(sumSquares / Math.max(1, input.length));
|
||||
const normalized = Math.min(1, Math.max(0, rms * 2));
|
||||
callbacks.onVolumeLevel(normalized);
|
||||
|
||||
if (refs.muted) {
|
||||
return;
|
||||
}
|
||||
|
||||
callbacks.onCaptureData(resampleToPcm16(input, context.sampleRate, 16000));
|
||||
};
|
||||
|
||||
source.connect(processor);
|
||||
processor.connect(gain);
|
||||
gain.connect(context.destination);
|
||||
|
||||
refs.started = true;
|
||||
refs.stream = stream;
|
||||
refs.source = source;
|
||||
refs.processor = processor;
|
||||
refs.gain = gain;
|
||||
} catch (error) {
|
||||
await stopCapture();
|
||||
const wrapped = error instanceof Error ? error : new Error(String(error));
|
||||
callbacks.onError?.(wrapped);
|
||||
throw wrapped;
|
||||
}
|
||||
},
|
||||
|
||||
async stopCapture() {
|
||||
await stopCapture();
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
refs.muted = !refs.muted;
|
||||
if (refs.muted) {
|
||||
callbacks.onVolumeLevel(0);
|
||||
}
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
isMuted() {
|
||||
return refs.muted;
|
||||
},
|
||||
|
||||
async play(audio: AudioPlaybackSource) {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
refs.queue.push({ audio, resolve, reject });
|
||||
if (!refs.processingQueue) {
|
||||
void processQueue();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
stop() {
|
||||
if (refs.activePlayback) {
|
||||
const active = refs.activePlayback;
|
||||
refs.activePlayback = null;
|
||||
try {
|
||||
active.source.onended = null;
|
||||
active.source.stop();
|
||||
} catch {
|
||||
// Ignore best-effort stop errors.
|
||||
}
|
||||
if (!active.settled) {
|
||||
active.settled = true;
|
||||
active.reject(new Error("Playback stopped"));
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
clearQueue() {
|
||||
while (refs.queue.length > 0) {
|
||||
refs.queue.shift()!.reject(new Error("Playback stopped"));
|
||||
}
|
||||
refs.processingQueue = false;
|
||||
},
|
||||
|
||||
isPlaying() {
|
||||
return refs.activePlayback !== null;
|
||||
},
|
||||
|
||||
playLooping(audio, gapMs) {
|
||||
if (refs.looping.source || refs.looping.timeout) {
|
||||
return;
|
||||
}
|
||||
|
||||
const token = refs.looping.token + 1;
|
||||
refs.looping.token = token;
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const context = await ensurePlaybackContext();
|
||||
const audioBuffer =
|
||||
audio.byteLength > 0
|
||||
? pcm16LeToAudioBuffer(context, audio, 16000)
|
||||
: await ensureThinkingToneBuffer();
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
|
||||
const gain = context.createGain();
|
||||
gain.gain.value = 1;
|
||||
gain.connect(context.destination);
|
||||
refs.looping.gain = gain;
|
||||
|
||||
const playOnce = () => {
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
const source = context.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(gain);
|
||||
refs.looping.source = source;
|
||||
source.onended = () => {
|
||||
if (refs.looping.source === source) {
|
||||
refs.looping.source = null;
|
||||
}
|
||||
if (refs.looping.token !== token) {
|
||||
return;
|
||||
}
|
||||
refs.looping.timeout = setTimeout(
|
||||
playOnce,
|
||||
gapMs || THINKING_TONE_REPEAT_GAP_MS
|
||||
);
|
||||
};
|
||||
source.start();
|
||||
};
|
||||
|
||||
playOnce();
|
||||
} catch (error) {
|
||||
callbacks.onError?.(
|
||||
error instanceof Error ? error : new Error(String(error))
|
||||
);
|
||||
}
|
||||
})();
|
||||
},
|
||||
|
||||
stopLooping,
|
||||
};
|
||||
}
|
||||
243
packages/app/src/voice/voice-runtime.test.ts
Normal file
243
packages/app/src/voice/voice-runtime.test.ts
Normal file
@@ -0,0 +1,243 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { DaemonServerInfo } from "@/stores/session-store";
|
||||
import type { AudioEngine } from "@/voice/audio-engine-types";
|
||||
import {
|
||||
createVoiceRuntime,
|
||||
type VoiceRuntime,
|
||||
type VoiceSessionAdapter,
|
||||
} from "@/voice/voice-runtime";
|
||||
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
|
||||
|
||||
function createAudioEngineMock(): AudioEngine {
|
||||
return {
|
||||
initialize: vi.fn().mockResolvedValue(undefined),
|
||||
destroy: vi.fn().mockResolvedValue(undefined),
|
||||
startCapture: vi.fn().mockResolvedValue(undefined),
|
||||
stopCapture: vi.fn().mockResolvedValue(undefined),
|
||||
toggleMute: vi.fn().mockReturnValue(true),
|
||||
isMuted: vi.fn().mockReturnValue(false),
|
||||
play: vi.fn().mockResolvedValue(0.1),
|
||||
stop: vi.fn(),
|
||||
clearQueue: vi.fn(),
|
||||
isPlaying: vi.fn().mockReturnValue(false),
|
||||
playLooping: vi.fn(),
|
||||
stopLooping: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createSessionAdapter(serverId = "server-1"): VoiceSessionAdapter {
|
||||
return {
|
||||
serverId,
|
||||
setVoiceMode: vi.fn().mockResolvedValue(undefined),
|
||||
sendVoiceAudioChunk: vi.fn().mockResolvedValue(undefined),
|
||||
abortRequest: vi.fn().mockResolvedValue(undefined),
|
||||
setAssistantAudioPlaying: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function createServerInfo(): DaemonServerInfo {
|
||||
return {
|
||||
serverId: "server-1",
|
||||
hostname: "host",
|
||||
version: "1.0.0",
|
||||
capabilities: {
|
||||
voice: {
|
||||
dictation: { enabled: true, reason: "" },
|
||||
voice: { enabled: true, reason: "" },
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createRuntime(options?: {
|
||||
engine?: AudioEngine;
|
||||
getServerInfo?: (serverId: string) => DaemonServerInfo | null;
|
||||
}) {
|
||||
const engine = options?.engine ?? createAudioEngineMock();
|
||||
const runtime = createVoiceRuntime({
|
||||
engine,
|
||||
getServerInfo: options?.getServerInfo ?? (() => createServerInfo()),
|
||||
activateKeepAwake: vi.fn().mockResolvedValue(undefined),
|
||||
deactivateKeepAwake: vi.fn().mockResolvedValue(undefined),
|
||||
});
|
||||
|
||||
return { runtime, engine };
|
||||
}
|
||||
|
||||
describe("voice runtime", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("starts voice when adapter is ready", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
|
||||
expect(engine.initialize).toHaveBeenCalled();
|
||||
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
|
||||
expect(engine.startCapture).toHaveBeenCalled();
|
||||
expect(runtime.getSnapshot()).toMatchObject({
|
||||
phase: "listening",
|
||||
isVoiceMode: true,
|
||||
activeServerId: "server-1",
|
||||
activeAgentId: "agent-1",
|
||||
});
|
||||
});
|
||||
|
||||
it("transitions from capturing to submitting to waiting on the final chunk", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.handleCaptureVolume(0.4);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0.4);
|
||||
runtime.handleCapturePcm(new Uint8Array(32000));
|
||||
expect(runtime.getSnapshot().phase).toBe("capturing");
|
||||
|
||||
runtime.handleCapturePcm(new Uint8Array(4000));
|
||||
runtime.handleCaptureVolume(0);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.silenceDurationMs + 10
|
||||
);
|
||||
runtime.handleCaptureVolume(0);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(adapter.sendVoiceAudioChunk).toHaveBeenLastCalledWith(
|
||||
expect.any(String),
|
||||
"audio/pcm;rate=16000;bits=16",
|
||||
true
|
||||
);
|
||||
expect(runtime.getSnapshot().phase).toBe("waiting");
|
||||
});
|
||||
|
||||
it("moves from waiting to playing on the first assistant audio", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
|
||||
expect(runtime.getSnapshot().phase).toBe("playing");
|
||||
expect(engine.stopLooping).toHaveBeenCalled();
|
||||
expect(adapter.setAssistantAudioPlaying).toHaveBeenCalledWith(true);
|
||||
});
|
||||
|
||||
it("returns to waiting after assistant playback when the turn is still active", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
runtime.onAssistantAudioFinished("server-1");
|
||||
|
||||
expect(runtime.getSnapshot().phase).toBe("waiting");
|
||||
expect(engine.playLooping).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns to listening after assistant playback once the turn is complete", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_completed");
|
||||
runtime.onAssistantAudioFinished("server-1");
|
||||
|
||||
expect(runtime.getSnapshot().phase).toBe("listening");
|
||||
expect(engine.playLooping).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("interrupts the active turn on barge-in after the grace period", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
runtime.onTurnEvent("server-1", "agent-1", "turn_started");
|
||||
runtime.onAssistantAudioStarted("server-1");
|
||||
|
||||
runtime.handleCaptureVolume(0.5);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs + 10
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(
|
||||
REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs
|
||||
);
|
||||
|
||||
expect(adapter.abortRequest).toHaveBeenCalled();
|
||||
expect(engine.stop).toHaveBeenCalled();
|
||||
expect(runtime.getSnapshot().phase).toBe("capturing");
|
||||
});
|
||||
|
||||
it("authoritatively stops and suppresses later voice audio", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime, engine } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
await runtime.stopVoice();
|
||||
|
||||
expect(adapter.setVoiceMode).toHaveBeenLastCalledWith(false);
|
||||
expect(engine.stopCapture).toHaveBeenCalled();
|
||||
expect(runtime.getSnapshot().phase).toBe("disabled");
|
||||
expect(runtime.shouldPlayVoiceAudio("server-1")).toBe(false);
|
||||
});
|
||||
|
||||
it("returns an explicit not-ready error when the adapter is missing", async () => {
|
||||
const { runtime } = createRuntime();
|
||||
await expect(runtime.startVoice("server-1", "agent-1")).rejects.toThrow(
|
||||
"Voice runtime is not ready for host server-1"
|
||||
);
|
||||
});
|
||||
|
||||
it("resyncs voice mode after connection recovers", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
vi.mocked(adapter.setVoiceMode).mockClear();
|
||||
|
||||
runtime.updateSessionConnection("server-1", false);
|
||||
runtime.updateSessionConnection("server-1", true);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(adapter.setVoiceMode).toHaveBeenCalledWith(true, "agent-1");
|
||||
});
|
||||
|
||||
it("does not emit when the snapshot is unchanged", async () => {
|
||||
const adapter = createSessionAdapter();
|
||||
const { runtime } = createRuntime();
|
||||
runtime.registerSession(adapter);
|
||||
await runtime.startVoice("server-1", "agent-1");
|
||||
|
||||
const listener = vi.fn();
|
||||
const unsubscribe = runtime.subscribe(listener);
|
||||
|
||||
runtime.handleCaptureVolume(0);
|
||||
runtime.handleCaptureVolume(0);
|
||||
|
||||
expect(listener).not.toHaveBeenCalled();
|
||||
unsubscribe();
|
||||
});
|
||||
});
|
||||
751
packages/app/src/voice/voice-runtime.ts
Normal file
751
packages/app/src/voice/voice-runtime.ts
Normal file
@@ -0,0 +1,751 @@
|
||||
import type { AgentStreamEventPayload } from "@server/shared/messages";
|
||||
import { resolveVoiceUnavailableMessage } from "@/utils/server-info-capabilities";
|
||||
import type { DaemonServerInfo } from "@/stores/session-store";
|
||||
import type { AudioEngine } from "@/voice/audio-engine-types";
|
||||
import { REALTIME_VOICE_VAD_CONFIG } from "@/voice/realtime-voice-config";
|
||||
import { SpeechSegmenter } from "@/voice/speech-segmenter";
|
||||
|
||||
const PCM_MIME_TYPE = "audio/pcm;rate=16000;bits=16";
|
||||
const KEEP_AWAKE_TAG = "paseo:voice";
|
||||
const THINKING_TONE_REPEAT_GAP_MS = 350;
|
||||
const DISPLAY_VOLUME_PUBLISH_INTERVAL_MS = 120;
|
||||
const DISPLAY_VOLUME_CHANGE_EPSILON = 0.02;
|
||||
const DISPLAY_VOLUME_ATTACK = 0.35;
|
||||
const DISPLAY_VOLUME_RELEASE = 0.18;
|
||||
|
||||
type TurnEventType = Extract<
|
||||
AgentStreamEventPayload["type"],
|
||||
"turn_started" | "turn_completed" | "turn_failed" | "turn_canceled"
|
||||
>;
|
||||
|
||||
export type VoiceRuntimePhase =
|
||||
| "disabled"
|
||||
| "starting"
|
||||
| "listening"
|
||||
| "capturing"
|
||||
| "submitting"
|
||||
| "waiting"
|
||||
| "playing"
|
||||
| "stopping";
|
||||
|
||||
export interface VoiceRuntimeSnapshot {
|
||||
phase: VoiceRuntimePhase;
|
||||
isVoiceMode: boolean;
|
||||
isVoiceSwitching: boolean;
|
||||
isMuted: boolean;
|
||||
activeServerId: string | null;
|
||||
activeAgentId: string | null;
|
||||
}
|
||||
|
||||
export interface VoiceRuntimeTelemetrySnapshot {
|
||||
volume: number;
|
||||
isDetecting: boolean;
|
||||
isSpeaking: boolean;
|
||||
segmentDuration: number;
|
||||
}
|
||||
|
||||
export interface VoiceSessionAdapter {
|
||||
serverId: string;
|
||||
setVoiceMode(enabled: boolean, agentId?: string): Promise<void>;
|
||||
sendVoiceAudioChunk(
|
||||
audioData: string,
|
||||
mimeType: string,
|
||||
isLast: boolean
|
||||
): Promise<void>;
|
||||
abortRequest(): Promise<void>;
|
||||
setAssistantAudioPlaying(isPlaying: boolean): void;
|
||||
}
|
||||
|
||||
export interface VoiceRuntimeDeps {
|
||||
engine: AudioEngine;
|
||||
getServerInfo(serverId: string): DaemonServerInfo | null;
|
||||
activateKeepAwake(tag: string): Promise<void>;
|
||||
deactivateKeepAwake(tag: string): Promise<void>;
|
||||
}
|
||||
|
||||
interface RuntimeSessionState {
|
||||
adapter: VoiceSessionAdapter;
|
||||
connected: boolean;
|
||||
}
|
||||
|
||||
interface RuntimeState {
|
||||
snapshot: VoiceRuntimeSnapshot;
|
||||
telemetry: VoiceRuntimeTelemetrySnapshot;
|
||||
turnInProgress: boolean;
|
||||
transportReady: boolean;
|
||||
generation: number;
|
||||
speechInterruptTimer: ReturnType<typeof setTimeout> | null;
|
||||
speechInterruptSent: boolean;
|
||||
segmentDurationTimer: ReturnType<typeof setInterval> | null;
|
||||
lastDisplayVolumePublishMs: number;
|
||||
}
|
||||
|
||||
const INITIAL_SNAPSHOT: VoiceRuntimeSnapshot = {
|
||||
phase: "disabled",
|
||||
isVoiceMode: false,
|
||||
isVoiceSwitching: false,
|
||||
isMuted: false,
|
||||
activeServerId: null,
|
||||
activeAgentId: null,
|
||||
};
|
||||
|
||||
const INITIAL_TELEMETRY: VoiceRuntimeTelemetrySnapshot = {
|
||||
volume: 0,
|
||||
isDetecting: false,
|
||||
isSpeaking: false,
|
||||
segmentDuration: 0,
|
||||
};
|
||||
|
||||
function snapshotsEqual(
|
||||
left: VoiceRuntimeSnapshot,
|
||||
right: VoiceRuntimeSnapshot
|
||||
): boolean {
|
||||
return (
|
||||
left.phase === right.phase &&
|
||||
left.isVoiceMode === right.isVoiceMode &&
|
||||
left.isVoiceSwitching === right.isVoiceSwitching &&
|
||||
left.isMuted === right.isMuted &&
|
||||
left.activeServerId === right.activeServerId &&
|
||||
left.activeAgentId === right.activeAgentId
|
||||
);
|
||||
}
|
||||
|
||||
function telemetryEqual(
|
||||
left: VoiceRuntimeTelemetrySnapshot,
|
||||
right: VoiceRuntimeTelemetrySnapshot
|
||||
): boolean {
|
||||
return (
|
||||
left.volume === right.volume &&
|
||||
left.isDetecting === right.isDetecting &&
|
||||
left.isSpeaking === right.isSpeaking &&
|
||||
left.segmentDuration === right.segmentDuration
|
||||
);
|
||||
}
|
||||
|
||||
export interface VoiceRuntime {
|
||||
subscribe(listener: () => void): () => void;
|
||||
getSnapshot(): VoiceRuntimeSnapshot;
|
||||
subscribeTelemetry(listener: () => void): () => void;
|
||||
getTelemetrySnapshot(): VoiceRuntimeTelemetrySnapshot;
|
||||
registerSession(adapter: VoiceSessionAdapter): () => void;
|
||||
updateSessionConnection(serverId: string, connected: boolean): void;
|
||||
handleCapturePcm(chunk: Uint8Array): void;
|
||||
handleCaptureVolume(level: number): void;
|
||||
startVoice(serverId: string, agentId: string): Promise<void>;
|
||||
stopVoice(): Promise<void>;
|
||||
destroy(): Promise<void>;
|
||||
toggleMute(): void;
|
||||
isVoiceModeForAgent(serverId: string, agentId: string): boolean;
|
||||
shouldPlayVoiceAudio(serverId: string): boolean;
|
||||
onAssistantAudioStarted(serverId: string): void;
|
||||
onAssistantAudioFinished(serverId: string): void;
|
||||
onTranscriptionResult(serverId: string, text: string): void;
|
||||
onTurnEvent(serverId: string, agentId: string, eventType: TurnEventType): void;
|
||||
}
|
||||
|
||||
export function createVoiceRuntime(deps: VoiceRuntimeDeps): VoiceRuntime {
|
||||
const listeners = new Set<() => void>();
|
||||
const telemetryListeners = new Set<() => void>();
|
||||
const sessions = new Map<string, RuntimeSessionState>();
|
||||
const state: RuntimeState = {
|
||||
snapshot: INITIAL_SNAPSHOT,
|
||||
telemetry: INITIAL_TELEMETRY,
|
||||
turnInProgress: false,
|
||||
transportReady: false,
|
||||
generation: 0,
|
||||
speechInterruptTimer: null,
|
||||
speechInterruptSent: false,
|
||||
segmentDurationTimer: null,
|
||||
lastDisplayVolumePublishMs: 0,
|
||||
};
|
||||
|
||||
function emit(): void {
|
||||
for (const listener of listeners) {
|
||||
listener();
|
||||
}
|
||||
}
|
||||
|
||||
function emitTelemetry(): void {
|
||||
for (const listener of telemetryListeners) {
|
||||
listener();
|
||||
}
|
||||
}
|
||||
|
||||
function patchSnapshot(
|
||||
patch:
|
||||
| Partial<VoiceRuntimeSnapshot>
|
||||
| ((previous: VoiceRuntimeSnapshot) => VoiceRuntimeSnapshot)
|
||||
): void {
|
||||
const next =
|
||||
typeof patch === "function"
|
||||
? patch(state.snapshot)
|
||||
: { ...state.snapshot, ...patch };
|
||||
if (snapshotsEqual(next, state.snapshot)) {
|
||||
return;
|
||||
}
|
||||
const previous = state.snapshot;
|
||||
state.snapshot = next;
|
||||
emit();
|
||||
}
|
||||
|
||||
function patchTelemetry(
|
||||
patch:
|
||||
| Partial<VoiceRuntimeTelemetrySnapshot>
|
||||
| ((
|
||||
previous: VoiceRuntimeTelemetrySnapshot
|
||||
) => VoiceRuntimeTelemetrySnapshot)
|
||||
): void {
|
||||
const next =
|
||||
typeof patch === "function"
|
||||
? patch(state.telemetry)
|
||||
: { ...state.telemetry, ...patch };
|
||||
if (telemetryEqual(next, state.telemetry)) {
|
||||
return;
|
||||
}
|
||||
state.telemetry = next;
|
||||
emitTelemetry();
|
||||
}
|
||||
|
||||
function getActiveSession(): RuntimeSessionState | null {
|
||||
if (!state.snapshot.activeServerId) {
|
||||
return null;
|
||||
}
|
||||
return sessions.get(state.snapshot.activeServerId) ?? null;
|
||||
}
|
||||
|
||||
function clearSpeechInterruptTimer(): void {
|
||||
if (state.speechInterruptTimer) {
|
||||
clearTimeout(state.speechInterruptTimer);
|
||||
state.speechInterruptTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
function clearSegmentDurationTimer(): void {
|
||||
if (state.segmentDurationTimer) {
|
||||
clearInterval(state.segmentDurationTimer);
|
||||
state.segmentDurationTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
function reconcileSegmentDurationTimer(
|
||||
segmenter: SpeechSegmenter
|
||||
): void {
|
||||
if (!state.telemetry.isDetecting && !state.telemetry.isSpeaking) {
|
||||
clearSegmentDurationTimer();
|
||||
patchTelemetry((prev) => ({ ...prev, segmentDuration: 0 }));
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.segmentDurationTimer) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.segmentDurationTimer = setInterval(() => {
|
||||
const startedAt = segmenter.getSpeechDetectionStartMs();
|
||||
patchTelemetry((prev) => ({
|
||||
...prev,
|
||||
segmentDuration: startedAt ? Date.now() - startedAt : 0,
|
||||
}));
|
||||
}, 100);
|
||||
}
|
||||
|
||||
function canPlayCue(): boolean {
|
||||
return (
|
||||
state.snapshot.isVoiceMode &&
|
||||
state.snapshot.phase === "waiting" &&
|
||||
!state.telemetry.isDetecting &&
|
||||
!state.telemetry.isSpeaking
|
||||
);
|
||||
}
|
||||
|
||||
function stopCue(): void {
|
||||
deps.engine.stopLooping();
|
||||
}
|
||||
|
||||
function resetSegmenter(segmenter: SpeechSegmenter): void {
|
||||
segmenter.reset();
|
||||
clearSpeechInterruptTimer();
|
||||
clearSegmentDurationTimer();
|
||||
patchTelemetry({ ...INITIAL_TELEMETRY });
|
||||
}
|
||||
|
||||
function reconcileCue(): void {
|
||||
if (!canPlayCue()) {
|
||||
stopCue();
|
||||
return;
|
||||
}
|
||||
deps.engine.playLooping(new Uint8Array(0), THINKING_TONE_REPEAT_GAP_MS);
|
||||
}
|
||||
|
||||
async function interruptActiveTurn(): Promise<void> {
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession) {
|
||||
return;
|
||||
}
|
||||
|
||||
stopCue();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
activeSession.adapter.setAssistantAudioPlaying(false);
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "capturing" }));
|
||||
|
||||
try {
|
||||
await activeSession.adapter.abortRequest();
|
||||
} catch (error) {
|
||||
console.error("[VoiceRuntime] Failed to abort active turn:", error);
|
||||
}
|
||||
}
|
||||
|
||||
const segmenter = new SpeechSegmenter(
|
||||
{
|
||||
enableContinuousStreaming: false,
|
||||
volumeThreshold: REALTIME_VOICE_VAD_CONFIG.volumeThreshold,
|
||||
confirmedDropGracePeriodMs:
|
||||
REALTIME_VOICE_VAD_CONFIG.confirmedDropGracePeriodMs,
|
||||
silenceDurationMs: REALTIME_VOICE_VAD_CONFIG.silenceDurationMs,
|
||||
speechConfirmationMs: REALTIME_VOICE_VAD_CONFIG.speechConfirmationMs,
|
||||
detectionGracePeriodMs: REALTIME_VOICE_VAD_CONFIG.detectionGracePeriodMs,
|
||||
},
|
||||
{
|
||||
onAudioSegment: ({ audioData, isLast }) => {
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession || !state.transportReady || !state.snapshot.isVoiceMode) {
|
||||
return;
|
||||
}
|
||||
|
||||
const generation = state.generation;
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
phase: isLast ? "submitting" : "capturing",
|
||||
}));
|
||||
if (isLast) {
|
||||
state.turnInProgress = true;
|
||||
}
|
||||
|
||||
void activeSession.adapter
|
||||
.sendVoiceAudioChunk(audioData, PCM_MIME_TYPE, isLast)
|
||||
.then(() => {
|
||||
if (
|
||||
!isLast ||
|
||||
generation !== state.generation ||
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== activeSession.adapter.serverId ||
|
||||
state.snapshot.phase !== "submitting"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
|
||||
reconcileCue();
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error("[VoiceRuntime] Failed to send audio segment:", error);
|
||||
});
|
||||
},
|
||||
onSpeechStart: () => {
|
||||
stopCue();
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptSent = false;
|
||||
reconcileCue();
|
||||
},
|
||||
onDetectingChange: (isDetecting) => {
|
||||
const previous = state.telemetry;
|
||||
patchTelemetry((prev) => ({ ...prev, isDetecting }));
|
||||
if (previous.isDetecting !== isDetecting) {
|
||||
}
|
||||
if (!previous.isDetecting && isDetecting) {
|
||||
stopCue();
|
||||
}
|
||||
reconcileSegmentDurationTimer(segmenter);
|
||||
reconcileCue();
|
||||
},
|
||||
onSpeakingChange: (isSpeaking) => {
|
||||
const previous = state.telemetry;
|
||||
patchTelemetry((prev) => ({ ...prev, isSpeaking }));
|
||||
if (previous.isSpeaking !== isSpeaking) {
|
||||
}
|
||||
|
||||
if (!previous.isSpeaking && isSpeaking) {
|
||||
stopCue();
|
||||
if (
|
||||
state.snapshot.phase === "waiting" ||
|
||||
state.snapshot.phase === "playing"
|
||||
) {
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptTimer = setTimeout(() => {
|
||||
state.speechInterruptTimer = null;
|
||||
if (
|
||||
state.speechInterruptSent ||
|
||||
!state.snapshot.isVoiceMode ||
|
||||
!state.telemetry.isSpeaking
|
||||
) {
|
||||
return;
|
||||
}
|
||||
state.speechInterruptSent = true;
|
||||
void interruptActiveTurn();
|
||||
}, REALTIME_VOICE_VAD_CONFIG.interruptGracePeriodMs);
|
||||
}
|
||||
}
|
||||
|
||||
if (!isSpeaking) {
|
||||
clearSpeechInterruptTimer();
|
||||
state.speechInterruptSent = false;
|
||||
}
|
||||
|
||||
reconcileSegmentDurationTimer(segmenter);
|
||||
reconcileCue();
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
function resetToDisabledState(): void {
|
||||
state.transportReady = false;
|
||||
state.turnInProgress = false;
|
||||
state.speechInterruptSent = false;
|
||||
state.lastDisplayVolumePublishMs = 0;
|
||||
resetSegmenter(segmenter);
|
||||
patchSnapshot({ ...INITIAL_SNAPSHOT });
|
||||
}
|
||||
|
||||
function publishDisplayVolume(level: number, nowMs: number): void {
|
||||
const previousVolume = state.telemetry.volume;
|
||||
const smoothing =
|
||||
level >= previousVolume ? DISPLAY_VOLUME_ATTACK : DISPLAY_VOLUME_RELEASE;
|
||||
const nextVolume = Math.max(
|
||||
0,
|
||||
Math.min(1, previousVolume + (level - previousVolume) * smoothing)
|
||||
);
|
||||
const enoughTimeElapsed =
|
||||
nowMs - state.lastDisplayVolumePublishMs >=
|
||||
DISPLAY_VOLUME_PUBLISH_INTERVAL_MS;
|
||||
const enoughChange =
|
||||
Math.abs(nextVolume - previousVolume) >= DISPLAY_VOLUME_CHANGE_EPSILON;
|
||||
|
||||
if (!enoughTimeElapsed && !enoughChange) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.lastDisplayVolumePublishMs = nowMs;
|
||||
patchTelemetry((prev) => ({
|
||||
...prev,
|
||||
volume: Number(nextVolume.toFixed(3)),
|
||||
}));
|
||||
}
|
||||
|
||||
async function performLocalStop(): Promise<void> {
|
||||
stopCue();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
await deps.engine.stopCapture().catch(() => undefined);
|
||||
await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
|
||||
resetToDisabledState();
|
||||
}
|
||||
|
||||
async function resyncVoiceMode(serverId: string): Promise<void> {
|
||||
if (
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== serverId ||
|
||||
!state.snapshot.activeAgentId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const activeSession = getActiveSession();
|
||||
if (!activeSession || !activeSession.connected) {
|
||||
return;
|
||||
}
|
||||
|
||||
patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: true }));
|
||||
try {
|
||||
await activeSession.adapter.setVoiceMode(true, state.snapshot.activeAgentId);
|
||||
state.transportReady = true;
|
||||
} finally {
|
||||
patchSnapshot((prev) => ({ ...prev, isVoiceSwitching: false }));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
subscribe(listener) {
|
||||
listeners.add(listener);
|
||||
return () => {
|
||||
listeners.delete(listener);
|
||||
};
|
||||
},
|
||||
|
||||
getSnapshot() {
|
||||
return state.snapshot;
|
||||
},
|
||||
|
||||
subscribeTelemetry(listener) {
|
||||
telemetryListeners.add(listener);
|
||||
return () => {
|
||||
telemetryListeners.delete(listener);
|
||||
};
|
||||
},
|
||||
|
||||
getTelemetrySnapshot() {
|
||||
return state.telemetry;
|
||||
},
|
||||
|
||||
registerSession(adapter) {
|
||||
sessions.set(adapter.serverId, {
|
||||
adapter,
|
||||
connected: true,
|
||||
});
|
||||
|
||||
return () => {
|
||||
const activeServerId = state.snapshot.activeServerId;
|
||||
sessions.delete(adapter.serverId);
|
||||
if (activeServerId === adapter.serverId) {
|
||||
void performLocalStop();
|
||||
}
|
||||
};
|
||||
},
|
||||
|
||||
updateSessionConnection(serverId, connected) {
|
||||
const session = sessions.get(serverId);
|
||||
if (!session) {
|
||||
return;
|
||||
}
|
||||
session.connected = connected;
|
||||
if (state.snapshot.activeServerId !== serverId) {
|
||||
return;
|
||||
}
|
||||
if (!connected) {
|
||||
state.transportReady = false;
|
||||
return;
|
||||
}
|
||||
void resyncVoiceMode(serverId);
|
||||
},
|
||||
|
||||
handleCapturePcm(chunk) {
|
||||
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
|
||||
return;
|
||||
}
|
||||
segmenter.pushPcmChunk(chunk);
|
||||
},
|
||||
|
||||
handleCaptureVolume(level) {
|
||||
const nowMs = Date.now();
|
||||
const displayLevel = state.snapshot.isMuted ? 0 : level;
|
||||
publishDisplayVolume(displayLevel, nowMs);
|
||||
if (!state.snapshot.isVoiceMode || state.snapshot.isMuted) {
|
||||
return;
|
||||
}
|
||||
segmenter.pushVolumeLevel(level, nowMs);
|
||||
},
|
||||
|
||||
async startVoice(serverId, agentId) {
|
||||
const session = sessions.get(serverId);
|
||||
if (!session) {
|
||||
throw new Error(`Voice runtime is not ready for host ${serverId}`);
|
||||
}
|
||||
if (!session.connected) {
|
||||
throw new Error(`Host ${serverId} is not connected`);
|
||||
}
|
||||
|
||||
const serverInfo = deps.getServerInfo(serverId);
|
||||
const unavailableMessage = resolveVoiceUnavailableMessage({
|
||||
serverInfo,
|
||||
mode: "voice",
|
||||
});
|
||||
if (unavailableMessage) {
|
||||
throw new Error(unavailableMessage);
|
||||
}
|
||||
|
||||
const previousServerId = state.snapshot.activeServerId;
|
||||
const previousAgentId = state.snapshot.activeAgentId;
|
||||
const generation = state.generation + 1;
|
||||
state.generation = generation;
|
||||
state.transportReady = false;
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isVoiceSwitching: true,
|
||||
phase: "starting",
|
||||
activeServerId: serverId,
|
||||
activeAgentId: agentId,
|
||||
}));
|
||||
|
||||
try {
|
||||
if (
|
||||
state.snapshot.isVoiceMode &&
|
||||
previousServerId &&
|
||||
(previousServerId !== serverId || previousAgentId !== agentId)
|
||||
) {
|
||||
const previousSession = sessions.get(previousServerId);
|
||||
if (previousSession) {
|
||||
previousSession.adapter.setAssistantAudioPlaying(false);
|
||||
await previousSession.adapter.setVoiceMode(false);
|
||||
}
|
||||
}
|
||||
|
||||
await deps.activateKeepAwake(KEEP_AWAKE_TAG).catch((error) => {
|
||||
console.warn("[VoiceRuntime] Failed to activate keep-awake:", error);
|
||||
});
|
||||
|
||||
await deps.engine.initialize();
|
||||
await session.adapter.setVoiceMode(true, agentId);
|
||||
await deps.engine.startCapture();
|
||||
if (state.generation !== generation) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.transportReady = true;
|
||||
state.turnInProgress = false;
|
||||
resetSegmenter(segmenter);
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isVoiceMode: true,
|
||||
isVoiceSwitching: false,
|
||||
phase: "listening",
|
||||
isMuted: deps.engine.isMuted(),
|
||||
}));
|
||||
} catch (error) {
|
||||
await performLocalStop();
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
async stopVoice() {
|
||||
const activeSession = getActiveSession();
|
||||
const generation = state.generation + 1;
|
||||
state.generation = generation;
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isVoiceSwitching: true,
|
||||
phase: "stopping",
|
||||
}));
|
||||
|
||||
try {
|
||||
state.transportReady = false;
|
||||
stopCue();
|
||||
deps.engine.stop();
|
||||
deps.engine.clearQueue();
|
||||
activeSession?.adapter.setAssistantAudioPlaying(false);
|
||||
if (activeSession) {
|
||||
await activeSession.adapter.setVoiceMode(false);
|
||||
}
|
||||
await deps.engine.stopCapture();
|
||||
await deps.deactivateKeepAwake(KEEP_AWAKE_TAG).catch(() => undefined);
|
||||
} finally {
|
||||
if (state.generation === generation) {
|
||||
resetToDisabledState();
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
async destroy() {
|
||||
await this.stopVoice().catch(() => undefined);
|
||||
await deps.engine.destroy();
|
||||
listeners.clear();
|
||||
telemetryListeners.clear();
|
||||
sessions.clear();
|
||||
},
|
||||
|
||||
toggleMute() {
|
||||
const nextMuted = deps.engine.toggleMute();
|
||||
if (nextMuted) {
|
||||
resetSegmenter(segmenter);
|
||||
patchSnapshot((prev) => ({
|
||||
...prev,
|
||||
isMuted: true,
|
||||
}));
|
||||
reconcileCue();
|
||||
return;
|
||||
}
|
||||
|
||||
patchSnapshot((prev) => ({ ...prev, isMuted: false }));
|
||||
},
|
||||
|
||||
isVoiceModeForAgent(serverId, agentId) {
|
||||
return (
|
||||
state.snapshot.isVoiceMode &&
|
||||
state.snapshot.activeServerId === serverId &&
|
||||
state.snapshot.activeAgentId === agentId
|
||||
);
|
||||
},
|
||||
|
||||
shouldPlayVoiceAudio(serverId) {
|
||||
return (
|
||||
state.snapshot.isVoiceMode &&
|
||||
state.snapshot.activeServerId === serverId &&
|
||||
state.snapshot.phase !== "stopping" &&
|
||||
state.snapshot.phase !== "disabled" &&
|
||||
!state.telemetry.isDetecting &&
|
||||
!state.telemetry.isSpeaking
|
||||
);
|
||||
},
|
||||
|
||||
onAssistantAudioStarted(serverId) {
|
||||
if (
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== serverId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
stopCue();
|
||||
getActiveSession()?.adapter.setAssistantAudioPlaying(true);
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "playing" }));
|
||||
},
|
||||
|
||||
onAssistantAudioFinished(serverId) {
|
||||
if (state.snapshot.activeServerId !== serverId) {
|
||||
return;
|
||||
}
|
||||
|
||||
getActiveSession()?.adapter.setAssistantAudioPlaying(false);
|
||||
if (!state.snapshot.isVoiceMode) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.turnInProgress) {
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "waiting" }));
|
||||
reconcileCue();
|
||||
return;
|
||||
}
|
||||
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
|
||||
reconcileCue();
|
||||
},
|
||||
|
||||
onTranscriptionResult(serverId, text) {
|
||||
if (
|
||||
serverId !== state.snapshot.activeServerId ||
|
||||
!state.snapshot.isVoiceMode
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (text.trim()) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.turnInProgress = false;
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
|
||||
stopCue();
|
||||
},
|
||||
|
||||
onTurnEvent(serverId, agentId, eventType) {
|
||||
if (
|
||||
!state.snapshot.isVoiceMode ||
|
||||
state.snapshot.activeServerId !== serverId ||
|
||||
state.snapshot.activeAgentId !== agentId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (eventType === "turn_started") {
|
||||
state.turnInProgress = true;
|
||||
return;
|
||||
}
|
||||
|
||||
state.turnInProgress = false;
|
||||
if (state.snapshot.phase !== "playing") {
|
||||
patchSnapshot((prev) => ({ ...prev, phase: "listening" }));
|
||||
}
|
||||
stopCue();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/cli",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"description": "Paseo CLI - control your AI coding agents from the command line",
|
||||
"type": "module",
|
||||
"files": [
|
||||
@@ -24,8 +24,8 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@clack/prompts": "^1.0.0",
|
||||
"@getpaseo/relay": "0.1.26",
|
||||
"@getpaseo/server": "0.1.26",
|
||||
"@getpaseo/relay": "0.1.27",
|
||||
"@getpaseo/server": "0.1.27",
|
||||
"chalk": "^5.3.0",
|
||||
"commander": "^12.0.0",
|
||||
"mime-types": "^2.1.35",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/desktop",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"private": true,
|
||||
"description": "Paseo desktop app (Tauri wrapper)",
|
||||
"scripts": {
|
||||
|
||||
@@ -288,26 +288,23 @@ async function installPackedWorkspaces(runtimeRoot, bundledNodeRoot, tarballs) {
|
||||
const npmCli = process.platform === "win32"
|
||||
? path.join(bundledNodeRoot, "node_modules", "npm", "bin", "npm-cli.js")
|
||||
: path.join(bundledNodeRoot, "lib", "node_modules", "npm", "bin", "npm-cli.js");
|
||||
const install = spawnSync(
|
||||
nodeExecutable,
|
||||
[
|
||||
npmCli,
|
||||
"install",
|
||||
"--omit=dev",
|
||||
"--no-package-lock",
|
||||
"--no-save",
|
||||
...tarballs,
|
||||
],
|
||||
{
|
||||
cwd: runtimeRoot,
|
||||
stdio: "inherit",
|
||||
env: {
|
||||
...process.env,
|
||||
npm_config_audit: "false",
|
||||
npm_config_fund: "false",
|
||||
},
|
||||
}
|
||||
);
|
||||
const install = spawnSync(nodeExecutable, [
|
||||
npmCli,
|
||||
"install",
|
||||
"--include=optional",
|
||||
"--omit=dev",
|
||||
"--no-package-lock",
|
||||
"--no-save",
|
||||
...tarballs,
|
||||
], {
|
||||
cwd: runtimeRoot,
|
||||
stdio: "inherit",
|
||||
env: {
|
||||
...process.env,
|
||||
npm_config_audit: "false",
|
||||
npm_config_fund: "false",
|
||||
},
|
||||
});
|
||||
if (install.status !== 0) {
|
||||
throw new Error(
|
||||
`Managed runtime dependency install failed with exit code ${install.status ?? 1}.`
|
||||
@@ -315,6 +312,67 @@ async function installPackedWorkspaces(runtimeRoot, bundledNodeRoot, tarballs) {
|
||||
}
|
||||
}
|
||||
|
||||
function resolveSherpaNativePackageName() {
|
||||
const sherpaPlatformMap = {
|
||||
darwin: "darwin",
|
||||
linux: "linux",
|
||||
win32: "win",
|
||||
};
|
||||
const sherpaPlatform = sherpaPlatformMap[process.platform];
|
||||
if (!sherpaPlatform) {
|
||||
throw new Error(`Managed runtime sherpa package is not implemented for ${process.platform}.`);
|
||||
}
|
||||
return `sherpa-onnx-${sherpaPlatform}-${process.arch}`;
|
||||
}
|
||||
|
||||
async function ensureSherpaNativePackage(runtimeRoot, bundledNodeRoot) {
|
||||
const packageName = resolveSherpaNativePackageName();
|
||||
const packageDir = path.join(runtimeRoot, "node_modules", packageName);
|
||||
if (await pathExists(packageDir)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sherpaNodePackageJson = JSON.parse(
|
||||
await fs.readFile(
|
||||
path.join(runtimeRoot, "node_modules", "sherpa-onnx-node", "package.json"),
|
||||
"utf8"
|
||||
)
|
||||
);
|
||||
const packageVersion = sherpaNodePackageJson.version;
|
||||
if (!packageVersion) {
|
||||
throw new Error("Missing version in sherpa-onnx-node package.json.");
|
||||
}
|
||||
|
||||
const nodeExecutable = process.platform === "win32"
|
||||
? path.join(bundledNodeRoot, "node.exe")
|
||||
: path.join(bundledNodeRoot, "bin", "node");
|
||||
const npmCli = process.platform === "win32"
|
||||
? path.join(bundledNodeRoot, "node_modules", "npm", "bin", "npm-cli.js")
|
||||
: path.join(bundledNodeRoot, "lib", "node_modules", "npm", "bin", "npm-cli.js");
|
||||
const install = spawnSync(nodeExecutable, [
|
||||
npmCli,
|
||||
"install",
|
||||
"--include=optional",
|
||||
"--omit=dev",
|
||||
"--no-package-lock",
|
||||
"--no-save",
|
||||
`${packageName}@${packageVersion}`,
|
||||
], {
|
||||
cwd: runtimeRoot,
|
||||
stdio: "inherit",
|
||||
env: {
|
||||
...process.env,
|
||||
npm_config_audit: "false",
|
||||
npm_config_fund: "false",
|
||||
},
|
||||
});
|
||||
if (install.status !== 0) {
|
||||
throw new Error(
|
||||
`Managed runtime sherpa package install failed with exit code ${install.status ?? 1}.`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeRuntimePackageJson(runtimeRoot) {
|
||||
await fs.writeFile(
|
||||
path.join(runtimeRoot, "package.json"),
|
||||
@@ -422,71 +480,12 @@ async function pruneClaudeAgentSdk(runtimeRoot) {
|
||||
}
|
||||
}
|
||||
|
||||
async function pruneCodexCli(runtimeRoot) {
|
||||
const codexTargetTriple = {
|
||||
darwin: { arm64: "aarch64-apple-darwin", x64: "x86_64-apple-darwin" },
|
||||
linux: { arm64: "aarch64-unknown-linux-musl", x64: "x86_64-unknown-linux-musl" },
|
||||
win32: { arm64: "aarch64-pc-windows-msvc", x64: "x86_64-pc-windows-msvc" },
|
||||
};
|
||||
const codexPlatformPackages = [
|
||||
"@openai/codex-darwin-arm64",
|
||||
"@openai/codex-darwin-x64",
|
||||
"@openai/codex-linux-arm64",
|
||||
"@openai/codex-linux-x64",
|
||||
"@openai/codex-win32-arm64",
|
||||
"@openai/codex-win32-x64",
|
||||
];
|
||||
const currentTriple = codexTargetTriple[process.platform]?.[process.arch];
|
||||
const currentPackage = currentTriple
|
||||
? `@openai/codex-${process.platform === "win32" ? "win32" : process.platform}-${process.arch}`
|
||||
: null;
|
||||
|
||||
const openaiScope = path.join(runtimeRoot, "node_modules", "@openai");
|
||||
for (const pkg of codexPlatformPackages) {
|
||||
const pkgName = pkg.replace("@openai/", "");
|
||||
if (currentPackage && pkg === currentPackage) {
|
||||
continue;
|
||||
}
|
||||
await removeIfExists(path.join(openaiScope, pkgName));
|
||||
}
|
||||
}
|
||||
|
||||
async function pruneOpenCodeCli(runtimeRoot) {
|
||||
const opencodePlatformPackages = [
|
||||
"opencode-darwin-arm64",
|
||||
"opencode-darwin-x64",
|
||||
"opencode-linux-arm64",
|
||||
"opencode-linux-x64",
|
||||
"opencode-linux-x64-baseline",
|
||||
"opencode-linux-arm64-musl",
|
||||
"opencode-linux-x64-musl",
|
||||
"opencode-linux-x64-baseline-musl",
|
||||
"opencode-windows-x64",
|
||||
"opencode-windows-arm64",
|
||||
];
|
||||
const platformMap = { darwin: "darwin", linux: "linux", win32: "windows" };
|
||||
const currentPlatform = platformMap[process.platform];
|
||||
const currentPackage = currentPlatform
|
||||
? `opencode-${currentPlatform}-${process.arch}`
|
||||
: null;
|
||||
|
||||
const nodeModules = path.join(runtimeRoot, "node_modules");
|
||||
for (const pkg of opencodePlatformPackages) {
|
||||
if (currentPackage && pkg === currentPackage) {
|
||||
continue;
|
||||
}
|
||||
await removeIfExists(path.join(nodeModules, pkg));
|
||||
}
|
||||
}
|
||||
|
||||
async function pruneManagedRuntime(runtimeRoot) {
|
||||
await Promise.all([
|
||||
pruneNodeDistribution(runtimeRoot),
|
||||
pruneOnnxRuntime(runtimeRoot),
|
||||
pruneNodePty(runtimeRoot),
|
||||
pruneClaudeAgentSdk(runtimeRoot),
|
||||
pruneCodexCli(runtimeRoot),
|
||||
pruneOpenCodeCli(runtimeRoot),
|
||||
]);
|
||||
}
|
||||
|
||||
@@ -528,6 +527,7 @@ async function main() {
|
||||
path.join(runtimeRoot, "node"),
|
||||
tarballs.map((entry) => entry.path)
|
||||
);
|
||||
await ensureSherpaNativePackage(runtimeRoot, path.join(runtimeRoot, "node"));
|
||||
await pruneManagedRuntime(runtimeRoot);
|
||||
|
||||
const nodeRelativePath = process.platform === "win32"
|
||||
|
||||
3
packages/desktop/src-tauri/Cargo.lock
generated
3
packages/desktop/src-tauri/Cargo.lock
generated
@@ -2629,7 +2629,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "paseo"
|
||||
version = "0.1.26"
|
||||
version = "0.1.27"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"dirs",
|
||||
@@ -2637,6 +2637,7 @@ dependencies = [
|
||||
"futures-util",
|
||||
"http",
|
||||
"log",
|
||||
"mac-notification-sys",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tauri",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "paseo"
|
||||
version = "0.1.26"
|
||||
version = "0.1.27"
|
||||
description = "Paseo Desktop"
|
||||
authors = ["moboudra"]
|
||||
license = "MIT"
|
||||
@@ -41,6 +41,9 @@ tauri-plugin-websocket = "2"
|
||||
tokio = { version = "1", features = ["net", "sync"] }
|
||||
tokio-tungstenite = "0.24"
|
||||
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
mac-notification-sys = "0.6"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -7,9 +7,9 @@
|
||||
<key>CFBundleName</key>
|
||||
<string>Paseo</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>0.1.26</string>
|
||||
<string>0.1.27</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>0.1.26</string>
|
||||
<string>0.1.27</string>
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>Paseo needs access to your microphone for voice dictation and voice mode.</string>
|
||||
</dict>
|
||||
|
||||
110
packages/desktop/src-tauri/src/desktop_notifications.rs
Normal file
110
packages/desktop/src-tauri/src/desktop_notifications.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tauri::{AppHandle, Runtime};
|
||||
|
||||
pub const DESKTOP_NOTIFICATION_CLICK_EVENT: &str = "desktop-notification-click";
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DesktopNotificationInput {
|
||||
pub title: String,
|
||||
pub body: Option<String>,
|
||||
pub data: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct DesktopNotificationClickPayload {
|
||||
data: Option<Value>,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
mod platform {
|
||||
use super::{
|
||||
AppHandle, DesktopNotificationClickPayload, DesktopNotificationInput,
|
||||
DESKTOP_NOTIFICATION_CLICK_EVENT,
|
||||
};
|
||||
use std::sync::OnceLock;
|
||||
use tauri::{async_runtime, Emitter, Runtime};
|
||||
|
||||
const MACOS_NOTIFICATION_SENDER_BUNDLE_ID: &str = "dev.paseo.desktop";
|
||||
static MACOS_NOTIFICATION_SENDER_INIT: OnceLock<Result<(), String>> = OnceLock::new();
|
||||
|
||||
fn ensure_notification_sender() -> Result<(), String> {
|
||||
MACOS_NOTIFICATION_SENDER_INIT
|
||||
.get_or_init(|| {
|
||||
mac_notification_sys::set_application(MACOS_NOTIFICATION_SENDER_BUNDLE_ID)
|
||||
.map_err(|error| {
|
||||
format!(
|
||||
"Failed to initialize macOS notification sender for {}: {error}",
|
||||
MACOS_NOTIFICATION_SENDER_BUNDLE_ID
|
||||
)
|
||||
})
|
||||
})
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub fn send_notification<R: Runtime>(
|
||||
app: AppHandle<R>,
|
||||
input: DesktopNotificationInput,
|
||||
) -> Result<(), String> {
|
||||
let _task = async_runtime::spawn_blocking(move || {
|
||||
if let Err(error) = ensure_notification_sender() {
|
||||
log::warn!("{error}");
|
||||
return;
|
||||
}
|
||||
|
||||
let mut notification = mac_notification_sys::Notification::default();
|
||||
notification
|
||||
.title(&input.title)
|
||||
.message(input.body.as_deref().unwrap_or(""))
|
||||
.wait_for_click(true);
|
||||
|
||||
match notification.send() {
|
||||
Ok(mac_notification_sys::NotificationResponse::Click) => {
|
||||
let payload = DesktopNotificationClickPayload { data: input.data };
|
||||
if let Err(error) = app.emit(DESKTOP_NOTIFICATION_CLICK_EVENT, payload) {
|
||||
log::warn!("failed to emit desktop notification click event: {error}");
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
log::warn!("failed to send macOS notification: {error}");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
mod platform {
|
||||
use super::{AppHandle, DesktopNotificationInput};
|
||||
use tauri::Runtime;
|
||||
use tauri_plugin_notification::NotificationExt;
|
||||
|
||||
pub fn send_notification<R: Runtime>(
|
||||
app: AppHandle<R>,
|
||||
input: DesktopNotificationInput,
|
||||
) -> Result<(), String> {
|
||||
let mut notification = app.notification().builder().title(&input.title);
|
||||
if let Some(body) = &input.body {
|
||||
notification = notification.body(body);
|
||||
}
|
||||
|
||||
notification
|
||||
.show()
|
||||
.map_err(|error| format!("Failed to send desktop notification: {error}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn send_desktop_notification<R: Runtime>(
|
||||
app: AppHandle<R>,
|
||||
input: DesktopNotificationInput,
|
||||
) -> Result<(), String> {
|
||||
platform::send_notification(app, input)
|
||||
}
|
||||
@@ -11,7 +11,9 @@ use tauri::menu::{Menu, MenuItemBuilder, MenuItemKind, PredefinedMenuItem, Subme
|
||||
use tauri::{AppHandle, Manager, WebviewWindow};
|
||||
use tauri_plugin_updater::UpdaterExt;
|
||||
|
||||
mod desktop_notifications;
|
||||
mod runtime_manager;
|
||||
use desktop_notifications::send_desktop_notification;
|
||||
use runtime_manager::{
|
||||
cli_symlink_instructions, close_local_daemon_transport, managed_daemon_logs,
|
||||
managed_daemon_pairing, managed_daemon_status, managed_runtime_status,
|
||||
@@ -544,6 +546,7 @@ pub fn run() {
|
||||
read_file_base64,
|
||||
delete_attachment_file,
|
||||
garbage_collect_attachment_files,
|
||||
send_desktop_notification,
|
||||
webview_log
|
||||
])
|
||||
.setup(|app| {
|
||||
|
||||
@@ -821,6 +821,7 @@ pub fn run_managed_cli_from_current_process(args: Vec<String>) -> Result<i32, St
|
||||
if let Some(contents_dir) = exe_dir.parent() {
|
||||
resource_dirs.push(contents_dir.join("Resources"));
|
||||
resource_dirs.push(contents_dir.join("resources"));
|
||||
resource_dirs.push(contents_dir.join("lib").join("Paseo").join("resources"));
|
||||
}
|
||||
let bundled_root = resource_dirs
|
||||
.into_iter()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
|
||||
"productName": "Paseo",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"identifier": "dev.paseo.desktop",
|
||||
"build": {
|
||||
"frontendDist": "../../app/dist",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/relay",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"description": "Paseo relay for bridging daemon and client connections",
|
||||
"type": "module",
|
||||
"publishConfig": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@getpaseo/server",
|
||||
"version": "0.1.26",
|
||||
"version": "0.1.27",
|
||||
"description": "Paseo backend server",
|
||||
"type": "module",
|
||||
"publishConfig": {
|
||||
@@ -62,9 +62,8 @@
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "2.0.52",
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.2.11",
|
||||
"@openai/codex": "^0.114.0",
|
||||
"@deepgram/sdk": "^3.4.0",
|
||||
"@getpaseo/relay": "0.1.26",
|
||||
"@getpaseo/relay": "0.1.27",
|
||||
"@lezer/common": "^1.5.0",
|
||||
"@lezer/css": "^1.3.0",
|
||||
"@lezer/highlight": "^1.2.3",
|
||||
@@ -88,13 +87,13 @@
|
||||
"node-pty": "1.2.0-beta.11",
|
||||
"onnxruntime-node": "^1.23.0",
|
||||
"openai": "^4.20.0",
|
||||
"opencode-ai": "^1.2.24",
|
||||
"pino": "^10.2.0",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"qrcode": "^1.5.4",
|
||||
"rotating-file-stream": "^3.2.9",
|
||||
"sherpa-onnx": "^1.12.23",
|
||||
"sherpa-onnx-node": "^1.12.23",
|
||||
"shell-env": "^4.0.3",
|
||||
"sherpa-onnx": "1.12.28",
|
||||
"sherpa-onnx-node": "1.12.28",
|
||||
"strip-ansi": "^7.1.2",
|
||||
"tiny-invariant": "^1.3.3",
|
||||
"uuid": "^9.0.1",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { execFileSync, execSync } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { platform } from "node:os";
|
||||
import { shellEnvSync } from "shell-env";
|
||||
import { z } from "zod";
|
||||
|
||||
import type { AgentProvider } from "./agent-sdk-types.js";
|
||||
@@ -104,11 +105,24 @@ export function resolveProviderCommandPrefix(
|
||||
};
|
||||
}
|
||||
|
||||
let cachedShellEnv: Record<string, string> | null = null;
|
||||
|
||||
export function resolveShellEnv(): Record<string, string> {
|
||||
if (cachedShellEnv) return cachedShellEnv;
|
||||
try {
|
||||
cachedShellEnv = shellEnvSync();
|
||||
} catch {
|
||||
cachedShellEnv = { ...process.env } as Record<string, string>;
|
||||
}
|
||||
return cachedShellEnv;
|
||||
}
|
||||
|
||||
export function applyProviderEnv(
|
||||
baseEnv: Record<string, string | undefined>,
|
||||
runtimeSettings?: ProviderRuntimeSettings
|
||||
): Record<string, string | undefined> {
|
||||
return {
|
||||
...resolveShellEnv(),
|
||||
...baseEnv,
|
||||
...(runtimeSettings?.env ?? {}),
|
||||
};
|
||||
|
||||
@@ -96,6 +96,12 @@ const ClaudeToolDetailPass2Schema = z.union([
|
||||
toolDetailBranchByName("search", ToolSearchInputSchema, z.unknown(), (input) =>
|
||||
toSearchToolDetail(input)
|
||||
),
|
||||
toolDetailBranchByName("Grep", ToolSearchInputSchema, z.unknown(), (input) =>
|
||||
toSearchToolDetail(input)
|
||||
),
|
||||
toolDetailBranchByName("grep", ToolSearchInputSchema, z.unknown(), (input) =>
|
||||
toSearchToolDetail(input)
|
||||
),
|
||||
toolDetailBranchByName("Glob", ToolSearchInputSchema, z.unknown(), (input) =>
|
||||
toSearchToolDetail(input)
|
||||
),
|
||||
|
||||
@@ -245,6 +245,30 @@ describe("claude tool-call mapper", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("maps Grep calls as search detail using pattern input", () => {
|
||||
const item = expectMapped(
|
||||
mapClaudeCompletedToolCall({
|
||||
callId: "claude-grep-1",
|
||||
name: "Grep",
|
||||
input: {
|
||||
pattern: '\\\\\\"cli\\\\\\""',
|
||||
path: "/Users/moboudra/dev/paseo/packages/desktop/src-tauri/src",
|
||||
output_mode: "content",
|
||||
"-n": true,
|
||||
},
|
||||
output: { output: "No matches found" },
|
||||
})
|
||||
);
|
||||
|
||||
expect(item.status).toBe("completed");
|
||||
expect(item.error).toBeNull();
|
||||
expect(item.name).toBe("Grep");
|
||||
expect(item.detail).toEqual({
|
||||
type: "search",
|
||||
query: '\\\\\\"cli\\\\\\""',
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes claude speak tool names through schema transforms", () => {
|
||||
const item = expectMapped(
|
||||
mapClaudeCompletedToolCall({
|
||||
|
||||
@@ -75,6 +75,8 @@ const ClaudeSearchToolNameSchema = z.union([
|
||||
z.literal("WebSearch"),
|
||||
z.literal("web_search"),
|
||||
z.literal("search"),
|
||||
z.literal("Grep"),
|
||||
z.literal("grep"),
|
||||
z.literal("Glob"),
|
||||
z.literal("glob"),
|
||||
]);
|
||||
|
||||
@@ -11,9 +11,12 @@ import {
|
||||
} from "./codex-app-server-agent.js";
|
||||
import { createTestLogger } from "../../../test-utils/test-logger.js";
|
||||
import { agentConfigs } from "../../daemon-e2e/agent-configs.js";
|
||||
import { AgentManager } from "../agent-manager.js";
|
||||
import { AgentStorage } from "../agent-storage.js";
|
||||
import type {
|
||||
AgentPermissionRequest,
|
||||
AgentPromptContentBlock,
|
||||
AgentStreamEvent,
|
||||
AgentTimelineItem,
|
||||
} from "../agent-sdk-types.js";
|
||||
|
||||
@@ -179,6 +182,22 @@ function readRolloutTurnContextEfforts(rolloutPath: string): string[] {
|
||||
return efforts;
|
||||
}
|
||||
|
||||
type Deferred<T> = {
|
||||
promise: Promise<T>;
|
||||
resolve: (value: T) => void;
|
||||
reject: (reason?: unknown) => void;
|
||||
};
|
||||
|
||||
function deferred<T>(): Deferred<T> {
|
||||
let resolve!: (value: T) => void;
|
||||
let reject!: (reason?: unknown) => void;
|
||||
const promise = new Promise<T>((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
describe("Codex app-server provider (integration)", () => {
|
||||
const logger = createTestLogger();
|
||||
|
||||
@@ -1457,6 +1476,134 @@ describe("Codex app-server provider (integration)", () => {
|
||||
120000
|
||||
);
|
||||
|
||||
test.runIf(isCodexInstalled())(
|
||||
"replaceAgentRun keeps the replacement Codex stream alive when the previous interrupted turn completes late",
|
||||
async () => {
|
||||
const cleanup = useTempCodexSessionDir();
|
||||
const cwd = tmpCwd("codex-replace-run-");
|
||||
const storageDir = tmpCwd("codex-replace-run-storage-");
|
||||
|
||||
try {
|
||||
const manager = new AgentManager({
|
||||
clients: {
|
||||
codex: new CodexAppServerAgentClient(logger),
|
||||
},
|
||||
registry: new AgentStorage(storageDir, logger),
|
||||
logger,
|
||||
});
|
||||
|
||||
const snapshot = await manager.createAgent({
|
||||
provider: "codex",
|
||||
cwd,
|
||||
modeId: "auto",
|
||||
model: CODEX_TEST_MODEL,
|
||||
thinkingOptionId: CODEX_TEST_THINKING_OPTION_ID,
|
||||
});
|
||||
|
||||
const managedAgent = ((manager as any).agents.get(snapshot.id) ?? null) as
|
||||
| { session?: any }
|
||||
| null;
|
||||
const session = managedAgent?.session as
|
||||
| {
|
||||
client?: {
|
||||
request: (method: string, params?: unknown, timeoutMs?: number) => Promise<unknown>;
|
||||
} | null;
|
||||
handleNotification?: (method: string, params: unknown) => void;
|
||||
}
|
||||
| undefined;
|
||||
if (!session?.client || !session.handleNotification) {
|
||||
throw new Error("Codex session internals unavailable for replaceAgentRun regression test");
|
||||
}
|
||||
|
||||
let turnStartCount = 0;
|
||||
const replacementTurnInjected = deferred<void>();
|
||||
const originalRequest = session.client.request.bind(session.client);
|
||||
session.client.request = async (method: string, params?: unknown, timeoutMs?: number) => {
|
||||
if (method === "turn/start") {
|
||||
turnStartCount += 1;
|
||||
if (turnStartCount === 1) {
|
||||
queueMicrotask(() => {
|
||||
session.handleNotification?.("turn/started", {
|
||||
turn: { id: "initial-turn" },
|
||||
});
|
||||
});
|
||||
} else if (turnStartCount === 2) {
|
||||
queueMicrotask(() => {
|
||||
session.handleNotification?.("turn/completed", {
|
||||
turn: { status: "interrupted" },
|
||||
});
|
||||
session.handleNotification?.("turn/started", {
|
||||
turn: { id: "replacement-turn" },
|
||||
});
|
||||
session.handleNotification?.("turn/completed", {
|
||||
turn: { status: "completed" },
|
||||
});
|
||||
replacementTurnInjected.resolve(undefined);
|
||||
});
|
||||
}
|
||||
}
|
||||
return originalRequest(method, params, timeoutMs);
|
||||
};
|
||||
|
||||
const firstRun = manager.streamAgent(
|
||||
snapshot.id,
|
||||
"Keep working until you are interrupted."
|
||||
);
|
||||
const firstRunReady = deferred<void>();
|
||||
const firstRunDrain = (async () => {
|
||||
for await (const event of firstRun) {
|
||||
if (event.type === "turn_started") {
|
||||
firstRunReady.resolve(undefined);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
await Promise.race([
|
||||
firstRunReady.promise,
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(
|
||||
() => reject(new Error("Timed out waiting for initial Codex turn to start")),
|
||||
15_000
|
||||
)
|
||||
),
|
||||
]);
|
||||
|
||||
const replacementEvents: AgentStreamEvent[] = [];
|
||||
await Promise.race([
|
||||
(async () => {
|
||||
for await (const event of manager.replaceAgentRun(
|
||||
snapshot.id,
|
||||
"Reply exactly REPLACED and stop."
|
||||
)) {
|
||||
replacementEvents.push(event);
|
||||
}
|
||||
})(),
|
||||
new Promise<never>((_, reject) =>
|
||||
setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new Error("Timed out waiting for replacement Codex stream to finish")
|
||||
),
|
||||
60_000
|
||||
)
|
||||
),
|
||||
]);
|
||||
|
||||
await replacementTurnInjected.promise;
|
||||
await firstRunDrain;
|
||||
|
||||
expect(turnStartCount).toBeGreaterThanOrEqual(2);
|
||||
expect(replacementEvents.some((event) => event.type === "turn_started")).toBe(true);
|
||||
expect(replacementEvents.some((event) => event.type === "turn_completed")).toBe(true);
|
||||
} finally {
|
||||
cleanup();
|
||||
rmSync(storageDir, { recursive: true, force: true });
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
120000
|
||||
);
|
||||
|
||||
test.runIf(isCodexInstalled())(
|
||||
"persists session metadata and resumes with history",
|
||||
async () => {
|
||||
|
||||
@@ -27,10 +27,8 @@ import type { Logger } from "pino";
|
||||
import { execSync, spawn } from "node:child_process";
|
||||
import type { ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { existsSync, Dirent } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import { Dirent } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import readline from "node:readline";
|
||||
@@ -42,9 +40,11 @@ import {
|
||||
} from "./codex/tool-call-mapper.js";
|
||||
import {
|
||||
applyProviderEnv,
|
||||
findExecutable,
|
||||
resolveProviderCommandPrefix,
|
||||
type ProviderRuntimeSettings,
|
||||
} from "../provider-launch-config.js";
|
||||
import { extractCodexTerminalSessionId, nonEmptyString } from "./tool-call-mapper-utils.js";
|
||||
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 14 * 24 * 60 * 60 * 1000;
|
||||
@@ -150,52 +150,13 @@ function mergeCodexConfiguredDefaults(
|
||||
};
|
||||
}
|
||||
|
||||
const CODEX_TARGET_TRIPLE: Record<string, Record<string, string>> = {
|
||||
linux: { x64: "x86_64-unknown-linux-musl", arm64: "aarch64-unknown-linux-musl" },
|
||||
darwin: { x64: "x86_64-apple-darwin", arm64: "aarch64-apple-darwin" },
|
||||
win32: { x64: "x86_64-pc-windows-msvc", arm64: "aarch64-pc-windows-msvc" },
|
||||
};
|
||||
|
||||
const CODEX_PLATFORM_PACKAGE: Record<string, string> = {
|
||||
"x86_64-unknown-linux-musl": "@openai/codex-linux-x64",
|
||||
"aarch64-unknown-linux-musl": "@openai/codex-linux-arm64",
|
||||
"x86_64-apple-darwin": "@openai/codex-darwin-x64",
|
||||
"aarch64-apple-darwin": "@openai/codex-darwin-arm64",
|
||||
"x86_64-pc-windows-msvc": "@openai/codex-win32-x64",
|
||||
"aarch64-pc-windows-msvc": "@openai/codex-win32-arm64",
|
||||
};
|
||||
|
||||
function resolveBundledCodexBinary(): string | null {
|
||||
const targetTriple = CODEX_TARGET_TRIPLE[process.platform]?.[process.arch];
|
||||
if (!targetTriple) {
|
||||
return null;
|
||||
}
|
||||
const platformPackage = CODEX_PLATFORM_PACKAGE[targetTriple];
|
||||
if (!platformPackage) {
|
||||
return null;
|
||||
}
|
||||
const binaryName = process.platform === "win32" ? "codex.exe" : "codex";
|
||||
try {
|
||||
const require = createRequire(import.meta.url);
|
||||
const packageJsonPath = require.resolve(`${platformPackage}/package.json`);
|
||||
const vendorRoot = path.join(path.dirname(packageJsonPath), "vendor");
|
||||
const binaryPath = path.join(vendorRoot, targetTriple, "codex", binaryName);
|
||||
if (existsSync(binaryPath)) {
|
||||
return binaryPath;
|
||||
}
|
||||
} catch {
|
||||
// Platform package not installed
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveCodexBinary(): string {
|
||||
const bundled = resolveBundledCodexBinary();
|
||||
if (bundled) {
|
||||
return bundled;
|
||||
const found = findExecutable("codex");
|
||||
if (found) {
|
||||
return found;
|
||||
}
|
||||
throw new Error(
|
||||
`Bundled Codex binary not found for platform ${process.platform}-${process.arch}. Ensure the @openai/codex package and its platform-specific optional dependency are installed.`
|
||||
"Codex binary not found. Install the Codex CLI (https://github.com/openai/codex) and ensure it is available in your shell PATH."
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1094,6 +1055,32 @@ function mapCodexPatchNotificationToToolCall(params: {
|
||||
return params.running ? toRunningToolCall(mapped) : mapped;
|
||||
}
|
||||
|
||||
function mapCodexTerminalInteractionToToolCall(params: {
|
||||
processId?: string | null;
|
||||
fallbackCallId?: string | null;
|
||||
command?: string | null;
|
||||
}): ToolCallTimelineItem {
|
||||
const processId = nonEmptyString(params.processId ?? undefined);
|
||||
const callId =
|
||||
processId
|
||||
? `terminal-session-${processId}`
|
||||
: nonEmptyString(params.fallbackCallId ?? undefined) ?? "terminal-interaction";
|
||||
const label = nonEmptyString(params.command ?? undefined);
|
||||
return {
|
||||
type: "tool_call",
|
||||
callId,
|
||||
name: "terminal",
|
||||
status: "completed",
|
||||
error: null,
|
||||
detail: {
|
||||
type: "plain_text",
|
||||
...(label ? { label } : {}),
|
||||
icon: "square_terminal",
|
||||
},
|
||||
...(processId ? { metadata: { processId } } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function threadItemToTimeline(
|
||||
item: any,
|
||||
options?: { includeUserMessage?: boolean; cwd?: string | null }
|
||||
@@ -1318,6 +1305,23 @@ const CodexEventExecCommandOutputDeltaNotificationSchema = z.object({
|
||||
.passthrough(),
|
||||
}).passthrough();
|
||||
|
||||
const CodexEventTerminalInteractionNotificationSchema = z.object({
|
||||
msg: z
|
||||
.object({
|
||||
type: z.literal("terminal_interaction"),
|
||||
call_id: z.string().optional(),
|
||||
process_id: z.union([z.string(), z.number()]).optional(),
|
||||
stdin: z.string().optional(),
|
||||
})
|
||||
.passthrough(),
|
||||
}).passthrough();
|
||||
|
||||
const ItemCommandExecutionTerminalInteractionNotificationSchema = z.object({
|
||||
itemId: z.string().optional(),
|
||||
processId: z.union([z.string(), z.number()]).optional(),
|
||||
stdin: z.string().optional(),
|
||||
}).passthrough();
|
||||
|
||||
const CodexEventPatchApplyBeginNotificationSchema = z.object({
|
||||
msg: z
|
||||
.object({
|
||||
@@ -1398,6 +1402,13 @@ type ParsedCodexNotification =
|
||||
stream: string | null;
|
||||
chunk: string | null;
|
||||
}
|
||||
| {
|
||||
kind: "terminal_interaction";
|
||||
source: "item" | "codex_event";
|
||||
callId: string | null;
|
||||
processId: string | null;
|
||||
stdin: string | null;
|
||||
}
|
||||
| {
|
||||
kind: "patch_apply_started";
|
||||
callId: string | null;
|
||||
@@ -1584,6 +1595,45 @@ const CodexNotificationSchema = z.union([
|
||||
}).transform(
|
||||
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
|
||||
),
|
||||
z.object({
|
||||
method: z.literal("codex/event/terminal_interaction"),
|
||||
params: CodexEventTerminalInteractionNotificationSchema,
|
||||
}).transform(
|
||||
({ params }): ParsedCodexNotification => ({
|
||||
kind: "terminal_interaction",
|
||||
source: "codex_event",
|
||||
callId: params.msg.call_id ?? null,
|
||||
processId:
|
||||
typeof params.msg.process_id === "number"
|
||||
? String(params.msg.process_id)
|
||||
: params.msg.process_id ?? null,
|
||||
stdin: params.msg.stdin ?? null,
|
||||
})
|
||||
),
|
||||
z.object({ method: z.literal("codex/event/terminal_interaction"), params: z.unknown() }).transform(
|
||||
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
|
||||
),
|
||||
z.object({
|
||||
method: z.literal("item/commandExecution/terminalInteraction"),
|
||||
params: ItemCommandExecutionTerminalInteractionNotificationSchema,
|
||||
}).transform(
|
||||
({ params }): ParsedCodexNotification => ({
|
||||
kind: "terminal_interaction",
|
||||
source: "item",
|
||||
callId: params.itemId ?? null,
|
||||
processId:
|
||||
typeof params.processId === "number"
|
||||
? String(params.processId)
|
||||
: params.processId ?? null,
|
||||
stdin: params.stdin ?? null,
|
||||
})
|
||||
),
|
||||
z.object({
|
||||
method: z.literal("item/commandExecution/terminalInteraction"),
|
||||
params: z.unknown(),
|
||||
}).transform(
|
||||
({ method, params }): ParsedCodexNotification => ({ kind: "invalid_payload", method, params })
|
||||
),
|
||||
z.object({
|
||||
method: z.literal("codex/event/patch_apply_begin"),
|
||||
params: CodexEventPatchApplyBeginNotificationSchema,
|
||||
@@ -1798,6 +1848,8 @@ class CodexAppServerAgentSession implements AgentSession {
|
||||
private pendingReasoning = new Map<string, string[]>();
|
||||
private pendingCommandOutputDeltas = new Map<string, string[]>();
|
||||
private pendingFileChangeOutputDeltas = new Map<string, string[]>();
|
||||
private terminalCommandByProcessId = new Map<string, string>();
|
||||
private emittedTerminalInteractionKeys = new Set<string>();
|
||||
private emittedExecCommandStartedCallIds = new Set<string>();
|
||||
private emittedExecCommandCompletedCallIds = new Set<string>();
|
||||
private emittedItemStartedIds = new Set<string>();
|
||||
@@ -2626,7 +2678,6 @@ class CodexAppServerAgentSession implements AgentSession {
|
||||
this.pendingCommandOutputDeltas.clear();
|
||||
this.pendingFileChangeOutputDeltas.clear();
|
||||
this.warnedIncompleteEditToolCallIds.clear();
|
||||
this.eventQueue?.end();
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2708,11 +2759,13 @@ class CodexAppServerAgentSession implements AgentSession {
|
||||
this.pendingCommandOutputDeltas,
|
||||
parsed.callId
|
||||
);
|
||||
const resolvedOutput = parsed.output ?? bufferedOutput;
|
||||
this.rememberTerminalProcessForCommand(parsed.command, resolvedOutput);
|
||||
const timelineItem = mapCodexExecNotificationToToolCall({
|
||||
callId: parsed.callId,
|
||||
command: parsed.command,
|
||||
cwd: parsed.cwd ?? this.config.cwd ?? null,
|
||||
output: parsed.output ?? bufferedOutput,
|
||||
output: resolvedOutput,
|
||||
exitCode: parsed.exitCode,
|
||||
success: parsed.success,
|
||||
stderr: parsed.stderr,
|
||||
@@ -2725,6 +2778,25 @@ class CodexAppServerAgentSession implements AgentSession {
|
||||
return;
|
||||
}
|
||||
|
||||
if (parsed.kind === "terminal_interaction") {
|
||||
const interactionKey = [
|
||||
parsed.processId ?? "",
|
||||
parsed.stdin ?? "",
|
||||
].join("\u0000");
|
||||
if (!this.shouldEmitTerminalInteractionKey(interactionKey)) {
|
||||
return;
|
||||
}
|
||||
const timelineItem = mapCodexTerminalInteractionToToolCall({
|
||||
processId: parsed.processId,
|
||||
fallbackCallId: parsed.callId,
|
||||
command:
|
||||
(parsed.processId ? this.terminalCommandByProcessId.get(parsed.processId) : undefined) ??
|
||||
null,
|
||||
});
|
||||
this.emitEvent({ type: "timeline", provider: CODEX_PROVIDER, item: timelineItem });
|
||||
return;
|
||||
}
|
||||
|
||||
if (parsed.kind === "patch_apply_started") {
|
||||
if (parsed.callId) {
|
||||
this.pendingFileChangeOutputDeltas.delete(parsed.callId);
|
||||
@@ -2917,6 +2989,33 @@ class CodexAppServerAgentSession implements AgentSession {
|
||||
return buffered.join("");
|
||||
}
|
||||
|
||||
private rememberTerminalProcessForCommand(command: unknown, output: string | null): void {
|
||||
const normalizedCommand = normalizeCodexCommandValue(command);
|
||||
if (!normalizedCommand) {
|
||||
return;
|
||||
}
|
||||
const displayCommand =
|
||||
typeof normalizedCommand === "string"
|
||||
? normalizedCommand
|
||||
: normalizedCommand.join(" ").trim();
|
||||
if (!displayCommand) {
|
||||
return;
|
||||
}
|
||||
const processId = extractCodexTerminalSessionId(output ?? undefined);
|
||||
if (!processId) {
|
||||
return;
|
||||
}
|
||||
this.terminalCommandByProcessId.set(processId, displayCommand);
|
||||
}
|
||||
|
||||
private shouldEmitTerminalInteractionKey(key: string): boolean {
|
||||
if (this.emittedTerminalInteractionKeys.has(key)) {
|
||||
return false;
|
||||
}
|
||||
this.emittedTerminalInteractionKeys.add(key);
|
||||
return true;
|
||||
}
|
||||
|
||||
private warnOnIncompleteEditToolCall(
|
||||
item: ToolCallTimelineItem,
|
||||
source: string,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user