Compare commits

...
2 changed files with 597 additions and 0 deletions
+252
View File
@@ -0,0 +1,252 @@
name: CI non-browser routing probe
on:
push:
branches:
- investigate-ci-throughput
paths:
- .github/workflows/ci-routing-probe.yml
workflow_dispatch: {}
permissions:
contents: read
env:
ONNXRUNTIME_NODE_INSTALL: skip
jobs:
changes:
runs-on: ubuntu-latest
outputs:
full: ${{ steps.route.outputs.full }}
browser: ${{ steps.route.outputs.browser }}
format: ${{ steps.route.outputs.format }}
quality: ${{ steps.route.outputs.quality }}
relay: ${{ steps.route.outputs.relay }}
sdk: ${{ steps.route.outputs.sdk }}
steps:
- id: route
run: >-
for contract in full browser quality relay sdk; do echo "$contract=false" >>
"$GITHUB_OUTPUT"; done
echo "format=true" >> "$GITHUB_OUTPUT"
quality:
name: quality
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.format !=
'false' || needs.changes.outputs.quality != 'false' || needs.changes.outputs.relay != 'false'
|| needs.changes.outputs.sdk != 'false') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Install dependencies
id: install
run: node scripts/npm-retry.mjs ci
- name: Check formatting
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.format != 'false') }}
run: npm run format:check
- name: Lint lockfile
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.quality != 'false') }}
run: npx --yes lockfile-lint --path package-lock.json --type npm --allowed-hosts npm
--validate-https --validate-integrity
- name: Verify dependency signatures
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.quality != 'false') }}
run: npm audit signatures
- name: Lint
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.quality != 'false') }}
run: npm run lint
- name: Build relay
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.relay != 'false') }}
run: npm run build:relay
- name: Run relay tests
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.relay != 'false') }}
run: npm run test --workspace=@getpaseo/relay
- name: Build client dependencies
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.sdk != 'false') }}
run: npm run build:relay && npm run build:client
- name: Build plugin SDK
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.sdk != 'false') }}
run: npm run build:plugin
- name: Run plugin SDK tests
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.sdk != 'false') }}
run: npm run test --workspace=@getpaseo/plugin
- name: Run protocol tests
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.sdk != 'false') }}
run: npm run test --workspace=@getpaseo/protocol
- name: Run client tests
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.sdk != 'false') }}
run: npm run test --workspace=@getpaseo/client
- name: Run highlight tests
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.sdk != 'false') }}
run: npm run test --workspace=@getpaseo/highlight
- name: Typecheck client examples
if:
${{ !cancelled() && steps.install.outcome == 'success' && (needs.changes.outputs.full != 'false'
|| needs.changes.outputs.sdk != 'false') }}
run: npm run typecheck:examples --workspace=@getpaseo/client
playwright-1:
name: playwright (shard 1/8)
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.browser !=
'false') }}
runs-on: ubuntu-latest
steps:
- run: echo "Browser coverage was intentionally unrouted; this must not execute"; exit 99
playwright-2:
name: playwright (shard 2/8)
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.browser !=
'false') }}
runs-on: ubuntu-latest
steps:
- run: echo "Browser coverage was intentionally unrouted; this must not execute"; exit 99
playwright-3:
name: playwright (shard 3/8)
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.browser !=
'false') }}
runs-on: ubuntu-latest
steps:
- run: echo "Browser coverage was intentionally unrouted; this must not execute"; exit 99
playwright-4:
name: playwright (shard 4/8)
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.browser !=
'false') }}
runs-on: ubuntu-latest
steps:
- run: echo "Browser coverage was intentionally unrouted; this must not execute"; exit 99
playwright-5:
name: playwright (shard 5/8)
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.browser !=
'false') }}
runs-on: ubuntu-latest
steps:
- run: echo "Browser coverage was intentionally unrouted; this must not execute"; exit 99
playwright-6:
name: playwright (shard 6/8)
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.browser !=
'false') }}
runs-on: ubuntu-latest
steps:
- run: echo "Browser coverage was intentionally unrouted; this must not execute"; exit 99
playwright-7:
name: playwright (shard 7/8)
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.browser !=
'false') }}
runs-on: ubuntu-latest
steps:
- run: echo "Browser coverage was intentionally unrouted; this must not execute"; exit 99
playwright-8:
name: playwright (shard 8/8)
needs: changes
if:
${{ !cancelled() && (needs.changes.outputs.full != 'false' || needs.changes.outputs.browser !=
'false') }}
runs-on: ubuntu-latest
steps:
- run: echo "Browser coverage was intentionally unrouted; this must not execute"; exit 99
playwright:
name: playwright
needs:
- changes
- playwright-1
- playwright-2
- playwright-3
- playwright-4
- playwright-5
- playwright-6
- playwright-7
- playwright-8
if: ${{ !cancelled() }}
runs-on: ubuntu-latest
steps:
- name: Require all routed Playwright shards
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: >
node --input-type=module <<'NODE'
const needs = JSON.parse(process.env.NEEDS_JSON);
const routed = needs.changes.outputs.full !== 'false' || needs.changes.outputs.browser !==
'false';
const failures = Object.entries(needs).filter(([name, job]) =>
job.result !== (name === 'changes' || routed ? 'success' : 'skipped'));
for (const [name, job] of failures) console.error(`${name}: ${job.result}`);
process.exitCode = failures.length ? 1 : 0;
NODE
verify:
needs:
- changes
- quality
- playwright
- playwright-1
- playwright-2
- playwright-3
- playwright-4
- playwright-5
- playwright-6
- playwright-7
- playwright-8
if: ${{ !cancelled() }}
runs-on: ubuntu-latest
steps:
- env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: >-
node --input-type=module <<'NODE'
import assert from 'node:assert/strict';
const needs=JSON.parse(process.env.NEEDS_JSON);
for(const name of ['changes','quality','playwright'])
assert.equal(needs[name].result,'success',name);
for(let i=1;i<=8;i++)
assert.equal(needs['playwright-'+i].result,'skipped','playwright-'+i);
console.log('PASS: real quality ran formatting only; all eight real browser job conditions
skipped; the actual result gate succeeded');
NODE
@@ -0,0 +1,345 @@
name: CI throughput experiment
on:
workflow_dispatch:
inputs:
arm:
description: Run only this comparison arm
type: choice
required: true
default: all
options:
- all
- pw-shard4-w2
- pw-shard4-w3
- pw-shard4-w4
- pw-shard4-w2-prod
- win-unit-mw2
- win-unit-mw3
- win-unit-shard1of3
permissions:
contents: read
concurrency:
group: ci-throughput-experiment-${{ github.run_id }}
cancel-in-progress: false
env:
ONNXRUNTIME_NODE_INSTALL: skip
jobs:
pw-shard4-w2:
name: pw-shard4-w2
runs-on: ubuntu-latest
timeout-minutes: 60
env:
PLAYWRIGHT_SHARD: 4/4
PLAYWRIGHT_ARTIFACT: pw-shard4-w2
E2E_WORKERS: "2"
steps:
- uses: actions/checkout@v4
- name: Record runner capacity
run: node -e "const os=require('node:os');
console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Install dependencies with retry
run: node scripts/npm-retry.mjs ci
- name: Install Playwright browsers
timeout-minutes: 10
run: npx playwright install chromium
- name: Build server stack
run: npm run build:server
- name: Build native app dependency
run: npm run build --workspace=@getpaseo/expo-two-way-audio
- name: Install agent CLIs for provider tests
run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0
opencode-ai
- name: Run Playwright E2E tests
run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ env.PLAYWRIGHT_SHARD }}
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PWTEST_SHARD_WEIGHTS: 111:172:137:160
- name: Upload test artifacts
uses: actions/upload-artifact@v4
if: always()
with:
name: pw-shard4-w2
path: |
packages/app/test-results/
packages/app/playwright-report/
retention-days: 7
if: ${{ inputs.arm == 'all' || inputs.arm == 'pw-shard4-w2' }}
pw-shard4-w3:
name: pw-shard4-w3
runs-on: ubuntu-latest
timeout-minutes: 60
env:
PLAYWRIGHT_SHARD: 4/4
PLAYWRIGHT_ARTIFACT: pw-shard4-w3
E2E_WORKERS: "3"
steps:
- uses: actions/checkout@v4
- name: Record runner capacity
run: node -e "const os=require('node:os');
console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Install dependencies with retry
run: node scripts/npm-retry.mjs ci
- name: Install Playwright browsers
timeout-minutes: 10
run: npx playwright install chromium
- name: Build server stack
run: npm run build:server
- name: Build native app dependency
run: npm run build --workspace=@getpaseo/expo-two-way-audio
- name: Install agent CLIs for provider tests
run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0
opencode-ai
- name: Run Playwright E2E tests
run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ env.PLAYWRIGHT_SHARD }}
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PWTEST_SHARD_WEIGHTS: 111:172:137:160
- name: Upload test artifacts
uses: actions/upload-artifact@v4
if: always()
with:
name: pw-shard4-w3
path: |
packages/app/test-results/
packages/app/playwright-report/
retention-days: 7
if: ${{ inputs.arm == 'all' || inputs.arm == 'pw-shard4-w3' }}
pw-shard4-w4:
name: pw-shard4-w4
runs-on: ubuntu-latest
timeout-minutes: 60
env:
PLAYWRIGHT_SHARD: 4/4
PLAYWRIGHT_ARTIFACT: pw-shard4-w4
E2E_WORKERS: "4"
steps:
- uses: actions/checkout@v4
- name: Record runner capacity
run: node -e "const os=require('node:os');
console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Install dependencies with retry
run: node scripts/npm-retry.mjs ci
- name: Install Playwright browsers
timeout-minutes: 10
run: npx playwright install chromium
- name: Build server stack
run: npm run build:server
- name: Build native app dependency
run: npm run build --workspace=@getpaseo/expo-two-way-audio
- name: Install agent CLIs for provider tests
run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0
opencode-ai
- name: Run Playwright E2E tests
run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ env.PLAYWRIGHT_SHARD }}
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PWTEST_SHARD_WEIGHTS: 111:172:137:160
- name: Upload test artifacts
uses: actions/upload-artifact@v4
if: always()
with:
name: pw-shard4-w4
path: |
packages/app/test-results/
packages/app/playwright-report/
retention-days: 7
if: ${{ inputs.arm == 'all' || inputs.arm == 'pw-shard4-w4' }}
pw-shard4-w2-prod:
name: pw-shard4-w2-prod
runs-on: ubuntu-latest
timeout-minutes: 60
env:
PLAYWRIGHT_SHARD: 4/4
PLAYWRIGHT_ARTIFACT: prod
E2E_WORKERS: "2"
steps:
- uses: actions/checkout@v4
- name: Record runner capacity
run: node -e "const os=require('node:os');
console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Install dependencies with retry
run: node scripts/npm-retry.mjs ci
- name: Install Playwright browsers
timeout-minutes: 10
run: npx playwright install chromium
- name: Build server stack
run: npm run build:server
- name: Build native app dependency
run: npm run build --workspace=@getpaseo/expo-two-way-audio
- name: Install agent CLIs for provider tests
run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0
opencode-ai
- name: Serve the production-mode bundle for this experiment only
run: >-
node <<'NODE'
const fs = require('node:fs');
const file = 'packages/app/e2e/support/global-setup.ts';
const source = fs.readFileSync(file, 'utf8');
const before = '[expoCli, "start", "--web", "--port", String(port)]';
if (source.split(before).length !== 2) throw new Error('Expected one Metro launch');
fs.writeFileSync(file, source.replace(before, '[expoCli, "start", "--web", "--no-dev",
"--minify", "--port", String(port)]'));
const warmup = 'packages/app/e2e/support/metro-warmup.mjs';
const warmupSource = fs.readFileSync(warmup, 'utf8');
if (warmupSource.split('AbortSignal.timeout(120_000)').length !== 3) throw new
Error('Expected one warmup deadline');
fs.writeFileSync(warmup, warmupSource.replaceAll('AbortSignal.timeout(120_000)',
'AbortSignal.timeout(600_000)'));
NODE
- name: Run Playwright E2E tests
run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ env.PLAYWRIGHT_SHARD }}
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
PWTEST_SHARD_WEIGHTS: 111:172:137:160
- name: Upload test artifacts
uses: actions/upload-artifact@v4
if: always()
with:
name: pw-shard4-w2-prod
path: |
packages/app/test-results/
packages/app/playwright-report/
retention-days: 7
if: ${{ inputs.arm == 'all' || inputs.arm == 'pw-shard4-w2-prod' }}
win-unit-mw2:
name: win-unit-mw2
runs-on: windows-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Record runner capacity
run: node -e "const os=require('node:os');
console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Fetch origin/main (worktree tests)
run: git fetch --no-tags origin main:refs/remotes/origin/main
- name: Install dependencies
run: node scripts/npm-retry.mjs ci
- name: Install agent CLIs for provider tests
run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code opencode-ai
- name: Build server dependencies
run: npm run build:server-deps
- name: Run Windows unit experiment
run: npm run test:unit --workspace=@getpaseo/server -- --maxWorkers=2 --reporter=default
--reporter=json --outputFile=unit-experiment.json
env:
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
- name: Upload unit results
uses: actions/upload-artifact@v4
if: always()
with:
name: win-unit-mw2
path: packages/server/unit-experiment.json
if-no-files-found: warn
if: ${{ inputs.arm == 'all' || inputs.arm == 'win-unit-mw2' }}
win-unit-mw3:
name: win-unit-mw3
runs-on: windows-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Record runner capacity
run: node -e "const os=require('node:os');
console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Fetch origin/main (worktree tests)
run: git fetch --no-tags origin main:refs/remotes/origin/main
- name: Install dependencies
run: node scripts/npm-retry.mjs ci
- name: Install agent CLIs for provider tests
run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code opencode-ai
- name: Build server dependencies
run: npm run build:server-deps
- name: Run Windows unit experiment
run: npm run test:unit --workspace=@getpaseo/server -- --maxWorkers=3 --reporter=default
--reporter=json --outputFile=unit-experiment.json
env:
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
- name: Upload unit results
uses: actions/upload-artifact@v4
if: always()
with:
name: win-unit-mw3
path: packages/server/unit-experiment.json
if-no-files-found: warn
if: ${{ inputs.arm == 'all' || inputs.arm == 'win-unit-mw3' }}
win-unit-shard1of3:
name: win-unit-shard1of3
runs-on: windows-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Record runner capacity
run: node -e "const os=require('node:os');
console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
- name: Fetch origin/main (worktree tests)
run: git fetch --no-tags origin main:refs/remotes/origin/main
- name: Install dependencies
run: node scripts/npm-retry.mjs ci
- name: Install agent CLIs for provider tests
run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code opencode-ai
- name: Build server dependencies
run: npm run build:server-deps
- name: Run Windows unit experiment
run:
npm run test:unit --workspace=@getpaseo/server -- --shard=1/3 --maxWorkers=1 --reporter=default
--reporter=json --outputFile=unit-experiment.json
env:
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
- name: Upload unit results
uses: actions/upload-artifact@v4
if: always()
with:
name: win-unit-shard1of3
path: packages/server/unit-experiment.json
if-no-files-found: warn
if: ${{ inputs.arm == 'all' || inputs.arm == 'win-unit-shard1of3' }}