From 866243f1d16e3aa3292d09dc136ede14fb29ec07 Mon Sep 17 00:00:00 2001
From: Mohamed Boudra <boudra.moha@gmail.com>
Date: Sun, 18 Oct 2026 05:17:33 +0200
Subject: [PATCH] test(ci): measure runner throughput limits

---
 .../workflows/ci-throughput-experiment.yml    | 318 ++++++++++++++++++
 1 file changed, 318 insertions(+)
 create mode 100644 .github/workflows/ci-throughput-experiment.yml

diff --git a/.github/workflows/ci-throughput-experiment.yml b/.github/workflows/ci-throughput-experiment.yml
new file mode 100644
index 0000000000..9651078597
--- /dev/null
+++ b/.github/workflows/ci-throughput-experiment.yml
@@ -0,0 +1,318 @@
+name: CI throughput experiment
+on:
+  push:
+    branches:
+      - investigate-ci-throughput
+    paths:
+      - .github/workflows/ci-throughput-experiment.yml
+  workflow_dispatch: {}
+permissions:
+  contents: read
+concurrency:
+  group: ci-throughput-experiment
+  cancel-in-progress: false
+env:
+  ONNXRUNTIME_NODE_INSTALL: skip
+jobs:
+  pw-shard4-w2:
+    name: pw-shard4-w2
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      PLAYWRIGHT_SHARD: 4/4
+      PLAYWRIGHT_ARTIFACT: pw-shard4-w2
+      E2E_WORKERS: "2"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Record runner capacity
+        run: node -e "const os=require('node:os');
+          console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+      - name: Install dependencies with retry
+        run: node scripts/npm-retry.mjs ci
+      - name: Install Playwright browsers
+        timeout-minutes: 10
+        run: npx playwright install chromium
+      - name: Build server stack
+        run: npm run build:server
+      - name: Build native app dependency
+        run: npm run build --workspace=@getpaseo/expo-two-way-audio
+      - name: Install agent CLIs for provider tests
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0
+          opencode-ai
+      - name: Run Playwright E2E tests
+        run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ env.PLAYWRIGHT_SHARD }}
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PWTEST_SHARD_WEIGHTS: 111:172:137:160
+      - name: Upload test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: pw-shard4-w2
+          path: |
+            packages/app/test-results/
+            packages/app/playwright-report/
+          retention-days: 7
+  pw-shard4-w3:
+    name: pw-shard4-w3
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      PLAYWRIGHT_SHARD: 4/4
+      PLAYWRIGHT_ARTIFACT: pw-shard4-w3
+      E2E_WORKERS: "3"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Record runner capacity
+        run: node -e "const os=require('node:os');
+          console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+      - name: Install dependencies with retry
+        run: node scripts/npm-retry.mjs ci
+      - name: Install Playwright browsers
+        timeout-minutes: 10
+        run: npx playwright install chromium
+      - name: Build server stack
+        run: npm run build:server
+      - name: Build native app dependency
+        run: npm run build --workspace=@getpaseo/expo-two-way-audio
+      - name: Install agent CLIs for provider tests
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0
+          opencode-ai
+      - name: Run Playwright E2E tests
+        run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ env.PLAYWRIGHT_SHARD }}
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PWTEST_SHARD_WEIGHTS: 111:172:137:160
+      - name: Upload test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: pw-shard4-w3
+          path: |
+            packages/app/test-results/
+            packages/app/playwright-report/
+          retention-days: 7
+  pw-shard4-w4:
+    name: pw-shard4-w4
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      PLAYWRIGHT_SHARD: 4/4
+      PLAYWRIGHT_ARTIFACT: pw-shard4-w4
+      E2E_WORKERS: "4"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Record runner capacity
+        run: node -e "const os=require('node:os');
+          console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+      - name: Install dependencies with retry
+        run: node scripts/npm-retry.mjs ci
+      - name: Install Playwright browsers
+        timeout-minutes: 10
+        run: npx playwright install chromium
+      - name: Build server stack
+        run: npm run build:server
+      - name: Build native app dependency
+        run: npm run build --workspace=@getpaseo/expo-two-way-audio
+      - name: Install agent CLIs for provider tests
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0
+          opencode-ai
+      - name: Run Playwright E2E tests
+        run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ env.PLAYWRIGHT_SHARD }}
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PWTEST_SHARD_WEIGHTS: 111:172:137:160
+      - name: Upload test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: pw-shard4-w4
+          path: |
+            packages/app/test-results/
+            packages/app/playwright-report/
+          retention-days: 7
+  pw-shard4-w2-prod:
+    name: pw-shard4-w2-prod
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      PLAYWRIGHT_SHARD: 4/4
+      PLAYWRIGHT_ARTIFACT: prod
+      E2E_WORKERS: "2"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Record runner capacity
+        run: node -e "const os=require('node:os');
+          console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+      - name: Install dependencies with retry
+        run: node scripts/npm-retry.mjs ci
+      - name: Install Playwright browsers
+        timeout-minutes: 10
+        run: npx playwright install chromium
+      - name: Build server stack
+        run: npm run build:server
+      - name: Build native app dependency
+        run: npm run build --workspace=@getpaseo/expo-two-way-audio
+      - name: Install agent CLIs for provider tests
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code @openai/codex@0.105.0
+          opencode-ai
+      - name: Serve the production-mode bundle for this experiment only
+        run: >-
+          node <<'NODE'
+
+          const fs = require('node:fs');
+
+          const file = 'packages/app/e2e/support/global-setup.ts';
+
+          const source = fs.readFileSync(file, 'utf8');
+
+          const before = '[expoCli, "start", "--web", "--port", String(port)]';
+
+          if (source.split(before).length !== 2) throw new Error('Expected one Metro launch');
+
+          fs.writeFileSync(file, source.replace(before, '[expoCli, "start", "--web", "--no-dev",
+          "--minify", "--port", String(port)]'));
+
+          NODE
+      - name: Run Playwright E2E tests
+        run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ env.PLAYWRIGHT_SHARD }}
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PWTEST_SHARD_WEIGHTS: 111:172:137:160
+      - name: Upload test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: pw-shard4-w2-prod
+          path: |
+            packages/app/test-results/
+            packages/app/playwright-report/
+          retention-days: 7
+  win-unit-mw2:
+    name: win-unit-mw2
+    runs-on: windows-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Record runner capacity
+        run: node -e "const os=require('node:os');
+          console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+      - name: Fetch origin/main (worktree tests)
+        run: git fetch --no-tags origin main:refs/remotes/origin/main
+      - name: Install dependencies
+        run: node scripts/npm-retry.mjs ci
+      - name: Install agent CLIs for provider tests
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code opencode-ai
+      - name: Build server dependencies
+        run: npm run build:server-deps
+      - name: Run Windows unit experiment
+        run: npm run test:unit --workspace=@getpaseo/server -- --maxWorkers=2 --reporter=default
+          --reporter=json --outputFile=unit-experiment.json
+        env:
+          CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      - name: Upload unit results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: win-unit-mw2
+          path: packages/server/unit-experiment.json
+          if-no-files-found: warn
+  win-unit-mw3:
+    name: win-unit-mw3
+    runs-on: windows-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Record runner capacity
+        run: node -e "const os=require('node:os');
+          console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+      - name: Fetch origin/main (worktree tests)
+        run: git fetch --no-tags origin main:refs/remotes/origin/main
+      - name: Install dependencies
+        run: node scripts/npm-retry.mjs ci
+      - name: Install agent CLIs for provider tests
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code opencode-ai
+      - name: Build server dependencies
+        run: npm run build:server-deps
+      - name: Run Windows unit experiment
+        run: npm run test:unit --workspace=@getpaseo/server -- --maxWorkers=3 --reporter=default
+          --reporter=json --outputFile=unit-experiment.json
+        env:
+          CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      - name: Upload unit results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: win-unit-mw3
+          path: packages/server/unit-experiment.json
+          if-no-files-found: warn
+  win-unit-shard1of3:
+    name: win-unit-shard1of3
+    runs-on: windows-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Record runner capacity
+        run: node -e "const os=require('node:os');
+          console.log(JSON.stringify({cpus:os.cpus().length,model:os.cpus()[0].model,memory:os.totalmem(),platform:os.platform()},null,2))"
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+          cache: npm
+      - name: Fetch origin/main (worktree tests)
+        run: git fetch --no-tags origin main:refs/remotes/origin/main
+      - name: Install dependencies
+        run: node scripts/npm-retry.mjs ci
+      - name: Install agent CLIs for provider tests
+        run: node scripts/npm-retry.mjs install -g @anthropic-ai/claude-code opencode-ai
+      - name: Build server dependencies
+        run: npm run build:server-deps
+      - name: Run Windows unit experiment
+        run:
+          npm run test:unit --workspace=@getpaseo/server -- --shard=1/3 --maxWorkers=1 --reporter=default
+          --reporter=json --outputFile=unit-experiment.json
+        env:
+          CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      - name: Upload unit results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: win-unit-shard1of3
+          path: packages/server/unit-experiment.json
+          if-no-files-found: warn