From cb486c3a5a8f3b709d8f32032afb05b5b4871230 Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Tue, 30 Jun 2026 12:37:47 +0200 Subject: [PATCH] Speed up app Playwright CI (#1830) * ci(playwright): shard app e2e in CI Run the app Playwright suite across isolated CI shards and keep restarted E2E daemons on the same speech-disabled setup path as global setup. * test(app): share disabled speech e2e env --- .github/workflows/ci.yml | 9 ++++-- docs/testing.md | 1 + packages/app/e2e/global-setup.ts | 26 ++------------- packages/app/e2e/helpers/daemon-restart.ts | 24 +++++++------- packages/app/e2e/helpers/speech-env.ts | 32 +++++++++++++++++++ .../app/e2e/workspace-model-restart.spec.ts | 7 ++-- 6 files changed, 59 insertions(+), 40 deletions(-) create mode 100644 packages/app/e2e/helpers/speech-env.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae37945f7..3967aed71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -241,6 +241,11 @@ jobs: run: npm run typecheck:examples --workspace=@getpaseo/client playwright: + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + name: playwright (shard ${{ matrix.shard }}/4) runs-on: ubuntu-latest env: ELECTRON_SKIP_BINARY_DOWNLOAD: "1" @@ -280,7 +285,7 @@ jobs: run: npm install -g @anthropic-ai/claude-code @openai/codex@0.105.0 opencode-ai - name: Run Playwright E2E tests - run: npm run test:e2e --workspace=@getpaseo/app + run: npm run test:e2e --workspace=@getpaseo/app -- --shard=${{ matrix.shard }}/4 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -288,7 +293,7 @@ jobs: uses: actions/upload-artifact@v4 if: failure() with: - name: playwright-results + name: playwright-results-${{ matrix.shard }} path: | packages/app/test-results/ packages/app/playwright-report/ diff --git a/docs/testing.md b/docs/testing.md index bbd418003..6c7e24109 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -131,6 +131,7 @@ Test suites in this repo are heavy. Running them in bulk freezes the machine, es - For full-suite confidence, push to CI and check GitHub Actions. - Never run the full Playwright E2E suite locally — defer whole-suite verification to CI. Targeted Playwright specs are allowed when you changed or need to prove that specific flow. - App Playwright specs share one isolated daemon per run. Helpers that create projects or workspaces must remove the daemon project record during cleanup, not only delete the temp directory. Agent helpers must pass the intended `workspaceId` through to agent creation; never infer ownership from `cwd`. +- CI can shard app Playwright across multiple jobs; each shard still owns a full isolated daemon/relay/Metro stack from global setup. Helpers that restart the daemon must preserve the global setup environment, including disabled speech/local-model settings, so a restart does not change the tested surface or start background downloads. ## Agent authentication in tests diff --git a/packages/app/e2e/global-setup.ts b/packages/app/e2e/global-setup.ts index 4b57a35c9..c64b58072 100644 --- a/packages/app/e2e/global-setup.ts +++ b/packages/app/e2e/global-setup.ts @@ -10,6 +10,7 @@ import dotenv from "dotenv"; import { loadDaemonClientConstructor } from "./helpers/daemon-client-loader"; import { createNodeWebSocketFactory, type NodeWebSocketFactory } from "./helpers/node-ws-factory"; import { forkPaseoHomeMetadata, resolvePaseoHomePath } from "./helpers/paseo-home-fork"; +import { withDisabledE2ESpeechEnv } from "./helpers/speech-env"; const wranglerCliPath = path.resolve(__dirname, "../node_modules/wrangler/bin/wrangler.js"); @@ -405,15 +406,6 @@ async function waitForPairingOfferFromDaemon(args: { ); } -const LOCAL_SPEECH_ENV_KEYS = [ - "PASEO_LOCAL_MODELS_DIR", - "PASEO_DICTATION_LOCAL_STT_MODEL", - "PASEO_VOICE_LOCAL_STT_MODEL", - "PASEO_VOICE_LOCAL_TTS_MODEL", - "PASEO_VOICE_LOCAL_TTS_SPEAKER_ID", - "PASEO_VOICE_LOCAL_TTS_SPEED", -] as const; - async function loadEnvTestFile(repoRoot: string): Promise { const envTestPath = path.join(repoRoot, ".env.test"); if (existsSync(envTestPath)) { @@ -675,7 +667,7 @@ interface DaemonSpawnArgs { function startDaemon(args: DaemonSpawnArgs): ChildProcess { const serverDir = path.resolve(__dirname, "../../..", "packages/server"); const tsxBin = execSync("which tsx").toString().trim(); - const env: NodeJS.ProcessEnv = { + const env = withDisabledE2ESpeechEnv({ ...process.env, PATH: `${args.fakeEditorBinDir}${path.delimiter}${process.env.PATH ?? ""}`, PASEO_HOME: args.paseoHome, @@ -684,21 +676,9 @@ function startDaemon(args: DaemonSpawnArgs): ChildProcess { PASEO_LISTEN: `0.0.0.0:${args.port}`, PASEO_RELAY_ENDPOINT: `127.0.0.1:${args.relayPort}`, PASEO_CORS_ORIGINS: `http://localhost:${args.metroPort}`, - // Default app E2E does not cover speech flows. Keep these disabled so - // unrelated tests never start background local-model downloads. - PASEO_DICTATION_ENABLED: "0", - PASEO_VOICE_MODE_ENABLED: "0", - PASEO_DICTATION_STT_PROVIDER: "openai", - PASEO_VOICE_TURN_DETECTION_PROVIDER: "openai", - PASEO_VOICE_STT_PROVIDER: "openai", - PASEO_VOICE_TTS_PROVIDER: "openai", PASEO_NODE_ENV: "development", NODE_ENV: "development", - }; - - for (const key of LOCAL_SPEECH_ENV_KEYS) { - delete env[key]; - } + }); const child = spawn(tsxBin, ["scripts/supervisor-entrypoint.ts", "--dev"], { cwd: serverDir, diff --git a/packages/app/e2e/helpers/daemon-restart.ts b/packages/app/e2e/helpers/daemon-restart.ts index 3f5920254..5490d449f 100644 --- a/packages/app/e2e/helpers/daemon-restart.ts +++ b/packages/app/e2e/helpers/daemon-restart.ts @@ -5,6 +5,7 @@ import net from "node:net"; import path from "node:path"; import { getE2EDaemonPort } from "./daemon-port"; +import { withDisabledE2ESpeechEnv } from "./speech-env"; /** * Restarts the isolated E2E daemon against the SAME PASEO_HOME and SAME port so @@ -93,20 +94,21 @@ function spawnSupervisor(args: { // inside the Playwright worker (the shim is a .mjs symlink, not an executable), // so resolve the CLI module and load it with node. const tsxCli = createRequire(path.join(serverDir, "package.json")).resolve("tsx/cli"); + const env = withDisabledE2ESpeechEnv({ + ...process.env, + PASEO_HOME: args.paseoHome, + PASEO_E2E_EDITOR_RECORD_PATH: args.editorRecordPath, + PASEO_SERVER_ID: "srv_e2e_test_daemon", + PASEO_LISTEN: `0.0.0.0:${args.port}`, + PASEO_RELAY_ENDPOINT: `127.0.0.1:${args.relayPort}`, + PASEO_CORS_ORIGINS: `http://localhost:${args.metroPort}`, + PASEO_NODE_ENV: "development", + NODE_ENV: "development", + }); const child = spawn(process.execPath, [tsxCli, "scripts/supervisor-entrypoint.ts", "--dev"], { cwd: serverDir, - env: { - ...process.env, - PASEO_HOME: args.paseoHome, - PASEO_E2E_EDITOR_RECORD_PATH: args.editorRecordPath, - PASEO_SERVER_ID: "srv_e2e_test_daemon", - PASEO_LISTEN: `0.0.0.0:${args.port}`, - PASEO_RELAY_ENDPOINT: `127.0.0.1:${args.relayPort}`, - PASEO_CORS_ORIGINS: `http://localhost:${args.metroPort}`, - PASEO_NODE_ENV: "development", - NODE_ENV: "development", - }, + env, stdio: ["ignore", "pipe", "pipe"], detached: false, }); diff --git a/packages/app/e2e/helpers/speech-env.ts b/packages/app/e2e/helpers/speech-env.ts new file mode 100644 index 000000000..1dbafe2dc --- /dev/null +++ b/packages/app/e2e/helpers/speech-env.ts @@ -0,0 +1,32 @@ +const LOCAL_SPEECH_ENV_KEYS = [ + "PASEO_LOCAL_MODELS_DIR", + "PASEO_DICTATION_LOCAL_STT_MODEL", + "PASEO_VOICE_LOCAL_STT_MODEL", + "PASEO_VOICE_LOCAL_TTS_MODEL", + "PASEO_VOICE_LOCAL_TTS_SPEAKER_ID", + "PASEO_VOICE_LOCAL_TTS_SPEED", +] as const; + +const DISABLED_E2E_SPEECH_ENV = { + PASEO_DICTATION_ENABLED: "0", + PASEO_VOICE_MODE_ENABLED: "0", + PASEO_DICTATION_STT_PROVIDER: "openai", + PASEO_VOICE_TURN_DETECTION_PROVIDER: "openai", + PASEO_VOICE_STT_PROVIDER: "openai", + PASEO_VOICE_TTS_PROVIDER: "openai", +} as const; + +export function withDisabledE2ESpeechEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + // Default app E2E does not cover speech flows; keep restarts from starting + // background local-model downloads for unrelated tests. + const next: NodeJS.ProcessEnv = { + ...env, + ...DISABLED_E2E_SPEECH_ENV, + }; + + for (const key of LOCAL_SPEECH_ENV_KEYS) { + delete next[key]; + } + + return next; +} diff --git a/packages/app/e2e/workspace-model-restart.spec.ts b/packages/app/e2e/workspace-model-restart.spec.ts index 82da902e7..295234094 100644 --- a/packages/app/e2e/workspace-model-restart.spec.ts +++ b/packages/app/e2e/workspace-model-restart.spec.ts @@ -10,6 +10,7 @@ import { buildHostWorkspaceRoute, decodeWorkspaceIdFromPathSegment } from "@/uti import { buildSeededHost } from "./helpers/daemon-registry"; import { loadDaemonClientConstructor } from "./helpers/daemon-client-loader"; import { createNodeWebSocketFactory, type NodeWebSocketFactory } from "./helpers/node-ws-factory"; +import { withDisabledE2ESpeechEnv } from "./helpers/speech-env"; import { expectNewWorkspaceProjectSelected, openGlobalNewWorkspaceComposer, @@ -243,18 +244,16 @@ async function startRestartDaemon(input: { const tsxBin = execSync("which tsx").toString().trim(); const child = spawn(tsxBin, ["scripts/supervisor-entrypoint.ts", "--dev"], { cwd: serverDir, - env: { + env: withDisabledE2ESpeechEnv({ ...process.env, PASEO_HOME: input.paseoHome, PASEO_SERVER_ID: SERVER_ID, PASEO_LISTEN: `127.0.0.1:${port}`, PASEO_CORS_ORIGINS: input.origin, PASEO_RELAY_ENABLED: "0", - PASEO_DICTATION_ENABLED: "0", - PASEO_VOICE_MODE_ENABLED: "0", PASEO_NODE_ENV: "development", NODE_ENV: "development", - }, + }), stdio: ["ignore", "ignore", "pipe"], detached: false, });