import assert from "node:assert/strict"; import { readFileSync, readdirSync } from "node:fs"; import { relative as relativePath } from "node:path"; import test from "node:test"; import { runInNewContext } from "node:vm"; const repoRoot = new URL("../", import.meta.url); const ciWorkflowPath = new URL(".github/workflows/ci.yml", repoRoot); const dockerWorkflowPath = new URL(".github/workflows/docker.yml", repoRoot); const nixWorkflowPath = new URL(".github/workflows/nix.yml", repoRoot); const filtersPath = new URL(".github/ci-paths.yml", repoRoot); const serverTsconfigPath = new URL("packages/server/tsconfig.server.json", repoRoot); const desktopPackagePath = new URL("packages/desktop/package.json", repoRoot); const gatedCiJobs = new Map([ ["quality", { name: "quality", contracts: ["format", "quality", "relay", "sdk"] }], ["typecheck", { name: "typecheck", contract: "quality" }], ["server-tests-ubuntu", { name: "server-tests (ubuntu-latest)", contracts: ["server", "hub"] }], ["server-tests-windows", { name: "server-tests (windows-latest)", contracts: ["server", "hub"] }], [ "server-integration-windows", { name: "server-integration (windows-latest)", contract: "server" }, ], ["server-tests-macos", { name: "server-tests (macos-14, file observation)", contract: "server" }], ["desktop-tests-ubuntu", { name: "desktop-tests (ubuntu-latest)", contract: "desktop" }], ["desktop-tests-windows", { name: "desktop-tests (windows-latest)", contract: "desktop" }], ["app-tests", { name: "app-tests", contract: "app" }], ["playwright-1", { name: "playwright (shard 1/8)", contract: "browser" }], ["playwright-2", { name: "playwright (shard 2/8)", contract: "browser" }], ["playwright-3", { name: "playwright (shard 3/8)", contract: "browser" }], ["playwright-4", { name: "playwright (shard 4/8)", contract: "browser" }], ["playwright-5", { name: "playwright (shard 5/8)", contract: "browser" }], ["playwright-6", { name: "playwright (shard 6/8)", contract: "browser" }], ["playwright-7", { name: "playwright (shard 7/8)", contract: "browser" }], ["playwright-8", { name: "playwright (shard 8/8)", contract: "browser" }], ["cli-tests-1", { name: "cli-tests (shard 1/3)", contract: "cli" }], ["cli-tests-2", { name: "cli-tests (shard 2/3)", contract: "cli" }], ["cli-tests-3", { name: "cli-tests (shard 3/3)", contract: "cli" }], ]); function jobBlocks(source) { const jobs = new Map(); let currentJob; for (const line of (source.split("\njobs:\n")[1] ?? source).split("\n")) { const jobMatch = /^ ([a-z0-9_-]+):\s*$/.exec(line); if (jobMatch) { currentJob = jobMatch[1]; jobs.set(currentJob, []); continue; } if (currentJob) jobs.get(currentJob).push(line); } return jobs; } function loadFilters(path) { const filters = {}; let currentFilter; for (const line of readFileSync(path, "utf8").split("\n")) { const filterMatch = /^([a-z_]+):\s*$/.exec(line); if (filterMatch) { currentFilter = filterMatch[1]; filters[currentFilter] = []; continue; } const patternMatch = /^ - "([^"]+)"\s*$/.exec(line); if (currentFilter && patternMatch) filters[currentFilter].push(patternMatch[1]); } return filters; } function filesUnder(relativeDirectory, predicate) { const directory = new URL(`${relativeDirectory}/`, repoRoot); return readdirSync(directory, { recursive: true, withFileTypes: true }) .filter((entry) => entry.isFile()) .map((entry) => [relativeDirectory, relativePath(directory.pathname, entry.parentPath), entry.name] .filter(Boolean) .join("/") .replaceAll("\\", "/"), ) .filter(predicate) .sort(); } test("gated checks are statically named jobs with real job-level gating", () => { const workflowSource = readFileSync(ciWorkflowPath, "utf8"); const jobs = jobBlocks(workflowSource); const trigger = workflowSource.split("jobs:", 1)[0]; assert.match(trigger, /^\s+merge_group:\s*$/m); assert.doesNotMatch(workflowSource, /strategy:\s*\n\s+matrix:/); assert.doesNotMatch(workflowSource, /RUN_TESTS|Skip unaffected|No .* changes detected/); for (const [jobId, expected] of gatedCiJobs) { const job = jobs.get(jobId)?.join("\n"); assert.ok(job, `missing static job ${jobId}`); assert.match(job, new RegExp(`^ name: ${expected.name.replace(/[()]/g, "\\$&")}$`, "m")); assert.match(job, /needs\.changes\.outputs\.full != 'false'/); for (const contract of expected.contracts ?? [expected.contract]) { assert.match(job, new RegExp(`needs\\.changes\\.outputs\\.${contract} != 'false'`)); } } }); test("change gating allows superseded workflow runs to cancel", () => { const ci = readFileSync(ciWorkflowPath, "utf8"); assert.match( ci, /^concurrency:\n group: ci-\$\{\{ github\.workflow \}\}-\$\{\{ github\.ref \}\}\n cancel-in-progress: true$/m, "a newer push to any ref, main included, cancels the superseded run", ); for (const workflowPath of [ciWorkflowPath, dockerWorkflowPath, nixWorkflowPath]) { const source = readFileSync(workflowPath, "utf8"); assert.doesNotMatch( source, /\$\{\{\s*always\(\)/, "always() keeps jobs alive after concurrency cancellation; use !cancelled() for fail-open gating", ); } }); test("focused contracts stay inside existing required checks", () => { const jobs = jobBlocks(readFileSync(ciWorkflowPath, "utf8")); const changes = jobs.get("changes")?.join("\n") ?? ""; const server = jobs.get("server-tests-ubuntu")?.join("\n") ?? ""; const desktop = jobs.get("desktop-tests-ubuntu")?.join("\n") ?? ""; assert.match(changes, /scripts\/daemon-launch-contract\.test\.mjs/); assert.doesNotMatch(changes, /Install dependencies|npm run build/); assert.match(server, /test:hub-cli-contract/); assert.match(server, /npm run test --workspace=@getpaseo\/server/); assert.ok(!jobs.has("hub-cli-contract")); assert.match(desktop, /test:e2e:renderer/); assert.match(desktop, /test:e2e:browser-tabs/); assert.match(desktop, /npm run test --workspace=@getpaseo\/desktop/); assert.ok(!jobs.has("desktop-browser-bridge")); assert.ok(!jobs.has("playwright-desktop")); }); test("server builds exclude test utilities at every domain depth", () => { const tsconfig = JSON.parse(readFileSync(serverTsconfigPath, "utf8")); assert.ok(tsconfig.exclude.includes("src/server/**/test-utils/**")); assert.ok(!tsconfig.exclude.includes("src/server/test-utils/**")); }); test("PR routing declares stable behavior ownership", () => { const filters = loadFilters(filtersPath); assert.deepEqual(filters, { routing: [".github/ci-paths.yml"], workspace: [ ".mise.toml", ".tool-versions", "package.json", "package-lock.json", "patches/**", "scripts/**", "tsconfig.json", "tsconfig.base.json", "vitest.config.ts", ], ci: [".github/actions/**", ".github/workflows/ci.yml"], format: [ ".agents/**/*.{cjs,css,html,js,json,jsonc,jsx,md,mjs,ts,tsx,yaml,yml}", ".github/**/*.{cjs,css,html,js,json,jsonc,jsx,md,mjs,ts,tsx,yaml,yml}", "**/*.{cjs,css,html,js,json,jsonc,jsx,md,mjs,ts,tsx,yaml,yml}", "packages/expo-two-way-audio/**", ], quality: ["**/*.{cjs,js,json,jsx,mjs,ts,tsx}", "packages/expo-two-way-audio/**"], hub: ["packages/cli/src/commands/hub/**", "packages/server/src/server/hub/**"], server: ["plugins/**", "packages/server/**", "packages/app/e2e/support/fixtures/recording.*"], desktop: [ "packages/desktop/**", "packages/app/src/desktop/**", "packages/server/src/server/browser-tools/**", "packages/app/e2e/support/**", "packages/app/*config.{cjs,js,ts}", "packages/app/package.json", ], app: ["packages/app/**", "packages/expo-two-way-audio/**"], sdk: [ "packages/plugin/**", "plugin-examples/**", "public-docs/plugins/**", "packages/client/**", "packages/highlight/**", "packages/protocol/**", ], browser: [ "packages/server/src/server/agent/provider-snapshot-manager.ts", "packages/server/src/server/session/provider/provider-catalog-session.ts", "packages/client/src/compat/normalize-provider-models.ts", "packages/protocol/src/client-capabilities.ts", "packages/server/src/server/agent/provider-registry.ts", "packages/server/src/server/agent/agent-sdk-types.ts", "packages/server/src/server/agent/providers/codex-app-server-agent.ts", "packages/server/src/server/agent/providers/claude/agent.ts", "packages/server/src/server/agent/plugin-provider.ts", "packages/server/src/server/plugins/{index,plugin-process,plugin-process-protocol,runtime}.ts", "packages/server/src/executable-resolution/**", "packages/plugin/src/server/provider.ts", "packages/app/src/!(desktop)/**", "packages/app/e2e/browser/**", "packages/app/e2e/support/**", "packages/app/assets/**", "packages/app/public/**", "packages/app/index.ts", "packages/app/*config.{cjs,js,ts}", "packages/app/package.json", ], relay: ["packages/relay/**"], cli: ["packages/cli/**"], }); }); test("cross-package invariants live in the suite that owns them", () => { const cliTests = filesUnder("packages/cli", (path) => path.endsWith(".test.ts")); assert.ok(cliTests.length > 0); for (const path of cliTests) { assert.doesNotMatch( readFileSync(new URL(path, repoRoot), "utf8"), /server\/src\/server\/test-utils/, path, ); } const protocolWireCompatibility = new URL( "packages/protocol/src/messages.wire-compat.test.ts", repoRoot, ); assert.match(readFileSync(protocolWireCompatibility, "utf8"), /wire schema compatibility/); }); test("browser and desktop tests have exclusive, directory-owned suites", () => { const filters = loadFilters(filtersPath); const browserSpecs = filesUnder("packages/app/e2e", (path) => path.endsWith(".spec.ts")); const desktopSpecs = filesUnder("packages/desktop/e2e", (path) => path.endsWith(".spec.ts")); const electronModules = filesUnder("packages/app/src", (path) => /\.electron\.tsx?$/.test(path)); assert.ok(browserSpecs.length > 0); assert.ok(desktopSpecs.length > 0); assert.ok(browserSpecs.every((path) => path.startsWith("packages/app/e2e/browser/"))); assert.ok(desktopSpecs.every((path) => path.startsWith("packages/desktop/e2e/"))); assert.ok(electronModules.every((path) => path.startsWith("packages/app/src/desktop/"))); const desktopPackage = JSON.parse(readFileSync(desktopPackagePath, "utf8")); assert.match(desktopPackage.scripts.test, /--exclude ["']e2e\/\*\*["']/); for (const path of browserSpecs) { assert.doesNotMatch( readFileSync(new URL(path, repoRoot), "utf8"), /paseoDesktop|injectDesktopBridge/, ); } for (const path of desktopSpecs) { assert.ok(path.startsWith("packages/desktop/e2e/")); } const routingSource = readFileSync(filtersPath, "utf8"); assert.doesNotMatch(routingSource, /desktop_bridge|playwright_desktop|browser-\*|browser-\*\//); assert.deepEqual(filters.desktop, [ "packages/desktop/**", "packages/app/src/desktop/**", "packages/server/src/server/browser-tools/**", "packages/app/e2e/support/**", "packages/app/*config.{cjs,js,ts}", "packages/app/package.json", ]); assert.deepEqual(filters.browser, [ "packages/server/src/server/agent/provider-snapshot-manager.ts", "packages/server/src/server/session/provider/provider-catalog-session.ts", "packages/client/src/compat/normalize-provider-models.ts", "packages/protocol/src/client-capabilities.ts", "packages/server/src/server/agent/provider-registry.ts", "packages/server/src/server/agent/agent-sdk-types.ts", "packages/server/src/server/agent/providers/codex-app-server-agent.ts", "packages/server/src/server/agent/providers/claude/agent.ts", "packages/server/src/server/agent/plugin-provider.ts", "packages/server/src/server/plugins/{index,plugin-process,plugin-process-protocol,runtime}.ts", "packages/server/src/executable-resolution/**", "packages/plugin/src/server/provider.ts", "packages/app/src/!(desktop)/**", "packages/app/e2e/browser/**", "packages/app/e2e/support/**", "packages/app/assets/**", "packages/app/public/**", "packages/app/index.ts", "packages/app/*config.{cjs,js,ts}", "packages/app/package.json", ]); }); test("packaging runs on main without allocating pull-request runners", () => { for (const workflowPath of [dockerWorkflowPath, nixWorkflowPath]) { const source = readFileSync(workflowPath, "utf8"); const trigger = source.split("jobs:", 1)[0]; assert.match(trigger, /push:\s*\n\s+branches: \[main\]/); assert.doesNotMatch(trigger, /pull_request/); assert.doesNotMatch(source, /dorny\/paths-filter/); } }); test("desktop packaging smokes main pushes and only the pull requests that touch packaging", () => { const source = readFileSync(new URL(".github/workflows/desktop-packages.yml", repoRoot), "utf8"); const trigger = source.split("jobs:", 1)[0]; assert.match(trigger, /push:\s*\n\s+branches: \[main\]/); assert.match(trigger, /pull_request:\s*\n\s+branches: \[main\]\s*\n\s+paths:/); assert.match(trigger, /- "packages\/desktop\/\*\*"/); assert.doesNotMatch(source, /dorny\/paths-filter/); for (const action of ["actions/checkout", "actions/setup-node", "actions/upload-artifact"]) { assert.match(source, new RegExp(`${action}@[0-9a-f]{40} # v\\d+\\.\\d+\\.\\d+`)); } }); test("stable iOS releases upload once and distribute to external testers before App Store review", () => { const source = readFileSync( new URL("packages/app/.eas/workflows/release-mobile.yml", repoRoot), "utf8", ); const jobs = jobBlocks(source); const uploads = [...jobs.entries()].filter( ([, lines]) => lines.some((line) => /^ type: (submit|testflight)$/.test(line)) && lines.some((line) => line.includes("needs.build_ios.outputs.build_id")), ); assert.equal(uploads.length, 1, "the iOS binary must be uploaded exactly once"); const [distributionId, distributionLines] = uploads[0]; const distribution = distributionLines.join("\n"); assert.match(distribution, /^ type: testflight$/m); assert.match(distribution, /external_groups:\s*\n\s+- Paseo Beta/); assert.match(distribution, /^ submit_beta_review: true$/m); const review = jobs.get("submit_ios_for_review").join("\n"); assert.match(review, new RegExp(`needs: \\[build_ios, ${distributionId}\\]`)); assert.match(review, /APP_VERSION="\$\{\{ needs\.build_ios\.outputs\.app_version \}\}"/); assert.match( review, /APP_BUILD_VERSION="\$\{\{ needs\.build_ios\.outputs\.app_build_version \}\}"/, ); assert.match(review, /bundle exec fastlane ios submit_review/); }); test("eight Playwright shards share two-worker coverage", () => { const source = readFileSync(ciWorkflowPath, "utf8"); const jobs = jobBlocks(source); assert.equal(jobs.size, 21); for (let shard = 1; shard <= 8; shard++) { const job = jobs.get(`playwright-${shard}`).join("\n"); assert.match(job, new RegExp(`PLAYWRIGHT_SHARD: "${shard}/8"`)); assert.match(job, new RegExp(`PLAYWRIGHT_ARTIFACT: "${shard}"`)); assert.match( job, shard === 1 ? /steps: &playwright_test_steps/ : /steps: \*playwright_test_steps/, ); } const shared = jobs.get("playwright-1").join("\n"); assert.match(shared, /E2E_WORKERS: "2"/); const weights = /PWTEST_SHARD_WEIGHTS: "([\d:]+)"/.exec(shared)[1].split(":").map(Number); assert.equal(weights.length, 8); assert.ok(weights.every((weight) => weight > 0)); assert.match(shared, /npm run test:e2e --workspace=@getpaseo\/app -- --shard=/); // No fan-in result job: a gate needs its own runner and under contention it // queued 8m08s behind the last shard. Required checks name the shards directly. assert.equal(jobs.has("playwright"), false); }); test("quality preserves commands and routes each domain independently after failures", () => { const jobs = jobBlocks(readFileSync(ciWorkflowPath, "utf8")); const quality = jobs.get("quality").join("\n"); assert.doesNotMatch(quality, /^\s+continue-on-error:/m); const commands = { format: ["npm run format:check"], quality: [ "npx --yes lockfile-lint --path package-lock.json --type npm --allowed-hosts npm --validate-https --validate-integrity", "npm audit signatures", "npm run lint", "node --test scripts/oxlint/paseo-plugin.test.mjs", ], relay: ["npm run build:relay", "npm run test --workspace=@getpaseo/relay"], sdk: [ "npm run build:relay && npm run build:client", "npm run build:plugin", "npm run test --workspace=@getpaseo/plugin", "npm run test --workspace=@getpaseo/protocol", "npm run test --workspace=@getpaseo/client", "npm run test --workspace=@getpaseo/highlight", "npm run typecheck:examples --workspace=@getpaseo/client", ], }; const checks = [...quality.matchAll(/ if: \$\{\{ (.+) \}\}\n run: (.+)/g)]; assert.deepEqual( checks.map(([, , command]) => command), Object.values(commands).flat(), ); const domains = Object.keys(commands); function accepts(expression, outputs, { cancelled = false, installed = true } = {}) { // Prior checks have failed: success() is false. Explicit !cancelled() must // override Actions' default success condition so later domains still run. assert.match(expression, /!cancelled\(\)/); return runInNewContext(expression, { needs: { changes: { outputs } }, steps: { install: { outcome: installed ? "success" : "failure" } }, cancelled: () => cancelled, success: () => false, }); } const disabled = { full: "false", ...Object.fromEntries(domains.map((key) => [key, "false"])) }; const jobCondition = / if: \$\{\{ (.+) \}\}/.exec(quality)[1]; assert.equal(accepts(jobCondition, disabled), false); for (const selected of domains) { const outputs = { ...disabled, [selected]: "true" }; assert.equal(accepts(jobCondition, outputs), true); for (const [, expression, command] of checks) { assert.equal(accepts(expression, outputs), commands[selected].includes(command), command); } } for (const [, expression] of checks) { assert.equal(accepts(expression, { ...disabled, full: "true" }), true); assert.equal(accepts(expression, {}), true, "missing routing outputs fail open"); assert.equal(accepts(expression, {}, { cancelled: true }), false); assert.equal(accepts(expression, {}, { installed: false }), false); } assert.equal(accepts(jobCondition, {}, { cancelled: true }), false); }); test("Windows units and integration run independently with complete routed coverage", () => { const source = readFileSync(ciWorkflowPath, "utf8"); const jobs = jobBlocks(source); const ubuntu = jobs.get("server-tests-ubuntu").join("\n"); const windows = jobs.get("server-tests-windows").join("\n"); const integration = jobs.get("server-integration-windows").join("\n"); assert.match(ubuntu, /run: npm run test --workspace=@getpaseo\/server/); assert.match(windows, /run: npm run test:unit --workspace=@getpaseo\/server/); assert.doesNotMatch(windows, /run: npm run test(?: |:integration)/); assert.match(integration, /run: npm run test:integration --workspace=@getpaseo\/server/); assert.doesNotMatch(integration, /test:unit|server_plugin_tests|server_hub_contract/); for (const job of [windows, integration]) { assert.match(job, /^ needs: changes$/m, "both phases become eligible independently"); assert.match(job, /^ runs-on: windows-latest$/m); assert.doesNotMatch(job, /continue-on-error|maxWorkers|--exclude/); for (const step of [ "server_capacity", "server_checkout", "server_node", "server_fetch", "server_install", "server_agent_clis", "server_build", ]) { assert.match(ubuntu, new RegExp(`&${step}\\b`)); assert.match(job, new RegExp(`- \\*${step}\\b`)); } assert.match(job, /env: \*server_test_env/); } assert.match(windows, /- \*server_plugin_tests/); assert.match(windows, /- \*server_hub_contract/); assert.match(ubuntu, /run: npm run test --workspace=@getpaseo\/builtin-plugins/); assert.match(ubuntu, /run: npm run test:hub-cli-contract --workspace=@getpaseo\/server/); const scripts = JSON.parse( readFileSync(new URL("packages/server/package.json", repoRoot), "utf8"), ).scripts; assert.equal(scripts.test, "npm run test:unit && npm run test:integration"); assert.equal(scripts["test:unit"], 'vitest run --fileParallelism --exclude "**/*.e2e.test.ts"'); const invocations = scripts["test:integration"].split(" && "); // Each vitest start costs about half a minute on Windows, so the four // name-filtered files share one invocation and one combined filter. assert.equal(invocations.length, 2); assert.ok(invocations.every((command) => command.startsWith("vitest run --maxWorkers=1 "))); assert.doesNotMatch(invocations[0], / -t /); assert.equal( invocations[1], 'vitest run --maxWorkers=1 src/server/daemon-e2e/git-operations.e2e.test.ts src/server/session.create-agent-worktree-autoarchive.e2e.test.ts src/server/agent/providers/opencode-bridge.local.e2e.test.ts src/server/daemon-e2e/relay-transport.e2e.test.ts -t "merges into|creates a worktree and auto-archives both|versioned runtime|oversized attachments"', ); const serverConfig = readFileSync(new URL("packages/server/vitest.config.ts", repoRoot), "utf8"); assert.doesNotMatch(serverConfig, /maxWorkers/); function condition(job) { return / if: \$\{\{ (.+) \}\}/.exec(job)[1]; } function selected(expression, outputs, cancelled = false) { return runInNewContext(expression, { needs: { changes: { outputs } }, cancelled: () => cancelled, }); } const unitCondition = condition(windows); const integrationCondition = condition(integration); const unitStep = / - name: Run server unit tests\n if: \$\{\{ (.+) \}\}/.exec( windows, )[1]; const hubStep = / - &server_hub_contract\n name: Run Hub CLI\/daemon contract\n if: \$\{\{ (.+) \}\}/.exec( ubuntu, )[1]; const pluginStep = / - &server_plugin_tests\n name: Run built-in plugin tests\n if: \$\{\{ (.+) \}\}/.exec( ubuntu, )[1]; for (const [outputs, units, integrations, hub] of [ [{ full: "true", server: "false", hub: "false" }, true, true, false], [{ full: "false", server: "true", hub: "false" }, true, true, false], [{ full: "false", server: "false", hub: "true" }, true, false, true], [{ full: "false", server: "false", hub: "false" }, false, false, false], [{}, true, true, false], ]) { assert.equal(selected(unitCondition, outputs), units); assert.equal(selected(integrationCondition, outputs), integrations); assert.equal(selected(unitStep, outputs), integrations); assert.equal(selected(pluginStep, outputs), integrations); assert.equal(selected(hubStep, outputs), hub); assert.equal(selected(unitCondition, outputs, true), false); assert.equal(selected(integrationCondition, outputs, true), false); } });