mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-05 07:54:38 +00:00
Make full CI finish sooner: eight browser shards, split Windows server job, cancel superseded runs (#7275)
Eight Playwright shards with two workers each, format/lint/relay/sdk combined into one routed quality job, Windows server integration in its own job, cancel-in-progress on every ref, one vitest start for the filtered integration files, no fan-in gate, and a deadline for the heap-bound directory search test that covers its 27,000-directory fixture on Windows. First run on the Team plan: 19m48s from creation to last job with zero queue, all 21 jobs green, 224 runner-minutes. The same workflow under the old 20-job cap finished in 58 minutes with up to 40 minutes of queue per job.
This commit is contained in:
Mohamed Boudra
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
|
||||
import { readFileSync, readdirSync } from "node:fs";
|
||||
import { relative as relativePath } from "node:path";
|
||||
import test from "node:test";
|
||||
import { runInNewContext } from "node:vm";
|
||||
|
||||
const repoRoot = new URL("../", import.meta.url);
|
||||
const ciWorkflowPath = new URL(".github/workflows/ci.yml", repoRoot);
|
||||
@@ -12,21 +13,26 @@ const serverTsconfigPath = new URL("packages/server/tsconfig.server.json", repoR
|
||||
const desktopPackagePath = new URL("packages/desktop/package.json", repoRoot);
|
||||
|
||||
const gatedCiJobs = new Map([
|
||||
["format", { name: "format", contract: "format" }],
|
||||
["lint", { name: "lint", contract: "quality" }],
|
||||
["quality", { name: "quality", contracts: ["format", "quality", "relay", "sdk"] }],
|
||||
["typecheck", { name: "typecheck", contract: "quality" }],
|
||||
["server-tests-ubuntu", { name: "server-tests (ubuntu-latest)", contracts: ["server", "hub"] }],
|
||||
["server-tests-windows", { name: "server-tests (windows-latest)", contracts: ["server", "hub"] }],
|
||||
[
|
||||
"server-integration-windows",
|
||||
{ name: "server-integration (windows-latest)", contract: "server" },
|
||||
],
|
||||
["server-tests-macos", { name: "server-tests (macos-14, file observation)", contract: "server" }],
|
||||
["desktop-tests-ubuntu", { name: "desktop-tests (ubuntu-latest)", contract: "desktop" }],
|
||||
["desktop-tests-windows", { name: "desktop-tests (windows-latest)", contract: "desktop" }],
|
||||
["app-tests", { name: "app-tests", contract: "app" }],
|
||||
["sdk-tests", { name: "sdk-tests", contract: "sdk" }],
|
||||
["playwright-1", { name: "playwright (shard 1/4)", contract: "browser" }],
|
||||
["playwright-2", { name: "playwright (shard 2/4)", contract: "browser" }],
|
||||
["playwright-3", { name: "playwright (shard 3/4)", contract: "browser" }],
|
||||
["playwright-4", { name: "playwright (shard 4/4)", contract: "browser" }],
|
||||
["relay-tests", { name: "relay-tests", contract: "relay" }],
|
||||
["playwright-1", { name: "playwright (shard 1/8)", contract: "browser" }],
|
||||
["playwright-2", { name: "playwright (shard 2/8)", contract: "browser" }],
|
||||
["playwright-3", { name: "playwright (shard 3/8)", contract: "browser" }],
|
||||
["playwright-4", { name: "playwright (shard 4/8)", contract: "browser" }],
|
||||
["playwright-5", { name: "playwright (shard 5/8)", contract: "browser" }],
|
||||
["playwright-6", { name: "playwright (shard 6/8)", contract: "browser" }],
|
||||
["playwright-7", { name: "playwright (shard 7/8)", contract: "browser" }],
|
||||
["playwright-8", { name: "playwright (shard 8/8)", contract: "browser" }],
|
||||
["cli-tests-1", { name: "cli-tests (shard 1/3)", contract: "cli" }],
|
||||
["cli-tests-2", { name: "cli-tests (shard 2/3)", contract: "cli" }],
|
||||
["cli-tests-3", { name: "cli-tests (shard 3/3)", contract: "cli" }],
|
||||
@@ -36,7 +42,7 @@ function jobBlocks(source) {
|
||||
const jobs = new Map();
|
||||
let currentJob;
|
||||
|
||||
for (const line of source.split("\n")) {
|
||||
for (const line of (source.split("\njobs:\n")[1] ?? source).split("\n")) {
|
||||
const jobMatch = /^ ([a-z0-9_-]+):\s*$/.exec(line);
|
||||
if (jobMatch) {
|
||||
currentJob = jobMatch[1];
|
||||
@@ -100,6 +106,12 @@ test("gated checks are statically named jobs with real job-level gating", () =>
|
||||
});
|
||||
|
||||
test("change gating allows superseded workflow runs to cancel", () => {
|
||||
const ci = readFileSync(ciWorkflowPath, "utf8");
|
||||
assert.match(
|
||||
ci,
|
||||
/^concurrency:\n group: ci-\$\{\{ github\.workflow \}\}-\$\{\{ github\.ref \}\}\n cancel-in-progress: true$/m,
|
||||
"a newer push to any ref, main included, cancels the superseded run",
|
||||
);
|
||||
for (const workflowPath of [ciWorkflowPath, dockerWorkflowPath, nixWorkflowPath]) {
|
||||
const source = readFileSync(workflowPath, "utf8");
|
||||
assert.doesNotMatch(
|
||||
@@ -331,3 +343,177 @@ test("stable iOS releases upload once and distribute to external testers before
|
||||
);
|
||||
assert.match(review, /bundle exec fastlane ios submit_review/);
|
||||
});
|
||||
|
||||
test("eight Playwright shards share two-worker coverage", () => {
|
||||
const source = readFileSync(ciWorkflowPath, "utf8");
|
||||
const jobs = jobBlocks(source);
|
||||
assert.equal(jobs.size, 21);
|
||||
for (let shard = 1; shard <= 8; shard++) {
|
||||
const job = jobs.get(`playwright-${shard}`).join("\n");
|
||||
assert.match(job, new RegExp(`PLAYWRIGHT_SHARD: "${shard}/8"`));
|
||||
assert.match(job, new RegExp(`PLAYWRIGHT_ARTIFACT: "${shard}"`));
|
||||
assert.match(
|
||||
job,
|
||||
shard === 1 ? /steps: &playwright_test_steps/ : /steps: \*playwright_test_steps/,
|
||||
);
|
||||
}
|
||||
const shared = jobs.get("playwright-1").join("\n");
|
||||
assert.match(shared, /E2E_WORKERS: "2"/);
|
||||
const weights = /PWTEST_SHARD_WEIGHTS: "([\d:]+)"/.exec(shared)[1].split(":").map(Number);
|
||||
assert.equal(weights.length, 8);
|
||||
assert.ok(weights.every((weight) => weight > 0));
|
||||
assert.match(shared, /npm run test:e2e --workspace=@getpaseo\/app -- --shard=/);
|
||||
|
||||
// No fan-in result job: a gate needs its own runner and under contention it
|
||||
// queued 8m08s behind the last shard. Required checks name the shards directly.
|
||||
assert.equal(jobs.has("playwright"), false);
|
||||
});
|
||||
|
||||
test("quality preserves commands and routes each domain independently after failures", () => {
|
||||
const jobs = jobBlocks(readFileSync(ciWorkflowPath, "utf8"));
|
||||
const quality = jobs.get("quality").join("\n");
|
||||
assert.doesNotMatch(quality, /^\s+continue-on-error:/m);
|
||||
const commands = {
|
||||
format: ["npm run format:check"],
|
||||
quality: [
|
||||
"npx --yes lockfile-lint --path package-lock.json --type npm --allowed-hosts npm --validate-https --validate-integrity",
|
||||
"npm audit signatures",
|
||||
"npm run lint",
|
||||
],
|
||||
relay: ["npm run build:relay", "npm run test --workspace=@getpaseo/relay"],
|
||||
sdk: [
|
||||
"npm run build:relay && npm run build:client",
|
||||
"npm run build:plugin",
|
||||
"npm run test --workspace=@getpaseo/plugin",
|
||||
"npm run test --workspace=@getpaseo/protocol",
|
||||
"npm run test --workspace=@getpaseo/client",
|
||||
"npm run test --workspace=@getpaseo/highlight",
|
||||
"npm run typecheck:examples --workspace=@getpaseo/client",
|
||||
],
|
||||
};
|
||||
const checks = [...quality.matchAll(/ if: \$\{\{ (.+) \}\}\n run: (.+)/g)];
|
||||
assert.deepEqual(
|
||||
checks.map(([, , command]) => command),
|
||||
Object.values(commands).flat(),
|
||||
);
|
||||
const domains = Object.keys(commands);
|
||||
function accepts(expression, outputs, { cancelled = false, installed = true } = {}) {
|
||||
// Prior checks have failed: success() is false. Explicit !cancelled() must
|
||||
// override Actions' default success condition so later domains still run.
|
||||
assert.match(expression, /!cancelled\(\)/);
|
||||
return runInNewContext(expression, {
|
||||
needs: { changes: { outputs } },
|
||||
steps: { install: { outcome: installed ? "success" : "failure" } },
|
||||
cancelled: () => cancelled,
|
||||
success: () => false,
|
||||
});
|
||||
}
|
||||
const disabled = { full: "false", ...Object.fromEntries(domains.map((key) => [key, "false"])) };
|
||||
const jobCondition = / if: \$\{\{ (.+) \}\}/.exec(quality)[1];
|
||||
assert.equal(accepts(jobCondition, disabled), false);
|
||||
for (const selected of domains) {
|
||||
const outputs = { ...disabled, [selected]: "true" };
|
||||
assert.equal(accepts(jobCondition, outputs), true);
|
||||
for (const [, expression, command] of checks) {
|
||||
assert.equal(accepts(expression, outputs), commands[selected].includes(command), command);
|
||||
}
|
||||
}
|
||||
for (const [, expression] of checks) {
|
||||
assert.equal(accepts(expression, { ...disabled, full: "true" }), true);
|
||||
assert.equal(accepts(expression, {}), true, "missing routing outputs fail open");
|
||||
assert.equal(accepts(expression, {}, { cancelled: true }), false);
|
||||
assert.equal(accepts(expression, {}, { installed: false }), false);
|
||||
}
|
||||
assert.equal(accepts(jobCondition, {}, { cancelled: true }), false);
|
||||
});
|
||||
|
||||
test("Windows units and integration run independently with complete routed coverage", () => {
|
||||
const source = readFileSync(ciWorkflowPath, "utf8");
|
||||
const jobs = jobBlocks(source);
|
||||
const ubuntu = jobs.get("server-tests-ubuntu").join("\n");
|
||||
const windows = jobs.get("server-tests-windows").join("\n");
|
||||
const integration = jobs.get("server-integration-windows").join("\n");
|
||||
assert.match(ubuntu, /run: npm run test --workspace=@getpaseo\/server/);
|
||||
assert.match(windows, /run: npm run test:unit --workspace=@getpaseo\/server/);
|
||||
assert.doesNotMatch(windows, /run: npm run test(?: |:integration)/);
|
||||
assert.match(integration, /run: npm run test:integration --workspace=@getpaseo\/server/);
|
||||
assert.doesNotMatch(integration, /test:unit|server_plugin_tests|server_hub_contract/);
|
||||
for (const job of [windows, integration]) {
|
||||
assert.match(job, /^ needs: changes$/m, "both phases become eligible independently");
|
||||
assert.match(job, /^ runs-on: windows-latest$/m);
|
||||
assert.doesNotMatch(job, /continue-on-error|maxWorkers|--exclude/);
|
||||
for (const step of [
|
||||
"server_capacity",
|
||||
"server_checkout",
|
||||
"server_node",
|
||||
"server_fetch",
|
||||
"server_install",
|
||||
"server_agent_clis",
|
||||
"server_build",
|
||||
]) {
|
||||
assert.match(ubuntu, new RegExp(`&${step}\\b`));
|
||||
assert.match(job, new RegExp(`- \\*${step}\\b`));
|
||||
}
|
||||
assert.match(job, /env: \*server_test_env/);
|
||||
}
|
||||
assert.match(windows, /- \*server_plugin_tests/);
|
||||
assert.match(windows, /- \*server_hub_contract/);
|
||||
assert.match(ubuntu, /run: npm run test --workspace=@getpaseo\/builtin-plugins/);
|
||||
assert.match(ubuntu, /run: npm run test:hub-cli-contract --workspace=@getpaseo\/server/);
|
||||
|
||||
const scripts = JSON.parse(
|
||||
readFileSync(new URL("packages/server/package.json", repoRoot), "utf8"),
|
||||
).scripts;
|
||||
assert.equal(scripts.test, "npm run test:unit && npm run test:integration");
|
||||
assert.equal(scripts["test:unit"], 'vitest run --fileParallelism --exclude "**/*.e2e.test.ts"');
|
||||
const invocations = scripts["test:integration"].split(" && ");
|
||||
// Each vitest start costs about half a minute on Windows, so the four
|
||||
// name-filtered files share one invocation and one combined filter.
|
||||
assert.equal(invocations.length, 2);
|
||||
assert.ok(invocations.every((command) => command.startsWith("vitest run --maxWorkers=1 ")));
|
||||
assert.doesNotMatch(invocations[0], / -t /);
|
||||
assert.equal(
|
||||
invocations[1],
|
||||
'vitest run --maxWorkers=1 src/server/daemon-e2e/git-operations.e2e.test.ts src/server/session.create-agent-worktree-autoarchive.e2e.test.ts src/server/agent/providers/opencode-bridge.local.e2e.test.ts src/server/daemon-e2e/relay-transport.e2e.test.ts -t "merges into|creates a worktree and auto-archives both|versioned runtime|oversized attachments"',
|
||||
);
|
||||
const serverConfig = readFileSync(new URL("packages/server/vitest.config.ts", repoRoot), "utf8");
|
||||
assert.doesNotMatch(serverConfig, /maxWorkers/);
|
||||
|
||||
function condition(job) {
|
||||
return / if: \$\{\{ (.+) \}\}/.exec(job)[1];
|
||||
}
|
||||
function selected(expression, outputs, cancelled = false) {
|
||||
return runInNewContext(expression, {
|
||||
needs: { changes: { outputs } },
|
||||
cancelled: () => cancelled,
|
||||
});
|
||||
}
|
||||
const unitCondition = condition(windows);
|
||||
const integrationCondition = condition(integration);
|
||||
const unitStep = / - name: Run server unit tests\n if: \$\{\{ (.+) \}\}/.exec(
|
||||
windows,
|
||||
)[1];
|
||||
const hubStep =
|
||||
/ - &server_hub_contract\n name: Run Hub CLI\/daemon contract\n if: \$\{\{ (.+) \}\}/.exec(
|
||||
ubuntu,
|
||||
)[1];
|
||||
const pluginStep =
|
||||
/ - &server_plugin_tests\n name: Run built-in plugin tests\n if: \$\{\{ (.+) \}\}/.exec(
|
||||
ubuntu,
|
||||
)[1];
|
||||
for (const [outputs, units, integrations, hub] of [
|
||||
[{ full: "true", server: "false", hub: "false" }, true, true, false],
|
||||
[{ full: "false", server: "true", hub: "false" }, true, true, false],
|
||||
[{ full: "false", server: "false", hub: "true" }, true, false, true],
|
||||
[{ full: "false", server: "false", hub: "false" }, false, false, false],
|
||||
[{}, true, true, false],
|
||||
]) {
|
||||
assert.equal(selected(unitCondition, outputs), units);
|
||||
assert.equal(selected(integrationCondition, outputs), integrations);
|
||||
assert.equal(selected(unitStep, outputs), integrations);
|
||||
assert.equal(selected(pluginStep, outputs), integrations);
|
||||
assert.equal(selected(hubStep, outputs), hub);
|
||||
assert.equal(selected(unitCondition, outputs, true), false);
|
||||
assert.equal(selected(integrationCondition, outputs, true), false);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user