test(server): reproduce checkout RPC timeouts with 200 workspaces

Record the headless Linux slow-request symptom from #7760 without changing production behavior. This is a diagnostic branch; the release-specific event-loop regression and original desktop freeze remain unconfirmed.

Co-authored-by: James Hu <187961+axsuul@users.noreply.github.com>

packages/server/src/server/daemon-e2e/checkout-status-workspace-burst.diagnostic.test.ts

@@ -0,0 +1,104 @@
+import { execFileSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { expect, test } from "vitest";
+import { DEFAULT_GIT_PROCESS_POLICY } from "../../utils/git-process-scheduler.js";
+import { DaemonClient } from "../test-utils/daemon-client.js";
+import { createTestPaseoDaemon } from "../test-utils/paseo-daemon.js";
+import {
+  createPersistedProjectRecord,
+  createPersistedWorkspaceRecord,
+} from "../workspace-registry.js";
+
+// Diagnostic for #7760's Linux report. These are small independent checkouts,
+// rather than the reporter's large linked worktrees. No agent turns are needed.
+test("checkout status answers before its client timeout during a 200-workspace observation burst", async () => {
+  // The regular Vitest config raises this cap to avoid fake-timer deadlocks.
+  // This reproduction needs the actual production admission policy.
+  const previousRate = process.env.PASEO_GIT_MAX_PROCESSES_PER_SECOND;
+  process.env.PASEO_GIT_MAX_PROCESSES_PER_SECOND = String(
+    DEFAULT_GIT_PROCESS_POLICY.maxProcessesPerSecond,
+  );
+  const root = mkdtempSync(join(tmpdir(), "paseo-checkout-burst-"));
+  const repo = join(root, "repo");
+  mkdirSync(repo);
+  const git = (cwd: string, args: string[]) => execFileSync("git", args, { cwd, stdio: "pipe" });
+  git(repo, ["init", "-b", "main"]);
+  writeFileSync(join(repo, "a.txt"), "base\n");
+  git(repo, ["add", "."]);
+  git(repo, [
+    "-c",
+    "user.name=Paseo Test",
+    "-c",
+    "user.email=test@example.com",
+    "-c",
+    "commit.gpgsign=false",
+    "commit",
+    "-m",
+    "init",
+  ]);
+  const cwds = Array.from({ length: 200 }, (_, i) => {
+    const cwd = join(root, `repo-${i}`);
+    git(root, ["clone", "--local", "--quiet", repo, cwd]);
+    git(cwd, ["remote", "remove", "origin"]);
+    return cwd;
+  });
+  const registry = join(root, ".paseo", "projects");
+  mkdirSync(registry, { recursive: true });
+  const now = new Date().toISOString();
+  writeFileSync(
+    join(registry, "projects.json"),
+    JSON.stringify(
+      cwds.map((cwd, i) =>
+        createPersistedProjectRecord({
+          projectId: `prj_test_${i}`,
+          rootPath: cwd,
+          kind: "git",
+          displayName: `repo-${i}`,
+          createdAt: now,
+          updatedAt: now,
+        }),
+      ),
+    ),
+  );
+  writeFileSync(
+    join(registry, "workspaces.json"),
+    JSON.stringify(
+      cwds.map((cwd, i) =>
+        createPersistedWorkspaceRecord({
+          workspaceId: `wks_test_${i}`,
+          projectId: `prj_test_${i}`,
+          cwd,
+          kind: "local_checkout",
+          displayName: `repo-${i}`,
+          createdAt: now,
+          updatedAt: now,
+        }),
+      ),
+    ),
+  );
+  const daemon = await createTestPaseoDaemon({
+    paseoHomeRoot: root,
+    cleanup: false,
+    agentClients: {},
+    mcpEnabled: false,
+  });
+  const client = new DaemonClient({
+    url: `ws://127.0.0.1:${daemon.port}/ws`,
+    clientType: "browser",
+  });
+  try {
+    await client.connect();
+    await client.observeEvents(["checkout_status_update"]).ready;
+    const status = await client.getCheckoutStatus(cwds[199]!);
+    expect(status.isGit).toBe(true);
+    expect(status.error).toBeNull();
+  } finally {
+    await client.close();
+    await daemon.close();
+    rmSync(root, { recursive: true, force: true });
+    if (previousRate === undefined) delete process.env.PASEO_GIT_MAX_PROCESSES_PER_SECOND;
+    else process.env.PASEO_GIT_MAX_PROCESSES_PER_SECOND = previousRate;
+  }
+}, 120_000);

packages/server/src/utils/run-git-command.test.ts

@@ -16,6 +16,7 @@ interface FakeSpawnBehavior {
 
 interface FakeSpawnController {
   activeCount: number;
+  nextPid: number;
   peakActiveCount: number;
   processes: FakeChildProcess[];
   queue: FakeSpawnBehavior[];
@@ -24,6 +25,7 @@ interface FakeSpawnController {
 
 const fakeSpawnController = vi.hoisted<FakeSpawnController>(() => ({
   activeCount: 0,
+  nextPid: 1000,
   peakActiveCount: 0,
   processes: [],
   queue: [],
@@ -33,6 +35,7 @@ const fakeSpawnController = vi.hoisted<FakeSpawnController>(() => ({
     }
 
     this.activeCount = 0;
+    this.nextPid = 1000;
     this.peakActiveCount = 0;
     this.processes = [];
     this.queue = [];
@@ -40,8 +43,7 @@ const fakeSpawnController = vi.hoisted<FakeSpawnController>(() => ({
 }));
 
 class FakeChildProcess extends EventEmitter {
-  // This is not an OS process; tree cleanup must use the direct-child fallback.
-  public readonly pid = undefined;
+  public readonly pid: number;
   public readonly stderr = new EventEmitter();
   public readonly stdout = new EventEmitter();
   public killed = false;
@@ -55,6 +57,8 @@ class FakeChildProcess extends EventEmitter {
   public constructor(behavior: FakeSpawnBehavior) {
     super();
     this.behavior = behavior;
+    this.pid = fakeSpawnController.nextPid;
+    fakeSpawnController.nextPid += 1;
 
     fakeSpawnController.activeCount += 1;
     fakeSpawnController.peakActiveCount = Math.max(

packages/server/src/utils/run-git-command.ts

@@ -13,7 +13,6 @@ import {
   submitGitCommandTrace,
 } from "./git-command-trace.js";
 import { spawnProcess } from "./spawn.js";
-import { signalProcessTree } from "./tree-kill.js";
 import {
   GitProcessScheduler,
   type GitProcessPriority,
@@ -423,7 +422,7 @@ function executeGitCommand<Output>(
 
       timer = setTimeout(() => {
         timeoutError = new Error(`Git command timed out after ${timeout}ms: ${command}`);
-        void signalProcessTree(child, "SIGKILL");
+        child.kill("SIGKILL");
         settle(() => reject(timeoutError));
         if (processExit) {
           settleTimeoutTrace(processExit.exitCode, processExit.signal);

packages/server/src/utils/run-git-command.windows-shell.test.ts

@@ -1,17 +1,12 @@
 import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
-import { createServer, type Socket } from "node:net";
 import { afterEach, describe, expect, it } from "vitest";
 
 import { runGitCommand } from "./run-git-command.js";
 
 const tempDirs: string[] = [];
 
-function allowConnectionReset(error: NodeJS.ErrnoException): void {
-  if (error.code !== "ECONNRESET") throw error;
-}
-
 function makeTempRepo(): string {
   const repo = mkdtempSync(path.join(tmpdir(), "paseo-git-shell-"));
   tempDirs.push(repo);
@@ -25,49 +20,6 @@ afterEach(() => {
 });
 
 describe("runGitCommand shell behavior", () => {
-  it.each(["http", "ssh"])(
-    "closes a stalled %s fetch connection on timeout and admits the next command",
-    async (protocol) => {
-      const repo = makeTempRepo();
-      await runGitCommand(["init"], { cwd: repo });
-      const sockets: Socket[] = [];
-      const server = createServer((socket) => {
-        sockets.push(socket);
-        // Force-killing a Windows peer resets the connection instead of sending EOF.
-        socket.on("error", allowConnectionReset);
-        // Consume the request but never answer, keeping the real transport helper busy.
-        socket.resume();
-      });
-      await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
-      const address = server.address();
-      if (!address || typeof address === "string") throw new Error("Missing TCP address");
-      const remote = `${protocol}://127.0.0.1:${address.port}/repo.git`;
-      try {
-        await expect(
-          runGitCommand(["fetch", remote, "main"], {
-            cwd: repo,
-            timeout: 2000,
-            envOverlay: {
-              NO_PROXY: "127.0.0.1",
-              no_proxy: "127.0.0.1",
-              GIT_SSH_COMMAND: "ssh -F none -o BatchMode=yes",
-            },
-          }),
-        ).rejects.toThrow("Git command timed out after 2000ms");
-
-        expect(sockets).toHaveLength(1);
-        await expect.poll(() => sockets[0]?.destroyed, { timeout: 3000 }).toBe(true);
-        await expect(runGitCommand(["--version"], { cwd: repo })).resolves.toMatchObject({
-          exitCode: 0,
-          truncated: false,
-        });
-      } finally {
-        for (const socket of sockets) socket.destroy();
-        await new Promise<void>((resolve) => server.close(() => resolve()));
-      }
-    },
-  );
-
   it("passes git arguments directly instead of through the platform shell", async () => {
     const repo = makeTempRepo();
     const literalName = "%PASEO_GIT_SHELL_SENTINEL%";

packages/server/src/utils/worktree.test.ts

@@ -55,18 +55,10 @@ function createLegacyWorktreeForTest(
   });
 }
 
-function allowConnectionReset(error: NodeJS.ErrnoException): void {
-  if (error.code !== "ECONNRESET") throw error;
-}
-
 // A remote that accepts connections and never answers, like a VPN-only host while off the VPN.
 async function startSilentRemote(): Promise<{ url: string; close: () => Promise<void> }> {
   const connections = new Set<Socket>();
-  const server = createServer((socket) => {
-    connections.add(socket);
-    // Timeout cleanup force-kills the peer, which can reset its connection on Windows.
-    socket.on("error", allowConnectionReset);
-  });
+  const server = createServer((socket) => connections.add(socket));
   await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
   const { port } = server.address() as AddressInfo;
   return {