fix(server): keep cold diffs out from behind background git fan-out

Add a background lane to the git process scheduler and run reconciliation
checkout reads, observation-driven workspace refreshes and background fetches
in it. Bound reconciliation checkout reads to four at a time. Stop the Linux
file observer's startup scan from reporting existing files as creates, which
made every cold diff compute and discard its first revision.

packages/server/src/server/diff/subscriptions.test.ts

@@ -6,6 +6,115 @@ import { expect, test } from "vitest";
 import { DiffOpcode, type DiffFrame } from "@getpaseo/protocol/binary-frames/index";
 import { DiffSubscriptions } from "./index.js";
 import { createFileObserver } from "../file-observer/index.js";
+import {
+  configureGitProcessPolicy,
+  runGitCommand,
+  runWithGitCommandPriority,
+} from "../../utils/run-git-command.js";
+import {
+  DEFAULT_GIT_PROCESS_POLICY,
+  resolveGitProcessPolicy,
+} from "../../utils/git-process-scheduler.js";
+
+async function createObservedWorktreeDiff(prefix: string) {
+  const cwd = await mkdtemp(join(tmpdir(), prefix));
+  const git = (...args: string[]) => execFileSync("git", args, { cwd, encoding: "utf8" }).trim();
+  const observer = createFileObserver();
+  const subscriptions = new DiffSubscriptions({
+    async requestWorkingTreeWatch(root, onChange) {
+      const watch = await observer.subscribe(root, () => onChange(), {
+        ignore: [join(root, ".git")],
+      });
+      return {
+        repoRoot: root,
+        unsubscribe: async () => {
+          await watch.unsubscribe();
+        },
+      };
+    },
+  });
+  git("init", "-b", "main");
+  git("config", "user.email", "test@example.com");
+  git("config", "user.name", "Test");
+  for (const directory of ["src", "src/nested", "docs"]) {
+    await mkdir(join(cwd, directory), { recursive: true });
+    for (let index = 0; index < 40; index++)
+      await writeFile(join(cwd, directory, `${index}.ts`), `export const value = ${index};\n`);
+  }
+  git("add", ".");
+  git("-c", "commit.gpgsign=false", "commit", "-m", "base");
+  for (let index = 0; index < 10; index++)
+    await writeFile(join(cwd, "src", `${index}.ts`), `export const value = ${index + 100};\n`);
+  return {
+    cwd,
+    subscriptions,
+    compare: {
+      cwd,
+      base: { kind: "rev" as const, rev: "HEAD" },
+      head: { kind: "worktree" as const },
+      highlight: true,
+    },
+    async close() {
+      await subscriptions.close();
+      await observer.close();
+      await rm(cwd, { recursive: true, force: true });
+    },
+  };
+}
+
+test("a cold worktree diff over existing files is complete at its first revision", async () => {
+  const fixture = await createObservedWorktreeDiff("diff-cold-revision-");
+  const frames: DiffFrame[] = [];
+  try {
+    const subscription = await fixture.subscriptions.subscribe(fixture.compare, async (frame) => {
+      frames.push(frame);
+    });
+    await subscription.start();
+    await writeFile(join(fixture.cwd, "docs", "0.ts"), "export const value = 999;\n");
+    await expect
+      .poll(() => frames.filter((frame) => frame.opcode === DiffOpcode.End).length, {
+        timeout: 10_000,
+      })
+      .toBe(2);
+    expect(
+      frames.filter((frame) => frame.opcode === DiffOpcode.Begin).map((frame) => frame.revision),
+    ).toEqual([1, 2]);
+    await subscription.release();
+  } finally {
+    await fixture.close();
+  }
+}, 30_000);
+
+test("a cold diff finishes ahead of a queued background Git burst", async () => {
+  const fixture = await createObservedWorktreeDiff("diff-cold-burst-");
+  // The suite lifts the start-rate cap; the burst needs the daemon's real one.
+  configureGitProcessPolicy(DEFAULT_GIT_PROCESS_POLICY);
+  try {
+    let burstSettledAt = Number.POSITIVE_INFINITY;
+    // ~3 s of queued work at 64 starts/s, the shape of a reconciliation rescan.
+    const burst = runWithGitCommandPriority("background", () =>
+      Promise.all(
+        Array.from({ length: 200 }, () =>
+          runGitCommand(["rev-parse", "HEAD"], { cwd: fixture.cwd }),
+        ),
+      ),
+    ).then(() => {
+      burstSettledAt = performance.now();
+      return undefined;
+    });
+    let readyAt = Number.POSITIVE_INFINITY;
+    const subscription = await fixture.subscriptions.subscribe(fixture.compare, async (frame) => {
+      if (frame.opcode === DiffOpcode.End && frame.error === null) readyAt = performance.now();
+    });
+    await subscription.start();
+    await burst;
+    expect(readyAt).toBeLessThan(burstSettledAt);
+    await subscription.release();
+  } finally {
+    configureGitProcessPolicy(resolveGitProcessPolicy({ env: process.env }));
+    await fixture.close();
+  }
+}, 30_000);
 
 test("shares computation, highlights only the changed file, recovers refs and releases observations", async () => {
   const cwd = await mkdtemp(join(tmpdir(), "diff-subscriptions-"));

packages/server/src/server/file-observer/index.test.ts

@@ -47,6 +47,29 @@ test("observes nested files while pruning excluded directories", async () => {
   await subscription.unsubscribe();
 });
 
+test("subscribing to a populated tree reports only later changes", async () => {
+  const root = await createRoot();
+  for (const directory of ["src", join("src", "nested"), "docs"]) {
+    await mkdir(join(root, directory), { recursive: true });
+    await Promise.all(
+      Array.from({ length: 20 }, (_, index) =>
+        writeFile(join(root, directory, `existing-${index}.txt`), `${index}`),
+      ),
+    );
+  }
+  const events: FileChange[] = [];
+  const subscription = await subscribeToFileChanges(root, (error, batch) => {
+    expect(error).toBeNull();
+    events.push(...batch);
+  });
+
+  const laterPath = join(root, "src", "nested", "later.txt");
+  await writeFile(laterPath, "later");
+  await expect.poll(() => events.map((event) => event.path)).toContain(laterPath);
+  expect(new Set(events.map((event) => event.path))).toEqual(new Set([laterPath]));
+  await subscription.unsubscribe();
+});
+
 test("covers files populated immediately inside a newly created directory", async () => {
   const root = await createRoot();
   const events: FileChange[] = [];

packages/server/src/server/file-observer/internal/linux.ts

@@ -23,6 +23,11 @@ const RECONCILIATION_DELAY_MS = 50;
 const MAX_CONCURRENT_CLASSIFICATIONS = 32;
 const MAX_QUEUED_CLASSIFICATIONS = 16_384;
 
+// "baseline" establishes coverage of what already exists when the subscription
+// starts, so it reports nothing. "changes" scans topology that appeared while
+// observed, so files found before their directory was covered are creates.
+type ReconcileMode = "baseline" | "changes";
+
 interface DirectorySnapshot {
   directories: Set<string>;
   filesDiscoveredBeforeCoverage: Set<string>;
@@ -62,7 +67,7 @@ class LinuxBackend implements ObservationBackend {
 
   async start(): Promise<void> {
     this.watchDirectory(this.host.root);
-    await this.enqueue([this.host.root]);
+    await this.enqueue([this.host.root], "baseline");
   }
 
   async updateIgnore(): Promise<void> {
@@ -75,7 +80,7 @@ class LinuxBackend implements ObservationBackend {
       }
     }
     this.cancelQueued();
-    await this.enqueue([this.host.root]);
+    await this.enqueue([this.host.root], "changes");
     await Promise.allSettled(inFlight);
   }
 
@@ -159,15 +164,17 @@ class LinuxBackend implements ObservationBackend {
       this.timer = null;
       const scopes = [...this.pendingScopes];
       this.pendingScopes.clear();
-      void this.enqueue(scopes).catch((error: unknown) => this.host.fail(toError(error)));
+      void this.enqueue(scopes, "changes").catch((error: unknown) =>
+        this.host.fail(toError(error)),
+      );
     }, RECONCILIATION_DELAY_MS);
     this.timer.unref();
   }
 
-  private enqueue(scopes: string[]): Promise<void> {
+  private enqueue(scopes: string[], mode: ReconcileMode): Promise<void> {
     this.queueDepth += 1;
     const result = this.reconcileTail
-      .then(() => this.run(scopes))
+      .then(() => this.run(scopes, mode))
       .finally(() => {
         this.queueDepth -= 1;
       });
@@ -175,12 +182,12 @@ class LinuxBackend implements ObservationBackend {
     return result;
   }
 
-  private async run(scopes: string[]): Promise<void> {
+  private async run(scopes: string[], mode: ReconcileMode): Promise<void> {
     if (!this.host.isActive()) return;
     this.inFlight = true;
     const startedAt = performance.now();
     try {
-      await this.reconcile(this.paths.collapse(scopes));
+      await this.reconcile(this.paths.collapse(scopes), mode);
     } catch (error) {
       this.host.metrics.reconciliationFailureCount += 1;
       throw error;
@@ -197,15 +204,17 @@ class LinuxBackend implements ObservationBackend {
     }
   }
 
-  private async reconcile(scopes: string[]): Promise<void> {
+  private async reconcile(scopes: string[], mode: ReconcileMode): Promise<void> {
     for (const scope of scopes) {
       const snapshot = await this.scan(scope);
       if (!this.host.isActive()) return;
       for (const directory of snapshot.directories) {
         if (!this.watchSnapshotDirectory(directory)) snapshot.directories.delete(directory);
       }
-      for (const path of snapshot.filesDiscoveredBeforeCoverage) {
-        this.host.queueEvent("create", path);
+      if (mode === "changes") {
+        for (const path of snapshot.filesDiscoveredBeforeCoverage) {
+          this.host.queueEvent("create", path);
+        }
       }
       for (const [directory, watcher] of this.watchers) {
         const disappeared =

packages/server/src/server/workspace-git-service.observation.integration.test.ts

@@ -1,4 +1,9 @@
 import { flushGitCommandTrace } from "../utils/git-command-trace.js";
+import {
+  startGitCommandMetrics,
+  stopGitCommandMetrics,
+  waitForGitCommandMetricsIdle,
+} from "../utils/run-git-command.js";
 import { createWorkspaceRuntimeStore } from "./workspace-runtime-store/index.js";
 import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
 import { execFileSync } from "node:child_process";
@@ -856,3 +861,59 @@ test.skipIf(process.platform !== "linux")(
   },
   60_000,
 );
+
+test("observation-driven refreshes run their Git work in the background lane", async () => {
+  const root = realpathSync(mkdtempSync(path.join(tmpdir(), "paseo-git-priority-")));
+  const repo = path.join(root, "repo");
+  mkdirSync(repo);
+  const git = (...args: string[]) =>
+    execFileSync("git", ["-c", "commit.gpgsign=false", ...args], { cwd: repo, stdio: "pipe" });
+  git("init", "-b", "main");
+  git("config", "user.email", "test@example.com");
+  git("config", "user.name", "Test");
+  writeFileSync(path.join(repo, "README.md"), "base\n");
+  git("add", ".");
+  git("commit", "-m", "base");
+  const service = new WorkspaceGitServiceImpl({
+    logger: pino({ enabled: false }),
+    paseoHome: path.join(root, "home"),
+  });
+  const heads: Array<string | null> = [];
+  const subscription = service.registerWorkspace({ cwd: repo }, (snapshot) => {
+    heads.push(snapshot.git.currentBranch);
+  });
+  cleanup.push(async () => {
+    subscription.unsubscribe();
+    await service.dispose();
+    rmSync(root, { recursive: true, force: true });
+  });
+  // Registration, observation setup and the first refresh settle before the
+  // measured change.
+  startGitCommandMetrics();
+  await service.getSnapshot(repo, { includeForge: false });
+  await waitForGitCommandMetricsIdle({ quietMs: 500, timeoutMs: 10_000 });
+  stopGitCommandMetrics();
+
+  startGitCommandMetrics();
+  const deliveredBefore = heads.length;
+  git("switch", "-c", "feature");
+  await expect.poll(() => heads.slice(deliveredBefore), { timeout: 10_000 }).toContain("feature");
+  await waitForGitCommandMetricsIdle({ quietMs: 300, timeoutMs: 10_000 });
+  const observed = stopGitCommandMetrics().submissions;
+  expect(observed.length).toBeGreaterThan(0);
+  expect(new Set(observed.map((command) => command.priority))).toEqual(new Set(["background"]));
+
+  startGitCommandMetrics();
+  await service.getSnapshot(repo, { force: true, reason: "test-demand", includeForge: false });
+  await service.getCheckout(repo, { priority: "background" });
+  await waitForGitCommandMetricsIdle({ quietMs: 0, timeoutMs: 10_000 });
+  const demand = stopGitCommandMetrics().submissions;
+  const checkoutStart = demand.findIndex((command) => command.priority === "background");
+  expect(checkoutStart).toBeGreaterThan(0);
+  expect(new Set(demand.slice(0, checkoutStart).map((command) => command.priority))).toEqual(
+    new Set(["normal"]),
+  );
+  expect(new Set(demand.slice(checkoutStart).map((command) => command.priority))).toEqual(
+    new Set(["background"]),
+  );
+}, 30_000);

packages/server/src/server/workspace-git-service.ts

@@ -48,6 +48,7 @@ import {
   runGitCommand,
   type RunGitCommand,
 } from "../utils/run-git-command.js";
+import type { GitProcessPriority } from "../utils/git-process-scheduler.js";
 import { GitObservationPaths } from "./git-observation-paths.js";
 import { branchNameFromRef } from "../utils/worktree-metadata.js";
 import { listPaseoWorktrees, type PaseoWorktreeInfo } from "../utils/worktree.js";
@@ -209,7 +210,10 @@ export interface WorkspaceGitService {
 
   /** Publishes computed snapshot changes independently of workspace UI subscriptions. */
   onSnapshotUpdated(listener: WorkspaceGitSnapshotUpdatedListener): WorkspaceGitSubscription;
-  getCheckout(cwd: string): Promise<ProjectCheckoutLitePayload>;
+  getCheckout(
+    cwd: string,
+    options?: WorkspaceGitCheckoutReadOptions,
+  ): Promise<ProjectCheckoutLitePayload>;
   getSnapshot(
     cwd: string,
     options?: WorkspaceGitSnapshotOptions,
@@ -329,7 +333,28 @@ export type WorkspaceGitSnapshotOptions =
       reason: string;
     };
 
+/**
+ * Who is waiting for a refresh. "demand" refreshes answer a caller; "observation"
+ * refreshes react to watchers, polls and fetches and run in the background Git
+ * lane so they never queue in front of interactive Git work.
+ */
+type WorkspaceGitRefreshOrigin = "demand" | "observation";
+
+export interface WorkspaceGitCheckoutReadOptions {
+  /** Git lane for the fresh checkout read; defaults to the caller's ambient lane. */
+  priority?: GitProcessPriority;
+}
+
+// A caller waiting on a merged refresh keeps it out of the background lane.
+function mergeRefreshOrigins(
+  pending: WorkspaceGitRefreshOrigin,
+  request: WorkspaceGitRefreshOrigin,
+): WorkspaceGitRefreshOrigin {
+  return pending === "demand" ? pending : request;
+}
+
 interface WorkspaceGitRefreshRequest {
+  origin: WorkspaceGitRefreshOrigin;
   force: boolean;
   refreshStructure: boolean;
   refreshWorktree: boolean;
@@ -341,6 +366,7 @@ interface WorkspaceGitRefreshRequest {
 }
 
 interface ScheduledWorkspaceGitRefreshOptions {
+  origin: WorkspaceGitRefreshOrigin;
   force?: boolean;
   scope?: "refs" | "structure" | "worktree";
   includeForge?: boolean;
@@ -782,13 +808,19 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
     return this.requestWorkspaceSnapshot(target, request);
   }
 
-  async getCheckout(cwd: string): Promise<ProjectCheckoutLitePayload> {
+  async getCheckout(
+    cwd: string,
+    options?: WorkspaceGitCheckoutReadOptions,
+  ): Promise<ProjectCheckoutLitePayload> {
     this.assertNotDisposed();
     const normalizedCwd = resolve(cwd);
     const status = await this.deps.getCheckoutStatus(normalizedCwd, {
       paseoHome: this.paseoHome,
       worktreesRoot: this.worktreesRoot,
       logger: this.logger,
+      ...(options?.priority
+        ? { runGitCommand: createRunGitCommand("checkout-read", { priority: options.priority }) }
+        : {}),
     });
     const cached = this.getCachedSnapshot(normalizedCwd);
     if (cached && cached.git.isGit !== status.isGit) {
@@ -1009,6 +1041,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
       await this.resumeWorkspaceIfFolderRestored(target);
     }
     await this.refreshWorkspaceTarget(target, {
+      origin: "demand",
       force: false,
       refreshStructure: true,
       refreshWorktree: true,
@@ -1048,7 +1081,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
       void this.resumeWorkspaceIfFolderRestored(target);
       return;
     }
-    this.scheduleWorkspaceRefresh(target, { queueIfBusy: false });
+    this.scheduleWorkspaceRefresh(target, { origin: "demand", queueIfBusy: false });
   }
 
   onWorkspaceStateMayHaveChanged(cwd: string): void {
@@ -1064,6 +1097,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
       return;
     }
     this.scheduleWorkspaceRefresh(target, {
+      origin: "demand",
       force: true,
       includeForge: true,
       reason: "external-state-change",
@@ -1292,6 +1326,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
         return;
       }
       void this.refreshWorkspaceTarget(target, {
+        origin: "demand",
         force: false,
         refreshStructure: true,
         refreshWorktree: true,
@@ -1762,6 +1797,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
     target.latestFacts = null;
     target.folderMissing = false;
     await this.refreshWorkspaceTarget(target, {
+      origin: "observation",
       force: false,
       refreshStructure: true,
       refreshWorktree: true,
@@ -1805,6 +1841,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
             return false;
           }
           const changed = await this.refreshDegradedPollTarget(workspaceTarget, {
+            origin: "observation",
             force: false,
             refreshStructure: target.repoRoot === null,
             refreshWorktree: true,
@@ -2286,7 +2323,11 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
     for (const workspaceKey of target.workspaceKeys) {
       const workspaceTarget = this.workspaceTargets.get(workspaceKey);
       if (workspaceTarget) {
-        this.scheduleWorkspaceRefresh(workspaceTarget, { scope: "worktree", reason });
+        this.scheduleWorkspaceRefresh(workspaceTarget, {
+          origin: "observation",
+          scope: "worktree",
+          reason,
+        });
       }
     }
   }
@@ -2869,6 +2910,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
           }
         }
         this.scheduleWorkspaceRefresh(workspaceTarget, {
+          origin: "observation",
           scope,
           emitUnchanged: refresh.refreshBase,
           reason,
@@ -2909,6 +2951,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
             workingTreeTargets.add(workingTreeTarget);
           }
           return await this.refreshDegradedPollTarget(workspaceTarget, {
+            origin: "observation",
             force: false,
             refreshStructure: true,
             refreshWorktree: true,
@@ -2940,7 +2983,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
 
   private scheduleWorkspaceRefresh(
     targetOrCwd: WorkspaceGitTarget | string,
-    options?: ScheduledWorkspaceGitRefreshOptions,
+    options: ScheduledWorkspaceGitRefreshOptions,
   ): void {
     const target =
       typeof targetOrCwd === "string"
@@ -3309,6 +3352,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
 
     const force = options?.force === true;
     return {
+      origin: "demand",
       force,
       refreshStructure: true,
       refreshWorktree: true,
@@ -3332,10 +3376,11 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
   }
 
   private buildScheduledRefreshRequest(
-    options: ScheduledWorkspaceGitRefreshOptions | undefined,
+    options: ScheduledWorkspaceGitRefreshOptions,
   ): WorkspaceGitRefreshRequest {
     const scope = options?.scope;
     return {
+      origin: options.origin,
       force: options?.force === true,
       refreshStructure: scope !== "worktree" && scope !== "refs",
       refreshWorktree: scope !== "structure" && scope !== "refs",
@@ -3366,6 +3411,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
       request.movedRemoteRefs,
     );
     return {
+      origin: mergeRefreshOrigins(pending.origin, request.origin),
       force,
       refreshStructure: pending.refreshStructure || request.refreshStructure,
       refreshWorktree: pending.refreshWorktree || request.refreshWorktree,
@@ -3404,7 +3450,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
         });
       }
       try {
-        const runRefreshGitCommand = createRunGitCommand(`workspace-refresh:${request.reason}`);
+        const runRefreshGitCommand = createRunGitCommand(`workspace-refresh:${request.reason}`, {
+          priority: request.origin === "demand" ? "normal" : "background",
+        });
         const admittedSnapshot = await this.workspaceRefreshLimit(() => {
           if (
             target.closed ||
@@ -3792,7 +3840,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
               }
             },
           },
-          createRunGitCommand("background-fetch"),
+          createRunGitCommand("background-fetch", { priority: "background" }),
         );
       } catch (error) {
         this.logger.warn(

packages/server/src/server/workspace-reconciliation-service.test.ts

@@ -772,6 +772,62 @@ describe("WorkspaceReconciliationService", () => {
     expect(projects.get("p1")?.kind).toBe("git");
   });
 
+  test("reads sibling checkouts a few at a time in the background Git lane", async () => {
+    const projectRoot = "/reconcile/fan-out/root";
+    const { projects, workspaces, projectRegistry, workspaceRegistry } = createTestRegistries();
+    projects.set(
+      "p1",
+      createPersistedProjectRecord({
+        projectId: "p1",
+        rootPath: projectRoot,
+        kind: "git",
+        displayName: "fan-out",
+        createdAt: timestamp,
+        updatedAt: timestamp,
+      }),
+    );
+    const siblingCwds = Array.from({ length: 40 }, (_, index) => `${projectRoot}-${index}`);
+    for (const [index, cwd] of siblingCwds.entries()) {
+      workspaces.set(
+        `w${index}`,
+        createPersistedWorkspaceRecord({
+          workspaceId: `w${index}`,
+          projectId: "p1",
+          cwd,
+          kind: "worktree",
+          displayName: `sibling-${index}`,
+          createdAt: timestamp,
+          updatedAt: timestamp,
+        }),
+      );
+    }
+    const reads: Array<{ cwd: string; priority: string | undefined }> = [];
+    let inFlight = 0;
+    let maxInFlight = 0;
+    const service = new WorkspaceReconciliationService({
+      projectRegistry,
+      workspaceRegistry,
+      logger: createTestLogger(),
+      inspectDirectory: async () => "directory",
+      workspaceGitService: {
+        async getCheckout(cwd, options) {
+          reads.push({ cwd, priority: options?.priority });
+          inFlight += 1;
+          maxInFlight = Math.max(maxInFlight, inFlight);
+          await new Promise((resolve) => setTimeout(resolve, 1));
+          inFlight -= 1;
+          return createCheckout(cwd, { isGit: true, currentBranch: "main", worktreeRoot: cwd });
+        },
+      },
+    });
+
+    await service.runOnce();
+
+    expect(reads.map((read) => read.cwd).sort()).toEqual([projectRoot, ...siblingCwds].sort());
+    expect(new Set(reads.map((read) => read.priority))).toEqual(new Set(["background"]));
+    expect(maxInFlight).toBe(4);
+  });
+
   test("deduplicates equivalent project and workspace paths across legacy duplicate projects", async () => {
     const projectRoot = realpathSync(mkdtempSync(path.join(tmpdir(), "reconcile-global-root-")));
     const workspaceRoot = realpathSync(

packages/server/src/server/workspace-reconciliation-service.ts

@@ -1,6 +1,7 @@
 import { watch as watchPath } from "node:fs";
 import { stat } from "node:fs/promises";
 import { setImmediate as yieldToLoop } from "node:timers/promises";
+import pLimit from "p-limit";
 import type { ProjectCheckoutLitePayload } from "@getpaseo/protocol/messages";
 import type pino from "pino";
 import type {
@@ -22,6 +23,9 @@ import { deriveProjectKey } from "./project-key.js";
 const DEFAULT_RESCAN_INTERVAL_MS = 5 * 60_000;
 const DEFAULT_DEBOUNCE_MS = 100;
 const INSPECTION_CONCURRENCY = 2;
+// Each checkout read runs ~15 Git commands. Reading every sibling at once
+// queued ~1,300 commands per rescan on a project with ~90 worktrees.
+const CHECKOUT_READ_CONCURRENCY = 4;
 const DEFAULT_INSPECTION_TIMEOUT_MS = 5000;
 
 export type ProjectUpdate =
@@ -346,10 +350,11 @@ export class WorkspaceReconciliationService {
     changes: ReconciliationChange[],
   ): Promise<void> {
     const checkoutReads: CachedCheckoutRead[] = [];
+    const checkoutReadLimit = pLimit(CHECKOUT_READ_CONCURRENCY);
     const readCheckout = (cwd: string): Promise<ProjectCheckoutLitePayload> => {
       const existing = checkoutReads.find((read) => areEquivalentPaths(read.cwd, cwd));
       if (existing) return existing.checkout;
-      const checkout = this.readCheckout(cwd);
+      const checkout = checkoutReadLimit(() => this.readCheckout(cwd));
       checkoutReads.push({ cwd, checkout });
       return checkout;
     };
@@ -581,7 +586,9 @@ export class WorkspaceReconciliationService {
         mainRepoRoot: null,
       };
     }
-    return this.workspaceGitService.getCheckout(cwd);
+    // Reconciliation converges in the background; it must not queue ahead of
+    // Git work a user is waiting for.
+    return this.workspaceGitService.getCheckout(cwd, { priority: "background" });
   }
 
   private wakeInspectionWaiters(): void {

packages/server/src/utils/git-process-scheduler.test.ts

@@ -68,6 +68,32 @@ describe("GitProcessScheduler", () => {
     expect(started).toEqual([0, 3, 2]);
   });
 
+  test("admits normal work before a queued background burst", async () => {
+    const scheduler = new GitProcessScheduler({
+      maxProcessesPerSecond: 100,
+      maxProcessConcurrency: 1,
+    });
+    const releases: Array<() => void> = [];
+    const started: number[] = [];
+
+    const activeBackground = scheduler.run(createConcurrencyTask(0, started, releases), {
+      priority: "background",
+    });
+    await vi.waitFor(() => expect(started).toEqual([0]));
+    const queuedBackground = [2, 3, 4].map((index) =>
+      scheduler.run(createConcurrencyTask(index, started, releases), { priority: "background" }),
+    );
+    const normalWork = scheduler.run(createConcurrencyTask(5, started, releases));
+    const highWork = scheduler.run(createConcurrencyTask(6, started, releases), {
+      priority: "high",
+    });
+
+    releases.shift()?.();
+    await Promise.all([activeBackground, ...queuedBackground, normalWork, highWork]);
+
+    expect(started).toEqual([0, 6, 5, 2, 3, 4]);
+  });
+
   test("yields to daemon liveness work between process starts", async () => {
     const scheduler = new GitProcessScheduler({
       maxProcessesPerSecond: 100,

packages/server/src/utils/git-process-scheduler.ts

@@ -16,7 +16,10 @@ export interface ScheduledGitProcess<T> {
   exited: Promise<void>;
 }
 
-export type GitProcessPriority = "normal" | "high";
+// Lanes drain in this order. "background" is for daemon-initiated fan-out
+// (reconciliation, observation-driven refreshes, fetches) so it never sits in
+// front of work a user is waiting for.
+export type GitProcessPriority = "high" | "normal" | "background";
 
 function parsePositiveInteger(value: string | undefined): number | undefined {
   if (value === undefined) {
@@ -46,8 +49,11 @@ export function resolveGitProcessPolicy(input: {
 
 export class GitProcessScheduler {
   private readonly startThrottled;
-  private readonly highPriorityQueue: Array<() => void> = [];
-  private readonly normalPriorityQueue: Array<() => void> = [];
+  private readonly queues: Record<GitProcessPriority, Array<() => void>> = {
+    high: [],
+    normal: [],
+    background: [],
+  };
   private admitted = 0;
   private running = 0;
   private waiting = 0;
@@ -84,9 +90,7 @@ export class GitProcessScheduler {
           this.scheduleDrain();
         });
       };
-      const queue =
-        options?.priority === "high" ? this.highPriorityQueue : this.normalPriorityQueue;
-      queue.push(admit);
+      this.queues[options?.priority ?? "normal"].push(admit);
       if (this.admitted === 0 && !this.drainScheduled) {
         this.drainOne();
       } else {
@@ -99,19 +103,28 @@ export class GitProcessScheduler {
     if (this.drainScheduled || this.admitted >= this.policy.maxProcessConcurrency) {
       return;
     }
-    if (this.highPriorityQueue.length === 0 && this.normalPriorityQueue.length === 0) {
+    if (!this.hasQueuedWork()) {
       return;
     }
     this.drainScheduled = true;
     scheduleImmediate(() => this.drainOne());
   }
 
+  private hasQueuedWork(): boolean {
+    return (
+      this.queues.high.length > 0 ||
+      this.queues.normal.length > 0 ||
+      this.queues.background.length > 0
+    );
+  }
+
   private drainOne(): void {
     this.drainScheduled = false;
     if (this.admitted >= this.policy.maxProcessConcurrency) {
       return;
     }
-    const admit = this.highPriorityQueue.shift() ?? this.normalPriorityQueue.shift();
+    const admit =
+      this.queues.high.shift() ?? this.queues.normal.shift() ?? this.queues.background.shift();
     if (!admit) {
       return;
     }

packages/server/src/utils/run-git-command.ts

@@ -67,8 +67,17 @@ export type RunGitCommand = (
   options: GitCommandOptions,
 ) => Promise<GitCommandResult>;
 
-export function createRunGitCommand(provenance: string): RunGitCommand {
-  return (args, options) => runGitCommandWithProvenance(args, options, provenance);
+export interface GitCommandRunnerOptions {
+  /** Overrides the ambient `runWithGitCommandPriority` lane for every command this runner starts. */
+  priority?: GitProcessPriority;
+}
+
+export function createRunGitCommand(
+  provenance: string,
+  runnerOptions: GitCommandRunnerOptions = {},
+): RunGitCommand {
+  return (args, options) =>
+    executeGitCommand(args, options, decodeUtf8, { provenance, priority: runnerOptions.priority });
 }
 
 export interface GitCommandMetric {
@@ -97,6 +106,7 @@ export interface GitCommandMetricsSnapshot {
 export interface GitCommandSubmissionMetric {
   args: string[];
   cwd: string;
+  priority: GitProcessPriority;
 }
 
 interface GitCommandMetricsState {
@@ -188,8 +198,8 @@ function snapshotGitCommandMetricsState(state: GitCommandMetricsState): GitComma
   return {
     commands: [...state.commands],
     submissions: state.submissions.map((submission) => ({
+      ...submission,
       args: [...submission.args],
-      cwd: submission.cwd,
     })),
     submitted: state.submitted,
     started: state.started,
@@ -209,12 +219,16 @@ export function snapshotGitCommandRuntimeMetrics(): GitCommandRuntimeMetricsSnap
   });
 }
 
-function submitGitCommandMetric(args: string[], cwd: string): GitCommandMetricsState | null {
+function submitGitCommandMetric(
+  args: string[],
+  cwd: string,
+  priority: GitProcessPriority,
+): GitCommandMetricsState | null {
   const state = gitCommandMetricsState;
   if (!state) {
     return null;
   }
-  state.submissions.push({ args: [...args], cwd });
+  state.submissions.push({ args: [...args], cwd, priority });
   state.submitted += 1;
   state.lastSubmittedAtMs = Date.now();
   return state;
@@ -258,12 +272,13 @@ function getEnvOverlayKeys(envOverlay: ProcessEnvRecord | undefined): string[] {
   return Object.keys(envOverlay ?? {}).sort();
 }
 
-function runGitCommandWithProvenance(
-  args: string[],
-  options: GitCommandOptions,
-  provenance?: string,
-): Promise<GitCommandResult> {
-  return executeGitCommand(args, options, (output) => output.toString("utf8"), provenance);
+function decodeUtf8(output: Buffer): string {
+  return output.toString("utf8");
+}
+
+interface GitCommandSubmission {
+  provenance?: string;
+  priority?: GitProcessPriority;
 }
 
 /** Binary stdout preserves cat-file byte framing before text decoding. */
@@ -271,21 +286,22 @@ export function runGitCommandBytes(
   args: string[],
   options: GitCommandOptions,
 ): Promise<GitCommandResult<Buffer>> {
-  return executeGitCommand(args, options, (output) => output);
+  return executeGitCommand(args, options, (output) => output, {});
 }
 
 function executeGitCommand<Output>(
   args: string[],
   options: GitCommandOptions,
   decode: (output: Buffer) => Output,
-  provenance?: string,
+  submission: GitCommandSubmission,
 ): Promise<GitCommandResult<Output>> {
-  const metricsState = submitGitCommandMetric(args, options.cwd);
+  const priority = submission.priority ?? gitCommandPriority.getStore() ?? "normal";
+  const metricsState = submitGitCommandMetric(args, options.cwd, priority);
   const commandTrace = submitGitCommandTrace(args, options.cwd, {
     active: gitProcessScheduler.activeCount,
     pending: gitProcessScheduler.pendingCount,
   });
-  const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args), provenance);
+  const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args), submission.provenance);
   const startCommand = () => {
     let releaseProcessSlot!: () => void;
     const exited = new Promise<void>((resolve) => {
@@ -558,9 +574,7 @@ function executeGitCommand<Output>(
     });
     return { result: resultPromise, exited };
   };
-  const promise = gitProcessScheduler.run(startCommand, {
-    priority: gitCommandPriority.getStore() ?? "normal",
-  });
+  const promise = gitProcessScheduler.run(startCommand, { priority });
   gitRuntimeMetrics.observeLimiter(
     gitProcessScheduler.activeCount,
     gitProcessScheduler.pendingCount,
@@ -568,7 +582,8 @@ function executeGitCommand<Output>(
   return promise;
 }
 
-export const runGitCommand: RunGitCommand = runGitCommandWithProvenance;
+export const runGitCommand: RunGitCommand = (args, options) =>
+  executeGitCommand(args, options, decodeUtf8, {});
 
 function formatGitCommand(args: string[]): string {
   return ["git", ...args].join(" ");