Load the Changes diff quickly while the daemon reconciles worktrees (#9243)

* fix(server): keep cold diffs out from behind background git fan-out

Add a background lane to the git process scheduler and run reconciliation
checkout reads, observation-driven workspace refreshes and background fetches
in it. Bound reconciliation checkout reads to four at a time. Stop the Linux
file observer's startup scan from reporting existing files as creates, which
made every cold diff compute and discard its first revision.

* fix(server): pick the git job when its start token fires

Background jobs that already reserved a start-rate token no longer start
ahead of normal work submitted after them.

* fix(server): re-read the working tree once its watcher covers it

The Linux observer no longer reports pre-existing files at startup, so an
edit made between a workspace's first snapshot and its watcher subscribing
reached no consumer. WorkspaceGit now runs one background working-tree read
once observation setup completes with a live watcher.

Demand refreshes keep the caller's Git lane instead of forcing normal, so
user actions running at high priority are not demoted.

Scope the populated-tree observer test to Linux (FSEvents replays recent
writes) and update the observer benchmark test, which expected the removed
startup batch.

* fix(server): re-read checkout metadata once observation covers it

A branch switch made before the Git metadata watcher subscribed left the
cached branch stale. The post-coverage read now refreshes structure as well
as the working tree, matching the refresh the Linux startup events used to
trigger.

packages/server/src/utils/git-process-scheduler.test.ts

@@ -68,6 +68,64 @@ describe("GitProcessScheduler", () => {
     expect(started).toEqual([0, 3, 2]);
   });
 
+  test("admits normal work before a queued background burst", async () => {
+    const scheduler = new GitProcessScheduler({
+      maxProcessesPerSecond: 100,
+      maxProcessConcurrency: 1,
+    });
+    const releases: Array<() => void> = [];
+    const started: number[] = [];
+
+    const activeBackground = scheduler.run(createConcurrencyTask(0, started, releases), {
+      priority: "background",
+    });
+    await vi.waitFor(() => expect(started).toEqual([0]));
+    const queuedBackground = [2, 3, 4].map((index) =>
+      scheduler.run(createConcurrencyTask(index, started, releases), { priority: "background" }),
+    );
+    const normalWork = scheduler.run(createConcurrencyTask(5, started, releases));
+    const highWork = scheduler.run(createConcurrencyTask(6, started, releases), {
+      priority: "high",
+    });
+
+    releases.shift()?.();
+    await Promise.all([activeBackground, ...queuedBackground, normalWork, highWork]);
+
+    expect(started).toEqual([0, 6, 5, 2, 3, 4]);
+  });
+
+  test("starts normal work at the next start slot when background work saturates the rate", async () => {
+    vi.useFakeTimers();
+    const scheduler = new GitProcessScheduler({
+      maxProcessesPerSecond: 2,
+      maxProcessConcurrency: 8,
+    });
+    const started: string[] = [];
+    const task = (name: string) => () => {
+      started.push(name);
+      const completed = Promise.resolve();
+      return { result: completed, exited: completed };
+    };
+    const background = Array.from({ length: 8 }, (_, index) =>
+      scheduler.run(task(`background-${index}`), { priority: "background" }),
+    );
+    // Admission yields a real event-loop turn per process; let every
+    // background job reach the start-rate limiter.
+    for (let turn = 0; turn < 10; turn += 1) {
+      await new Promise<void>((resolve) => scheduleImmediate(resolve));
+    }
+    expect(started).toEqual(["background-0", "background-1"]);
+
+    const normal = scheduler.run(task("normal"));
+    while (started.length < 3) {
+      await vi.advanceTimersToNextTimerAsync();
+    }
+
+    expect(started[2]).toBe("normal");
+    await vi.runAllTimersAsync();
+    await Promise.all([...background, normal]);
+  });
+
   test("yields to daemon liveness work between process starts", async () => {
     const scheduler = new GitProcessScheduler({
       maxProcessesPerSecond: 100,

packages/server/src/utils/git-process-scheduler.ts

@@ -16,7 +16,10 @@ export interface ScheduledGitProcess<T> {
   exited: Promise<void>;
 }
 
-export type GitProcessPriority = "normal" | "high";
+// Lanes drain in this order. "background" is for daemon-initiated fan-out
+// (reconciliation, observation-driven refreshes, fetches) so it never sits in
+// front of work a user is waiting for.
+export type GitProcessPriority = "high" | "normal" | "background";
 
 function parsePositiveInteger(value: string | undefined): number | undefined {
   if (value === undefined) {
@@ -44,11 +47,19 @@ export function resolveGitProcessPolicy(input: {
   };
 }
 
+// Admission reserves a process slot and a start-rate token; the job runs only
+// once its token fires. The job is chosen when the token fires, not when the
+// slot is reserved, so work submitted while background jobs hold every reserved
+// slot still takes the next start.
 export class GitProcessScheduler {
   private readonly startThrottled;
-  private readonly highPriorityQueue: Array<() => void> = [];
-  private readonly normalPriorityQueue: Array<() => void> = [];
+  private readonly queues: Record<GitProcessPriority, Array<() => Promise<void>>> = {
+    high: [],
+    normal: [],
+    background: [],
+  };
   private admitted = 0;
+  private reservedStarts = 0;
   private running = 0;
   private waiting = 0;
   private drainScheduled = false;
@@ -59,7 +70,7 @@ export class GitProcessScheduler {
       interval: 1_000,
       strict: true,
     });
-    this.startThrottled = throttle((start: () => Promise<void>) => start());
+    this.startThrottled = throttle(() => this.startNext());
   }
 
   get activeCount(): number {
@@ -76,17 +87,7 @@ export class GitProcessScheduler {
   ): Promise<T> {
     this.waiting += 1;
     return new Promise<T>((resolve, reject) => {
-      const admit = () => {
-        void this.startThrottled(() => this.execute(start, resolve, reject)).catch((error) => {
-          this.waiting = Math.max(0, this.waiting - 1);
-          this.admitted = Math.max(0, this.admitted - 1);
-          reject(error);
-          this.scheduleDrain();
-        });
-      };
-      const queue =
-        options?.priority === "high" ? this.highPriorityQueue : this.normalPriorityQueue;
-      queue.push(admit);
+      this.queues[options?.priority ?? "normal"].push(() => this.execute(start, resolve, reject));
       if (this.admitted === 0 && !this.drainScheduled) {
         this.drainOne();
       } else {
@@ -95,11 +96,12 @@ export class GitProcessScheduler {
     });
   }
 
+  private canReserveStart(): boolean {
+    return this.admitted < this.policy.maxProcessConcurrency && this.waiting > this.reservedStarts;
+  }
+
   private scheduleDrain(): void {
-    if (this.drainScheduled || this.admitted >= this.policy.maxProcessConcurrency) {
-      return;
-    }
-    if (this.highPriorityQueue.length === 0 && this.normalPriorityQueue.length === 0) {
+    if (this.drainScheduled || !this.canReserveStart()) {
       return;
     }
     this.drainScheduled = true;
@@ -108,20 +110,28 @@ export class GitProcessScheduler {
 
   private drainOne(): void {
     this.drainScheduled = false;
-    if (this.admitted >= this.policy.maxProcessConcurrency) {
-      return;
-    }
-    const admit = this.highPriorityQueue.shift() ?? this.normalPriorityQueue.shift();
-    if (!admit) {
+    if (!this.canReserveStart()) {
       return;
     }
     this.admitted += 1;
-    admit();
+    this.reservedStarts += 1;
+    void this.startThrottled();
     // Spawning can be synchronous and expensive on a throttled host. Let timers,
     // sockets, and child exits run before admitting another Git process.
     this.scheduleDrain();
   }
 
+  private startNext(): Promise<void> {
+    this.reservedStarts -= 1;
+    // Every reservation is backed by a queued job: jobs leave the queues only here.
+    const next =
+      this.queues.high.shift() ?? this.queues.normal.shift() ?? this.queues.background.shift();
+    if (!next) {
+      throw new Error("Git process start fired without a queued process");
+    }
+    return next();
+  }
+
   private async execute<T>(
     start: () => ScheduledGitProcess<T>,
     resolve: (value: T | PromiseLike<T>) => void,

packages/server/src/utils/run-git-command.ts

@@ -67,8 +67,17 @@ export type RunGitCommand = (
   options: GitCommandOptions,
 ) => Promise<GitCommandResult>;
 
-export function createRunGitCommand(provenance: string): RunGitCommand {
-  return (args, options) => runGitCommandWithProvenance(args, options, provenance);
+export interface GitCommandRunnerOptions {
+  /** Overrides the ambient `runWithGitCommandPriority` lane for every command this runner starts. */
+  priority?: GitProcessPriority;
+}
+
+export function createRunGitCommand(
+  provenance: string,
+  runnerOptions: GitCommandRunnerOptions = {},
+): RunGitCommand {
+  return (args, options) =>
+    executeGitCommand(args, options, decodeUtf8, { provenance, priority: runnerOptions.priority });
 }
 
 export interface GitCommandMetric {
@@ -97,6 +106,7 @@ export interface GitCommandMetricsSnapshot {
 export interface GitCommandSubmissionMetric {
   args: string[];
   cwd: string;
+  priority: GitProcessPriority;
 }
 
 interface GitCommandMetricsState {
@@ -188,8 +198,8 @@ function snapshotGitCommandMetricsState(state: GitCommandMetricsState): GitComma
   return {
     commands: [...state.commands],
     submissions: state.submissions.map((submission) => ({
+      ...submission,
       args: [...submission.args],
-      cwd: submission.cwd,
     })),
     submitted: state.submitted,
     started: state.started,
@@ -209,12 +219,16 @@ export function snapshotGitCommandRuntimeMetrics(): GitCommandRuntimeMetricsSnap
   });
 }
 
-function submitGitCommandMetric(args: string[], cwd: string): GitCommandMetricsState | null {
+function submitGitCommandMetric(
+  args: string[],
+  cwd: string,
+  priority: GitProcessPriority,
+): GitCommandMetricsState | null {
   const state = gitCommandMetricsState;
   if (!state) {
     return null;
   }
-  state.submissions.push({ args: [...args], cwd });
+  state.submissions.push({ args: [...args], cwd, priority });
   state.submitted += 1;
   state.lastSubmittedAtMs = Date.now();
   return state;
@@ -258,12 +272,13 @@ function getEnvOverlayKeys(envOverlay: ProcessEnvRecord | undefined): string[] {
   return Object.keys(envOverlay ?? {}).sort();
 }
 
-function runGitCommandWithProvenance(
-  args: string[],
-  options: GitCommandOptions,
-  provenance?: string,
-): Promise<GitCommandResult> {
-  return executeGitCommand(args, options, (output) => output.toString("utf8"), provenance);
+function decodeUtf8(output: Buffer): string {
+  return output.toString("utf8");
+}
+
+interface GitCommandSubmission {
+  provenance?: string;
+  priority?: GitProcessPriority;
 }
 
 /** Binary stdout preserves cat-file byte framing before text decoding. */
@@ -271,21 +286,22 @@ export function runGitCommandBytes(
   args: string[],
   options: GitCommandOptions,
 ): Promise<GitCommandResult<Buffer>> {
-  return executeGitCommand(args, options, (output) => output);
+  return executeGitCommand(args, options, (output) => output, {});
 }
 
 function executeGitCommand<Output>(
   args: string[],
   options: GitCommandOptions,
   decode: (output: Buffer) => Output,
-  provenance?: string,
+  submission: GitCommandSubmission,
 ): Promise<GitCommandResult<Output>> {
-  const metricsState = submitGitCommandMetric(args, options.cwd);
+  const priority = submission.priority ?? gitCommandPriority.getStore() ?? "normal";
+  const metricsState = submitGitCommandMetric(args, options.cwd, priority);
   const commandTrace = submitGitCommandTrace(args, options.cwd, {
     active: gitProcessScheduler.activeCount,
     pending: gitProcessScheduler.pendingCount,
   });
-  const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args), provenance);
+  const runtimeMetric = gitRuntimeMetrics.submit(getGitOperation(args), submission.provenance);
   const startCommand = () => {
     let releaseProcessSlot!: () => void;
     const exited = new Promise<void>((resolve) => {
@@ -558,9 +574,7 @@ function executeGitCommand<Output>(
     });
     return { result: resultPromise, exited };
   };
-  const promise = gitProcessScheduler.run(startCommand, {
-    priority: gitCommandPriority.getStore() ?? "normal",
-  });
+  const promise = gitProcessScheduler.run(startCommand, { priority });
   gitRuntimeMetrics.observeLimiter(
     gitProcessScheduler.activeCount,
     gitProcessScheduler.pendingCount,
@@ -568,7 +582,8 @@ function executeGitCommand<Output>(
   return promise;
 }
 
-export const runGitCommand: RunGitCommand = runGitCommandWithProvenance;
+export const runGitCommand: RunGitCommand = (args, options) =>
+  executeGitCommand(args, options, decodeUtf8, {});
 
 function formatGitCommand(args: string[]): string {
   return ["git", ...args].join(" ");