feat(diff): stream Changes revisions without blocking the socket (#8907)

* feat(protocol): define streamed diff revisions and resolve tag bases

* feat(server): stream shared diff revisions with paced file bodies

* feat(client): observe streamed diffs with atomic revision updates

* feat(app): show the Changes tree while diff bodies stream

* fix(server): retain the legacy structured diff payload budget

* fix(diff): preserve revision ordering and readable files

* fix(diff): select renamed patches and read dangling symlinks

* fix(diff): preserve nested checkouts and ambiguous patch paths

Only regular files and symlinks are readable untracked entries. Prefer patch metadata to ambiguous unquoted headers so paths containing a prefix delimiter keep their hunks.

* fix(diff): preserve BOM text and complete file replacements

Decode each string without consuming its leading BOM. File-type replacements emit multiple patch sections for one listed path, so retain all matching hunks. Assert the optional rename metadata in the existing commit-file journey.

* fix(diff): highlight symlinks from their stored target text

Working tree source context must match the symlink blob emitted by Git rather than the contents reached by following the link. Otherwise highlight runs do not cover the patch line and encoding fails the revision.

* fix(diff): tolerate disappearing highlight sources

Source context is optional after Git has produced the patch. Missing worktree link metadata must preserve the existing hunk-based highlighting fallback instead of failing the revision.

* fix(diff): batch cold reads and refresh logical comparison inputs

Acquire bounded pages of tracked patches lazily after Begin while retaining per-file highlighting and awaited frame delivery. Keep legacy preferred-base selection live across revisions and include submodule Git state in worktree cache identity.

* fix(diff): distinguish gitlink directories from replacements

A raw old gitlink mode also occurs when a submodule becomes a regular file. Query nested Git state only for a current directory, preserving normal file identity for replacements.

* fix(diff): invalidate timestamp-preserving file replacements

Size and modification time do not identify live file contents after a preserving copy. Include filesystem change time and file identity for tracked and untracked reuse, and verify live revisions agree with fresh reads.

* fix(diff): align highlight tokens with normalized patch lines

Git can normalize CRLF worktrees while highlighting reads their original line endings. Match tokens to the emitted patch text and trim only a normalization CR before encoding runs; mismatched source tokens are not attached.

* fix(diff): refresh equivalent worktree paths across platforms

Explicit refreshes must match filesystem-equivalent roots, including Windows path aliases. Use the existing realpath-aware matcher, budget real Git completion waits consistently, and assert optional status metadata in the filesystem commit-file journey.

* fix(diff): validate Git inputs and resolve refresh roots

Reuse now accounts for effective attributes, configuration, and canonical worktree blobs as well as filesystem state. Explicit refresh resolves repository identity like subscription creation, and callers await that resolution before acknowledging the command.

* fix(diff): keep non-Git workspace refresh successful

Diff invalidation has no target repository for an ordinary folder. Treat that expected absence as a no-op while preserving other root-resolution errors.

* fix(diff): observe scoped Git interpretation settings

Classify each metadata root with its worktree and common scopes, retaining both when the directories coincide. Shared attributes affect all checkouts, while config.worktree belongs to the owning checkout. Real subscriptions verify automatic binary and hunk-context refreshes in main and linked worktrees.

* fix(diff): observe interpretation inputs for pinned comparisons

Immutable commit objects still depend on mutable Git configuration and attributes. Declare those metadata inputs independently of ref, index, and worktree mutability, while keeping unrelated endpoint events out of pinned comparisons.

* fix(diff): observe interpretation independently of comparison endpoints

Commit comparisons still read worktree attributes. Watch those inputs independently, checking effective Git interpretation before invalidating fixed contents. Retain paths hidden by whitespace filtering so attribute changes can expose them. Forward binary frames through the branch-loading test interceptor.

packages/server/src/utils/checkout-git.commits.test.ts

@@ -3,7 +3,12 @@ import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs"
 import { tmpdir } from "os";
 import { join } from "path";
 import { afterEach, describe, expect, it } from "vitest";
-import { getCommitFileDiff, listCheckoutCommits } from "./checkout-git.js";
+import {
+  getCommitFileDiff,
+  listCheckoutCommits,
+  getCheckoutStatus,
+  getCheckoutDiff,
+} from "./checkout-git.js";
 import { writePaseoWorktreeMetadata } from "./worktree-metadata.js";
 
 const tempDirs: string[] = [];
@@ -393,3 +398,45 @@ describe("listCheckoutCommits", () => {
     ]);
   });
 });
+
+it("compares tag-based worktrees in status, history and legacy diff modes", async () => {
+  const { repoDir: mainRepoDir, tempDir } = initRepoOnMain();
+  const worktreesRoot = join(tempDir, "worktrees");
+  const repoDir = join(worktreesRoot, "repo-hash", "tag-feature");
+  mkdirSync(join(worktreesRoot, "repo-hash"), { recursive: true });
+  const context = { worktreesRoot };
+  git(["tag", "v1"], mainRepoDir);
+  git(["worktree", "add", "-b", "tag-feature", repoDir, "v1"], mainRepoDir);
+  await writePaseoWorktreeMetadata(repoDir, { baseRefName: "refs/tags/v1" });
+  commitFile(repoDir, "tag.txt", "committed\n", "tag feature");
+  writeFileSync(join(repoDir, "tag.txt"), "edited\n");
+  const status = await getCheckoutStatus(repoDir, context);
+  expect(status).toMatchObject({
+    isGit: true,
+    comparisonBaseRef: "refs/tags/v1",
+    aheadBehind: { ahead: 1, behind: 0 },
+  });
+  const history = await listCheckoutCommits({ cwd: repoDir, context });
+  expect(history.baseRef).toBe("refs/tags/v1");
+  expect(history.commits[0].subject).toBe("tag feature");
+  const committed = await getCheckoutDiff(repoDir, { mode: "base", baseRef: "refs/tags/v1" });
+  expect(committed.diff).toContain("+committed");
+  const all = await getCheckoutDiff(repoDir, { mode: "all", baseRef: "refs/tags/v1" });
+  expect(all.diff).toContain("+edited");
+});
+
+it("announces a usable comparison base while checked out on the base branch", async () => {
+  const cwd = makeTempDir();
+  git(["init", "-b", "main"], cwd);
+  git(["config", "user.name", "Test"], cwd);
+  git(["config", "user.email", "test@example.com"], cwd);
+  writeFileSync(join(cwd, "a.txt"), "before\n");
+  git(["add", "."], cwd);
+  commit(cwd, "base");
+  writeFileSync(join(cwd, "a.txt"), "after\n");
+  const status = await getCheckoutStatus(cwd);
+  expect(status).toMatchObject({ isGit: true, comparisonBaseRef: "refs/heads/main" });
+  if (!status.isGit || !status.comparisonBaseRef) throw new Error("Missing comparison base");
+  const all = await getCheckoutDiff(cwd, { mode: "all", baseRef: status.comparisonBaseRef });
+  expect(all.diff).toContain("+after");
+});

packages/server/src/utils/checkout-git.ts

@@ -40,7 +40,7 @@ import {
 } from "../services/forge-cli-command.js";
 import { parseGitRevParsePath, resolveGitRevParsePath } from "./git-rev-parse-path.js";
 import { runGitCommand, type RunGitCommand } from "./run-git-command.js";
-import { readGitFileContents } from "./git-file-contents.js";
+import { readGitFileContents, readGitFileContentAtRef } from "./git-file-contents.js";
 import { unquoteGitPath } from "./git-quoted-path.js";
 import { isPaseoOwnedWorktreeCwd, resolvePaseoWorktreesBaseRoot } from "./worktree.js";
 import {
@@ -613,22 +613,6 @@ async function listCheckoutFileChanges(
   return Array.from(byPath.values());
 }
 
-async function readGitFileContentAtRef(
-  cwd: string,
-  ref: string,
-  path: string,
-): Promise<string | null> {
-  try {
-    const { stdout } = await runGitCommand(["show", `${ref}:${path}`], {
-      cwd,
-      envOverlay: READ_ONLY_GIT_ENV,
-    });
-    return stdout;
-  } catch {
-    return null;
-  }
-}
-
 async function tryResolveMergeBase(cwd: string, baseRef: string): Promise<string | null> {
   try {
     const { stdout } = await runGitCommand(["merge-base", baseRef, "HEAD"], {
@@ -669,7 +653,12 @@ function normalizeNumstatPath(pathField: string): string {
 }
 
 // Parsed patches need a stable wire format regardless of terminal presentation settings.
-const GIT_PATCH_FORMAT_ARGS = ["--no-ext-diff", "--no-color", "--src-prefix=a/", "--dst-prefix=b/"];
+export const GIT_PATCH_FORMAT_ARGS = [
+  "--no-ext-diff",
+  "--no-color",
+  "--src-prefix=a/",
+  "--dst-prefix=b/",
+] as const;
 
 function buildGitDiffArgs(args: { ignoreWhitespace?: boolean; extra: string[] }): string[] {
   return [
@@ -906,6 +895,7 @@ export interface CheckoutStatusGitNonPaseo {
   isDirty: boolean;
   baseRef: string | null;
   baseRefExact?: string | null;
+  comparisonBaseRef?: string | null;
   aheadBehind: AheadBehind | null;
   // Remote-tracking ref for currentBranch, e.g. "refs/remotes/origin/main". Null when the
   // branch has no upstream or git could not resolve it. aheadOfOrigin/behindOfOrigin are
@@ -926,6 +916,7 @@ export interface CheckoutStatusGitPaseo {
   isDirty: boolean;
   baseRef: string;
   baseRefExact?: string | null;
+  comparisonBaseRef?: string | null;
   aheadBehind: AheadBehind | null;
   upstreamRef: string | null;
   aheadOfOrigin: number | null;
@@ -1748,6 +1739,24 @@ function isQualifiedBranchRef(ref: string): boolean {
   return ref.startsWith("refs/heads/") || ref.startsWith("refs/remotes/");
 }
 
+async function resolveCommitComparisonRef(
+  cwd: string,
+  name: string,
+  context?: CheckoutContext,
+): Promise<string> {
+  await getRunGitCommand(context)(["rev-parse", "--verify", `${name}^{commit}`], {
+    cwd,
+    envOverlay: READ_ONLY_GIT_ENV,
+    logger: context?.logger,
+  });
+  const result = await getRunGitCommand(context)(["rev-parse", "--symbolic-full-name", name], {
+    cwd,
+    envOverlay: READ_ONLY_GIT_ENV,
+    logger: context?.logger,
+  });
+  return result.stdout.trim() || name;
+}
+
 async function resolveBestComparisonBaseRef(
   cwd: string,
   baseRef: string,
@@ -1774,11 +1783,17 @@ async function resolveBestComparisonBaseRef(
     return normalized.localName;
   }
 
-  const refName =
-    baseRef.startsWith("origin/") || baseRef.startsWith("refs/remotes/origin/")
-      ? normalized.originRef
-      : normalized.localName;
-  throw new Error(`Base branch not found locally or on origin: ${refName}`);
+  return resolveCommitComparisonRef(cwd, baseRef, context);
+}
+
+async function resolveQualifiedComparisonBaseRef(
+  cwd: string,
+  baseRef: string,
+  context: CheckoutContext,
+): Promise<string> {
+  const comparison = context.facts?.isGit ? context.facts.comparisonBaseRef : null;
+  const ref = comparison ?? (await resolveBestComparisonBaseRef(cwd, baseRef, context));
+  return resolveCommitComparisonRef(cwd, ref, context);
 }
 
 async function resolveMostAheadBaseRef(cwd: string, baseRef: string): Promise<string> {
@@ -1801,7 +1816,7 @@ async function resolveMostAheadBaseRef(cwd: string, baseRef: string): Promise<st
     return `origin/${normalizedBaseRef}`;
   }
   if (!hasLocal && !hasOrigin) {
-    throw new Error(`Base branch not found locally or on origin: ${normalizedBaseRef}`);
+    return resolveCommitComparisonRef(cwd, baseRef);
   }
 
   const { stdout } = await runGitCommand(
@@ -2239,8 +2254,8 @@ export async function getCheckoutSnapshotFacts(
   };
 }
 
-const PER_FILE_DIFF_MAX_BYTES = 1024 * 1024; // 1MB
-const TOTAL_DIFF_MAX_BYTES = 2 * 1024 * 1024; // 2MB
+export const PER_FILE_DIFF_MAX_BYTES = 1024 * 1024; // 1MB
+export const TOTAL_DIFF_MAX_BYTES = 2 * 1024 * 1024; // 2MB
 const CHECKOUT_DIFF_FRAME_HEADROOM_BYTES = 1024 * 1024;
 // Temporary until diffs load lazily per file. The Paseo relay's 32 MiB frame limit is
 // binding: string frames are encrypted and base64-encoded. Reserve 1 MiB plaintext for
@@ -2301,7 +2316,7 @@ async function isLikelyBinaryFile(absolutePath: string): Promise<boolean> {
   }
 }
 
-async function inspectUntrackedFile(
+export async function inspectUntrackedFile(
   cwd: string,
   relativePath: string,
 ): Promise<{ stat: FileStat; truncated: boolean }> {
@@ -2399,6 +2414,10 @@ export async function getCheckoutStatus(
   const baseRef = facts.resolvedBaseRef;
   const mainRepoRoot = facts.mainRepoRoot;
   const factsContext = { ...context, facts };
+  // Ref-derived counts skip the current base branch; the diff consumer still needs its ref.
+  const comparisonBaseRef = baseRef
+    ? await resolveQualifiedComparisonBaseRef(cwd, baseRef, factsContext).catch(() => null)
+    : null;
   const aheadBehind =
     baseRef && currentBranch
       ? await getAheadBehind(cwd, baseRef, currentBranch, factsContext)
@@ -2421,6 +2440,7 @@ export async function getCheckoutStatus(
       isDirty,
       baseRef: displayBaseRef ?? baseRef,
       baseRefExact: baseRef,
+      comparisonBaseRef,
       aheadBehind,
       upstreamRef,
       aheadOfOrigin,
@@ -2440,6 +2460,7 @@ export async function getCheckoutStatus(
     isDirty,
     baseRef: displayBaseRef,
     baseRefExact: baseRef,
+    comparisonBaseRef,
     aheadBehind,
     upstreamRef,
     aheadOfOrigin,
@@ -2804,6 +2825,7 @@ async function readCommitRenames(cwd: string, sha: string): Promise<ReadonlyMap<
  * binary-only (no textual hunks). Throws on git failure (e.g. an unknown sha),
  * which the caller maps to a typed checkout error.
  */
+// COMPAT(diffSubscription): added in v0.15.0, remove after 2027-05-06.
 export async function getCommitFileDiff({
   cwd,
   sha,
@@ -3570,6 +3592,21 @@ async function resolveCheckoutDiffRefs(
   if (compare.mode === "uncommitted") {
     return { baseRef: "HEAD", includeUntracked: true };
   }
+  const bestBaseRef = await resolveCheckoutDiffBase(cwd, compare, context);
+  if (!bestBaseRef) return null;
+  return {
+    baseRef: (await tryResolveMergeBase(cwd, bestBaseRef)) ?? bestBaseRef,
+    targetRef: compare.mode === "base" ? "HEAD" : undefined,
+    includeUntracked: compare.mode === "all",
+  };
+}
+
+export async function resolveCheckoutDiffBase(
+  cwd: string,
+  compare: CheckoutDiffCompare,
+  context?: CheckoutContext,
+): Promise<string | null> {
+  if (compare.mode === "uncommitted") return "HEAD";
   const { storedBaseRef, resolvedBaseRef } = await resolveBaseRefForCwd(cwd, context);
   const baseRef = resolveOperationBaseRef({
     storedBaseRef,
@@ -3579,17 +3616,12 @@ async function resolveCheckoutDiffRefs(
   if (!baseRef) {
     return null;
   }
-  const bestBaseRef = await resolveBestComparisonBaseRef(cwd, baseRef);
-  return {
-    baseRef: (await tryResolveMergeBase(cwd, bestBaseRef)) ?? bestBaseRef,
-    targetRef: compare.mode === "base" ? "HEAD" : undefined,
-    includeUntracked: compare.mode === "all",
-  };
+  return resolveBestComparisonBaseRef(cwd, baseRef);
 }
 
 // Git's working-tree diff ignores files absent from the index, even when they
 // exist in the comparison tree. Restore those entries only in a private index.
-async function prepareAllChangesIndex(cwd: string, baseRef: string) {
+export async function prepareAllChangesIndex(cwd: string, baseRef: string) {
   const untracked = await runGitCommand(["ls-files", "-z", "--others", "--exclude-standard"], {
     cwd,
     envOverlay: READ_ONLY_GIT_ENV,
@@ -3612,21 +3644,41 @@ async function prepareAllChangesIndex(cwd: string, baseRef: string) {
     entries += result.stdout;
   }
   if (!entries) return null;
-  const directory = await mkdtemp(resolve(tmpdir(), "paseo-all-changes-"));
+  const snapshot = await snapshotComparisonIndex(cwd);
+  try {
+    await runGitCommand(["update-index", "-z", "--index-info"], {
+      cwd,
+      input: entries,
+      envOverlay: { ...READ_ONLY_GIT_ENV, GIT_INDEX_FILE: snapshot.indexFile },
+    });
+    return snapshot;
+  } catch (error) {
+    await snapshot.cleanup();
+    throw error;
+  }
+}
+
+// write-tree rewrites its index. A private snapshot prevents an index read from
+// invalidating the observation that requested it.
+export async function snapshotComparisonIndex(cwd: string) {
+  const directory = await mkdtemp(resolve(tmpdir(), "paseo-comparison-index-"));
   const indexFile = resolve(directory, "index");
   try {
     const index = await runGitCommand(["rev-parse", "--git-path", "index"], {
       cwd,
       envOverlay: READ_ONLY_GIT_ENV,
     });
-    const indexPath = resolveGitRevParsePath(cwd, index.stdout);
-    if (!indexPath) throw new Error("Git did not return an index path");
-    await copyFile(indexPath, indexFile);
-    await runGitCommand(["update-index", "-z", "--index-info"], {
-      cwd,
-      input: entries,
-      envOverlay: { ...READ_ONLY_GIT_ENV, GIT_INDEX_FILE: indexFile },
-    });
+    const source = resolveGitRevParsePath(cwd, index.stdout);
+    if (!source) throw new Error("Git did not return an index path");
+    try {
+      await copyFile(source, indexFile);
+    } catch (error) {
+      if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error;
+      await runGitCommand(["read-tree", "--empty"], {
+        cwd,
+        envOverlay: { ...READ_ONLY_GIT_ENV, GIT_INDEX_FILE: indexFile },
+      });
+    }
     return { indexFile, cleanup: () => rm(directory, { recursive: true, force: true }) };
   } catch (error) {
     await rm(directory, { recursive: true, force: true });

packages/server/src/utils/git-file-contents.ts

@@ -1,4 +1,4 @@
-import { runGitCommandBytes } from "./run-git-command.js";
+import { runGitCommandBytes, runGitCommand } from "./run-git-command.js";
 
 // Match the existing git-show ceiling. Oversized batches use bounded per-file reads.
 const MAX_CONTENT_BYTES = 20 * 1024 * 1024;
@@ -46,3 +46,17 @@ export async function readGitFileContents(
   if (offset !== result.stdout.length) throw new Error("Unexpected git object output");
   return contents;
 }
+
+export async function readGitFileContentAtRef(
+  cwd: string,
+  ref: string,
+  path: string,
+): Promise<string | null> {
+  const result = await runGitCommand(["show", `${ref}:${path}`], {
+    cwd,
+    envOverlay: { GIT_OPTIONAL_LOCKS: "0", LC_ALL: "C" },
+    acceptExitCodes: [0, 128],
+  });
+  if (result.exitCode !== 0 || result.truncated) return null;
+  return result.stdout;
+}

packages/server/src/utils/run-git-command.ts

@@ -20,7 +20,7 @@ import {
 } from "./git-process-scheduler.js";
 
 const DEFAULT_TIMEOUT_MS = 30_000;
-const DEFAULT_MAX_OUTPUT_BYTES = 20 * 1024 * 1024; // 20MB
+export const DEFAULT_MAX_OUTPUT_BYTES = 20 * 1024 * 1024; // 20MB
 const DEFAULT_STDERR_LIMIT = 2048;
 
 let gitProcessScheduler = new GitProcessScheduler(resolveGitProcessPolicy({ env: process.env }));