Compare commits

...
Author SHA1 Message Date
paseo-bot[bot] 3652049de4 fix(server): preserve workspace snapshots through suspension
Drop queued refreshes and withhold publication when a folder disappears during an active read. Wait for that read before restoring observation so its stale result cannot overwrite the recovered snapshot.
2026-11-01 11:13:59 +00:00
paseo-bot[bot]andDominic Anaeto c13e5f0310 fix(server): keep missing workspaces suspended during refresh
Admit Git refreshes only after the folder has returned. Cover missing-folder polling, explicit refreshes, and restoration with real Git and the Linux observer.

Co-authored-by: Dominic Anaeto <dominic.anaeto@gmail.com>
2026-11-01 10:53:54 +00:00
Dominic Anaeto 5158830f2a fix(server): stop polling git for a workspace whose folder is gone
When a workspace's folder disappears together with its project (the
repository deleted, a volume unmounted), reconciliation leaves the
workspace registered, and the daemon kept polling it: a failed git
command and a warning about once a minute, and a working-tree watcher
retry every minute or two, for as long as the daemon ran.

The degraded poll now checks that the workspace folder exists before
refreshing. A workspace whose folder is gone is suspended once: it is
detached from its repository and working-tree targets, its timers stop,
and one warning is logged. A worktree that shared a repository target
hands the repository's polling and background fetch to a surviving
worktree. Nothing is archived. A working-tree watch that an open diff
still holds skips its consumer notifications and watcher recovery while
its folder is missing.

While suspended, the workspace runs no git. A stat every 60 seconds, a
new subscription, a forced snapshot or a refresh request checks whether
the folder is back; when it is, the workspace reloads its git facts,
refreshes, and rebuilds its observation.

Fixes #7856
2026-11-01 00:19:09 -04:00
4 changed files with 891 additions and 21 deletions
@@ -1,3 +1,4 @@
import { flushGitCommandTrace } from "../utils/git-command-trace.js";
import { createWorkspaceRuntimeStore } from "./workspace-runtime-store/index.js";
import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
@@ -730,3 +731,103 @@ test("Git event identity retains a symlink's literal target before canonicalizin
rmSync(root, { recursive: true, force: true });
}
});
test.skipIf(process.platform !== "linux")(
"missing folders stop degraded Git polling and explicit refreshes, then recover",
async () => {
const artifacts = path.resolve(".artifacts");
mkdirSync(artifacts, { recursive: true });
const root = mkdtempSync(path.join(artifacts, "workspace-git-missing-"));
const cwd = path.join(root, "repo");
const initializeRepo = (branch: string) => {
mkdirSync(cwd);
execFileSync("git", ["init", "-b", branch], { cwd, stdio: "pipe" });
writeFileSync(path.join(cwd, "README.md"), "fixture\n");
execFileSync("git", ["add", "."], { cwd, stdio: "pipe" });
execFileSync(
"git",
[
"-c",
"user.name=Test",
"-c",
"user.email=test@example.invalid",
"-c",
"commit.gpgsign=false",
"commit",
"-m",
"fixture",
],
{ cwd, stdio: "pipe" },
);
};
initializeRepo("main");
// Exceed the real Linux observer's directory limit to enter degraded polling.
// No watcher errors or Git responses are injected.
for (let index = 0; index < 5100; index++) {
mkdirSync(path.join(cwd, `generated-${index}`));
}
const trace = path.join(root, "git.jsonl");
writeFileSync(trace, "");
const previousTrace = process.env.PASEO_GIT_TRACE_FILE;
process.env.PASEO_GIT_TRACE_FILE = trace;
let degraded = false;
let suspended = false;
const logger = pino(
{ level: "warn" },
{
write(line) {
const message = JSON.parse(line).msg;
if (message === "Failed to start working tree watcher; using degraded polling")
degraded = true;
if (
message ===
"Workspace folder no longer exists; suspended git observation until it is back"
)
suspended = true;
},
},
);
const service = new WorkspaceGitServiceImpl({ logger, paseoHome: path.join(root, "home") });
const subscription = service.registerWorkspace({ cwd }, () => {});
const commandCount = async () => {
await flushGitCommandTrace();
return readFileSync(trace, "utf8").trim().split("\n").filter(Boolean).length;
};
try {
await expect
.poll(() => degraded && service.getMetrics().workspaceRefreshInFlightCount === 0, {
timeout: 15_000,
})
.toBe(true);
rmSync(cwd, { recursive: true, force: true });
await expect.poll(() => suspended, { timeout: 15_000 }).toBe(true);
await expect.poll(() => service.getMetrics().workspaceRefreshInFlightCount).toBe(0);
const commandsWhileMissing = await commandCount();
await service.getSnapshot(cwd, {
force: true,
includeForge: false,
reason: "refresh-button",
});
await service.refresh(cwd);
expect(await commandCount()).toBe(commandsWhileMissing);
await new Promise((resolve) => setTimeout(resolve, 16_000));
expect(await commandCount()).toBe(commandsWhileMissing);
initializeRepo("restored");
const snapshot = await service.getSnapshot(cwd, {
force: true,
includeForge: false,
reason: "refresh-button",
});
expect(snapshot.git).toMatchObject({ isGit: true, currentBranch: "restored" });
await expect.poll(() => service.getMetrics().repositoryWorkspaceLinkCount).toBe(1);
} finally {
subscription.unsubscribe();
await service.dispose();
if (previousTrace === undefined) delete process.env.PASEO_GIT_TRACE_FILE;
else process.env.PASEO_GIT_TRACE_FILE = previousTrace;
rmSync(root, { recursive: true, force: true });
}
},
60_000,
);
@@ -244,6 +244,7 @@ function createService(
filterEvents: (events) => events,
verify: vi.fn(async () => {}),
})),
workspaceFolderExists: vi.fn(async () => true),
...overrides,
} as never,
});
@@ -1,5 +1,7 @@
import { createWorkspaceRuntimeStore } from "./workspace-runtime-store/index.js";
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import { existsSync } from "node:fs";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path, { join } from "node:path";
import type pino from "pino";
@@ -10,11 +12,14 @@ import type {
PullRequestStatusResult,
} from "../utils/checkout-git.js";
import {
BACKGROUND_GIT_FETCH_INTERVAL_MS,
WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS,
WORKSPACE_GIT_OBSERVATION_SETUP_CONCURRENCY,
WORKSPACE_GIT_REFRESH_CONCURRENCY,
WORKSPACE_GIT_WATCHER_SUBSCRIBE_TIMEOUT_MS,
WorkspaceGitServiceImpl,
type WorkspaceGitRuntimeSnapshot,
workspaceFolderExists,
} from "./workspace-git-service.js";
const REPO_CWD = path.resolve("/tmp/repo");
@@ -23,6 +28,7 @@ function createLogger() {
const logger = {
child: () => logger,
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
};
return logger;
@@ -251,7 +257,7 @@ function createGitHubServiceStub(): ForgeService {
interface CreateServiceTestOptions {
subscribe?: ReturnType<typeof vi.fn>;
getCheckoutStatus?: ReturnType<typeof vi.fn>;
getCheckoutSnapshotFacts?: ReturnType<typeof vi.fn>;
getCheckoutSnapshotFacts?: (cwd: string) => Promise<CheckoutSnapshotFacts>;
getCheckoutShortstat?: ReturnType<typeof vi.fn>;
getCheckoutWorktreeState?: ReturnType<typeof vi.fn>;
getPullRequestStatus?: ReturnType<typeof vi.fn>;
@@ -261,6 +267,7 @@ interface CreateServiceTestOptions {
runGitFetch?: ReturnType<typeof vi.fn>;
runGitCommand?: ReturnType<typeof vi.fn>;
getWorkspaceGitSelfHealPhaseMs?: (cwd: string) => number;
workspaceFolderExists?: (cwd: string) => Promise<boolean>;
now?: () => Date;
}
@@ -294,6 +301,7 @@ function buildDefaultTestServiceDeps() {
signal: null,
})),
getWorkspaceGitSelfHealPhaseMs: vi.fn(() => 30_000),
workspaceFolderExists: vi.fn(async () => true),
createWatcherLivenessCanary: vi.fn(() => ({
path: "",
filterEvents: (events) => events,
@@ -324,6 +332,145 @@ function createService(options?: CreateServiceTestOptions) {
});
}
type WatchCallback = (error: Error | null, events: Array<{ path: string; type: "update" }>) => void;
interface FolderBackedGitOptions {
facts?: (cwd: string) => CheckoutSnapshotFacts;
unwatchableDirectories?: string[];
heldWatch?: { directory: string; release: Promise<void> };
}
function createFolderBackedGit(options?: FolderBackedGitOptions) {
const watchCallbacks = new Map<string, WatchCallback>();
const subscribe = vi.fn(async (directory: string, callback: WatchCallback) => {
if (!existsSync(directory) || options?.unwatchableDirectories?.includes(directory)) {
throw new Error(`Cannot watch ${directory}`);
}
if (options?.heldWatch?.directory === directory) {
await options.heldWatch.release;
}
watchCallbacks.set(directory, callback);
return createAsyncSubscription();
});
const getCheckoutSnapshotFacts = vi.fn(
async (cwd: string): Promise<CheckoutSnapshotFacts> =>
existsSync(cwd) ? (options?.facts ?? createCheckoutSnapshotFacts)(cwd) : { isGit: false },
);
const getCheckoutStatus = vi.fn(async (cwd: string) =>
existsSync(cwd) ? createCheckoutStatus(cwd) : ({ isGit: false } as const),
);
const getCheckoutShortstat = vi.fn(async (_cwd: string) => ({ additions: 1, deletions: 0 }));
const getCheckoutWorktreeState = vi.fn(async (_cwd: string) => ({
isDirty: false,
diffStat: { additions: 1, deletions: 0 },
}));
const runGitCommand = vi.fn(async (args: string[], runOptions: { cwd: string }) => ({
stdout: args[0] === "rev-parse" ? `${runOptions.cwd}\n` : "",
stderr: "",
truncated: false,
exitCode: 0,
signal: null,
}));
const runGitFetch = vi.fn(async (_cwd: string) => ({ changes: [], error: null }));
const callsIn = (fake: ReturnType<typeof vi.fn>, cwd: string) =>
fake.mock.calls.filter(([calledCwd]) => calledCwd === cwd).length;
return {
deps: {
subscribe,
getCheckoutSnapshotFacts,
getCheckoutStatus,
getCheckoutShortstat,
getCheckoutWorktreeState,
runGitCommand,
runGitFetch,
workspaceFolderExists: async (cwd: string) => existsSync(cwd),
now: () => new Date(),
},
watchCallbackFor(directory: string): WatchCallback {
const callback = watchCallbacks.get(directory);
if (!callback) {
throw new Error(`No watcher was started for ${directory}`);
}
return callback;
},
watchAttemptsFor: (directory: string) => callsIn(subscribe, directory),
pollsOf: (cwd: string) => callsIn(getCheckoutShortstat, cwd),
fetchesIn: (cwd: string) => callsIn(runGitFetch, cwd),
gitCommandsIn(cwd: string): number {
const cwdCommands = [
getCheckoutSnapshotFacts,
getCheckoutStatus,
getCheckoutShortstat,
getCheckoutWorktreeState,
runGitFetch,
].reduce((count, fake) => count + callsIn(fake, cwd), 0);
const commandsRunInCwd = runGitCommand.mock.calls.filter(
([, runOptions]) => runOptions.cwd === cwd,
).length;
return cwdCommands + commandsRunInCwd;
},
};
}
function linkedWorktreeFacts(gitCommonDir: string) {
return (cwd: string): CheckoutSnapshotFacts => ({
...createCheckoutSnapshotFacts(cwd),
absoluteGitDir: join(gitCommonDir, "worktrees", path.basename(cwd)),
gitCommonDir,
});
}
const SETTLE_ATTEMPTS = 1_000;
async function settle(assertion: () => void): Promise<void> {
let lastFailure: unknown = null;
for (let attempt = 0; attempt < SETTLE_ATTEMPTS; attempt += 1) {
try {
assertion();
return;
} catch (failure) {
lastFailure = failure;
}
await vi.advanceTimersByTimeAsync(0);
}
throw lastFailure;
}
async function waitForRefreshesToSettle(service: WorkspaceGitServiceImpl): Promise<void> {
await settle(() => {
expect(service.getMetrics().workspaceRefreshInFlightCount).toBe(0);
});
}
async function waitForObservationToSettle(
service: WorkspaceGitServiceImpl,
repositoryWorkspaceLinkCount: number,
): Promise<void> {
await settle(() => {
const metrics = service.getMetrics();
expect(metrics.repositoryWorkspaceLinkCount).toBe(repositoryWorkspaceLinkCount);
expect(metrics.workspaceObservationSetupInFlightCount).toBe(0);
expect(metrics.workspaceRefreshInFlightCount).toBe(0);
});
}
async function advanceThroughDegradedPolls(service: WorkspaceGitServiceImpl): Promise<void> {
for (const delayMs of [5_000, 10_000, 20_000, 40_000, 60_000, 60_000]) {
await vi.advanceTimersByTimeAsync(delayMs);
await waitForRefreshesToSettle(service);
}
}
async function advanceThroughWorkingTreeRecoveryAttempts(
service: WorkspaceGitServiceImpl,
): Promise<void> {
for (const delayMs of [30_000, 60_000, 120_000]) {
await vi.advanceTimersByTimeAsync(delayMs);
await waitForRefreshesToSettle(service);
}
}
describe("WorkspaceGitServiceImpl", () => {
beforeEach(() => {
vi.useFakeTimers();
@@ -1553,3 +1700,460 @@ describe("WorkspaceGitServiceImpl", () => {
service.dispose();
});
});
describe("WorkspaceGitServiceImpl with workspace folders that no longer exist", () => {
const temporaryFolders: string[] = [];
async function createTemporaryFolder(): Promise<string> {
const folder = await mkdtemp(join(os.tmpdir(), "paseo-workspace-git-"));
temporaryFolders.push(folder);
return folder;
}
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(async () => {
vi.useRealTimers();
await Promise.all(
temporaryFolders.splice(0).map((folder) => rm(folder, { recursive: true, force: true })),
);
});
test("polling stops for a workspace whose folder no longer exists", async () => {
const workspace = await createTemporaryFolder();
const git = createFolderBackedGit();
const service = createService(git.deps);
const subscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
await rm(workspace, { recursive: true, force: true });
const gitCommandsAtDeletion = git.gitCommandsIn(workspace);
await advanceThroughDegradedPolls(service);
expect(git.gitCommandsIn(workspace)).toBe(gitCommandsAtDeletion);
expect(service.getMetrics()).toMatchObject({
repositoryTargetCount: 0,
workingTreeWatchTargetCount: 0,
});
expect(vi.getTimerCount()).toBe(1);
subscription.unsubscribe();
await service.dispose();
});
test("a deleted worktree leaves its shared repository polling and fetch to the surviving worktree", async () => {
const gitCommonDir = join(await createTemporaryFolder(), ".git");
const deletedWorktree = await createTemporaryFolder();
const survivingWorktree = await createTemporaryFolder();
const git = createFolderBackedGit({ facts: linkedWorktreeFacts(gitCommonDir) });
let survivingWorktreeEdits = 0;
git.deps.getCheckoutShortstat.mockImplementation(async (cwd: string) => ({
additions: cwd === survivingWorktree ? ++survivingWorktreeEdits : 1,
deletions: 0,
}));
const service = createService(git.deps);
const deletedSubscription = service.registerWorkspace({ cwd: deletedWorktree }, vi.fn());
await waitForObservationToSettle(service, 1);
const survivingSubscription = service.registerWorkspace({ cwd: survivingWorktree }, vi.fn());
await waitForObservationToSettle(service, 2);
await rm(deletedWorktree, { recursive: true, force: true });
await vi.advanceTimersByTimeAsync(60_000);
await waitForRefreshesToSettle(service);
const deletedWorktreeCommands = git.gitCommandsIn(deletedWorktree);
for (let tick = 0; tick < 3; tick += 1) {
const survivingWorktreePolls = git.pollsOf(survivingWorktree);
await vi.advanceTimersByTimeAsync(5_000);
await settle(() => {
expect(git.pollsOf(survivingWorktree)).toBe(survivingWorktreePolls + 1);
});
}
expect(git.gitCommandsIn(deletedWorktree)).toBe(deletedWorktreeCommands);
const fetchesBefore = git.deps.runGitFetch.mock.calls.length;
await vi.advanceTimersByTimeAsync(2 * BACKGROUND_GIT_FETCH_INTERVAL_MS);
expect(git.deps.runGitFetch.mock.calls.slice(fetchesBefore).map(([cwd]) => cwd)).toEqual([
survivingWorktree,
survivingWorktree,
]);
expect(git.gitCommandsIn(deletedWorktree)).toBe(deletedWorktreeCommands);
deletedSubscription.unsubscribe();
survivingSubscription.unsubscribe();
await service.dispose();
});
test("deleting every worktree of a repository stops its polling and background fetch", async () => {
const gitCommonDir = join(await createTemporaryFolder(), ".git");
const firstWorktree = await createTemporaryFolder();
const secondWorktree = await createTemporaryFolder();
const git = createFolderBackedGit({ facts: linkedWorktreeFacts(gitCommonDir) });
const service = createService(git.deps);
const firstSubscription = service.registerWorkspace({ cwd: firstWorktree }, vi.fn());
await waitForObservationToSettle(service, 1);
const secondSubscription = service.registerWorkspace({ cwd: secondWorktree }, vi.fn());
await waitForObservationToSettle(service, 2);
await rm(firstWorktree, { recursive: true, force: true });
await rm(secondWorktree, { recursive: true, force: true });
const commandsAtDeletion = [
git.gitCommandsIn(firstWorktree),
git.gitCommandsIn(secondWorktree),
];
await advanceThroughDegradedPolls(service);
expect([git.gitCommandsIn(firstWorktree), git.gitCommandsIn(secondWorktree)]).toEqual(
commandsAtDeletion,
);
expect(service.getMetrics()).toMatchObject({
repositoryTargetCount: 0,
workingTreeWatchTargetCount: 0,
});
expect(vi.getTimerCount()).toBe(2);
firstSubscription.unsubscribe();
secondSubscription.unsubscribe();
await service.dispose();
});
test("working tree fallback polling and watcher recovery stop for a workspace whose folder no longer exists", async () => {
const workspace = await createTemporaryFolder();
await mkdir(join(workspace, ".git"));
const git = createFolderBackedGit();
const service = createService(git.deps);
const subscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
await rm(workspace, { recursive: true, force: true });
git.watchCallbackFor(workspace)(new Error("watched directory was deleted"), []);
await vi.advanceTimersByTimeAsync(1_000);
await waitForRefreshesToSettle(service);
const commandsAfterWatcherError = git.gitCommandsIn(workspace);
const watchAttemptsAfterWatcherError = git.watchAttemptsFor(workspace);
await advanceThroughDegradedPolls(service);
expect(git.gitCommandsIn(workspace)).toBe(commandsAfterWatcherError);
expect(git.watchAttemptsFor(workspace)).toBe(watchAttemptsAfterWatcherError);
expect(vi.getTimerCount()).toBe(1);
subscription.unsubscribe();
await service.dispose();
});
test("a workspace whose folder comes back resumes polling on its next subscription", async () => {
const workspace = await createTemporaryFolder();
const git = createFolderBackedGit();
const service = createService(git.deps);
const firstSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
await rm(workspace, { recursive: true, force: true });
const commandsWhileMissing = git.gitCommandsIn(workspace);
await advanceThroughDegradedPolls(service);
expect(git.gitCommandsIn(workspace)).toBe(commandsWhileMissing);
await mkdir(workspace);
const secondSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
const pollsAfterResume = git.pollsOf(workspace);
await vi.advanceTimersByTimeAsync(5_000);
await waitForRefreshesToSettle(service);
expect(git.pollsOf(workspace)).toBe(pollsAfterResume + 1);
firstSubscription.unsubscribe();
secondSubscription.unsubscribe();
await service.dispose();
});
test("a suspended workspace costs no git while its folder is missing, even when a refresh is requested", async () => {
const workspace = await createTemporaryFolder();
const git = createFolderBackedGit();
const service = createService(git.deps);
const subscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
await rm(workspace, { recursive: true, force: true });
await advanceThroughDegradedPolls(service);
const commandsWhileMissing = git.gitCommandsIn(workspace);
for (let probe = 0; probe < 10; probe += 1) {
await vi.advanceTimersByTimeAsync(WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS);
await waitForRefreshesToSettle(service);
}
service.scheduleRefreshForCwd(workspace);
service.onWorkspaceStateMayHaveChanged(workspace);
await vi.advanceTimersByTimeAsync(5_000);
await waitForRefreshesToSettle(service);
expect(git.gitCommandsIn(workspace)).toBe(commandsWhileMissing);
expect(vi.getTimerCount()).toBe(1);
subscription.unsubscribe();
await service.dispose();
});
test("a suspended worktree resumes polling and its repository fetch once its folder is back, without a new subscription", async () => {
const gitCommonDir = join(await createTemporaryFolder(), ".git");
const worktree = await createTemporaryFolder();
const git = createFolderBackedGit({ facts: linkedWorktreeFacts(gitCommonDir) });
const service = createService(git.deps);
const subscription = service.registerWorkspace({ cwd: worktree }, vi.fn());
await waitForObservationToSettle(service, 1);
await rm(worktree, { recursive: true, force: true });
await advanceThroughDegradedPolls(service);
await waitForObservationToSettle(service, 0);
await mkdir(worktree);
const fetchesBeforeRestore = git.fetchesIn(worktree);
await vi.advanceTimersByTimeAsync(WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS);
await waitForObservationToSettle(service, 1);
expect(git.fetchesIn(worktree)).toBe(fetchesBeforeRestore + 1);
const pollsAfterRestore = git.pollsOf(worktree);
await vi.advanceTimersByTimeAsync(5_000);
await waitForRefreshesToSettle(service);
expect(git.pollsOf(worktree)).toBe(pollsAfterRestore + 1);
subscription.unsubscribe();
await service.dispose();
});
test("a forced snapshot resumes a suspended workspace whose folder is back", async () => {
const workspace = await createTemporaryFolder();
const git = createFolderBackedGit();
const service = createService(git.deps);
const subscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
await rm(workspace, { recursive: true, force: true });
await advanceThroughDegradedPolls(service);
await waitForObservationToSettle(service, 0);
await mkdir(workspace);
const snapshot = await service.getSnapshot(workspace, {
force: true,
reason: "refresh-button",
});
await waitForObservationToSettle(service, 1);
expect(snapshot.git.isGit).toBe(true);
subscription.unsubscribe();
await service.dispose();
});
test("a resumed workspace rebuilds observation from fresh facts, not the non-git facts cached while its folder was gone", async () => {
const workspace = await createTemporaryFolder();
const git = createFolderBackedGit();
const service = createService(git.deps);
const firstSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
await rm(workspace, { recursive: true, force: true });
service.onWorkspaceStateMayHaveChanged(workspace);
await vi.advanceTimersByTimeAsync(1_000);
await waitForRefreshesToSettle(service);
expect((await service.getSnapshot(workspace)).git.isGit).toBe(false);
await advanceThroughDegradedPolls(service);
await waitForObservationToSettle(service, 0);
await mkdir(workspace);
const secondSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
expect((await service.getSnapshot(workspace)).git.isGit).toBe(true);
firstSubscription.unsubscribe();
secondSubscription.unsubscribe();
await service.dispose();
});
test("resuming waits for an observation setup that was still running when the folder vanished", async () => {
const workspace = await createTemporaryFolder();
const gitDir = join(workspace, ".git");
await mkdir(gitDir);
const metadataWatchRelease = createDeferred<void>();
const git = createFolderBackedGit({
unwatchableDirectories: [workspace],
heldWatch: { directory: gitDir, release: metadataWatchRelease.promise },
});
const service = createService(git.deps);
const firstSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await settle(() => {
expect(git.watchAttemptsFor(gitDir)).toBe(1);
});
await rm(workspace, { recursive: true, force: true });
await vi.advanceTimersByTimeAsync(5_000);
await settle(() => {
expect(service.getMetrics()).toMatchObject({
repositoryTargetCount: 0,
workingTreeWatchTargetCount: 0,
});
});
await mkdir(workspace);
const secondSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
metadataWatchRelease.resolve();
await waitForObservationToSettle(service, 1);
expect(service.getMetrics().workingTreeWatchTargetCount).toBe(1);
firstSubscription.unsubscribe();
secondSubscription.unsubscribe();
await service.dispose();
});
test("an open working tree watch neither notifies nor recovers while its folder is missing, and picks up once it is back", async () => {
const workspace = await createTemporaryFolder();
await mkdir(join(workspace, ".git"));
const git = createFolderBackedGit();
const service = createService(git.deps);
const workspaceSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn());
await waitForObservationToSettle(service, 1);
const diffListener = vi.fn();
const diffWatch = await service.requestWorkingTreeWatch(workspace, diffListener);
await rm(workspace, { recursive: true, force: true });
git.watchCallbackFor(workspace)(new Error("watched directory was deleted"), []);
await vi.advanceTimersByTimeAsync(1_000);
await waitForRefreshesToSettle(service);
const commandsAfterWatcherError = git.gitCommandsIn(workspace);
const notificationsAfterWatcherError = diffListener.mock.calls.length;
const watchAttemptsAfterWatcherError = git.watchAttemptsFor(workspace);
await advanceThroughWorkingTreeRecoveryAttempts(service);
expect(git.gitCommandsIn(workspace)).toBe(commandsAfterWatcherError);
expect(diffListener.mock.calls.length).toBe(notificationsAfterWatcherError);
expect(git.watchAttemptsFor(workspace)).toBe(watchAttemptsAfterWatcherError);
expect(service.getMetrics().workingTreeWatchTargetCount).toBe(1);
await mkdir(join(workspace, ".git"), { recursive: true });
await vi.advanceTimersByTimeAsync(60_000);
await waitForRefreshesToSettle(service);
expect(diffListener.mock.calls.length).toBe(notificationsAfterWatcherError + 1);
await vi.advanceTimersByTimeAsync(240_000);
await waitForRefreshesToSettle(service);
expect(git.watchAttemptsFor(workspace)).toBe(watchAttemptsAfterWatcherError + 1);
const notificationsAfterResubscribe = diffListener.mock.calls.length;
await vi.advanceTimersByTimeAsync(120_000);
await waitForRefreshesToSettle(service);
expect(diffListener.mock.calls.length).toBe(notificationsAfterResubscribe);
diffWatch.unsubscribe();
workspaceSubscription.unsubscribe();
await service.dispose();
});
test("a refresh crossing suspension preserves the cached snapshot and drops queued work", async () => {
const workspace = await createTemporaryFolder();
const git = createFolderBackedGit();
const started = createDeferred<void>();
const release = createDeferred<void>();
let holdNextRead = false;
let factsReads = 0;
const service = createService({
...git.deps,
getCheckoutSnapshotFacts: async (cwd: string) => {
factsReads++;
if (holdNextRead) {
holdNextRead = false;
started.resolve();
await release.promise;
}
return git.deps.getCheckoutSnapshotFacts(cwd);
},
});
const updates: WorkspaceGitRuntimeSnapshot[] = [];
const subscription = service.registerWorkspace({ cwd: workspace }, (snapshot) =>
updates.push(snapshot),
);
await waitForObservationToSettle(service, 1);
const cached = await service.getSnapshot(workspace);
holdNextRead = true;
const active = service.getSnapshot(workspace, {
force: true,
includeForge: false,
reason: "overlapping-refresh",
});
await started.promise;
const queued = service.getSnapshot(workspace, {
force: true,
includeForge: true,
reason: "queued-refresh",
});
await rm(workspace, { recursive: true, force: true });
await vi.advanceTimersByTimeAsync(5_000);
await settle(() => expect(service.getMetrics().repositoryWorkspaceLinkCount).toBe(0));
const readsAtSuspension = factsReads;
const updatesAtSuspension = updates.length;
release.resolve();
expect(await active).toEqual(cached);
expect(await queued).toEqual(cached);
expect(await service.getSnapshot(workspace)).toEqual(cached);
expect(updates).toHaveLength(updatesAtSuspension);
expect(factsReads).toBe(readsAtSuspension);
subscription.unsubscribe();
await service.dispose();
});
test("restoration settles a suspended refresh before reloading facts", async () => {
const workspace = await createTemporaryFolder();
const git = createFolderBackedGit();
const staleFacts = createDeferred<CheckoutSnapshotFacts>();
const started = createDeferred<void>();
let holdNextRead = false;
const service = createService({
...git.deps,
getCheckoutSnapshotFacts: async (cwd: string) => {
if (holdNextRead) {
holdNextRead = false;
started.resolve();
return staleFacts.promise;
}
return git.deps.getCheckoutSnapshotFacts(cwd);
},
});
const updates: WorkspaceGitRuntimeSnapshot[] = [];
const subscription = service.registerWorkspace({ cwd: workspace }, (snapshot) =>
updates.push(snapshot),
);
await waitForObservationToSettle(service, 1);
const cached = await service.getSnapshot(workspace);
holdNextRead = true;
const active = service.getSnapshot(workspace, {
force: true,
includeForge: false,
reason: "overlapping-refresh",
});
await started.promise;
await rm(workspace, { recursive: true, force: true });
await vi.advanceTimersByTimeAsync(5_000);
await settle(() => expect(service.getMetrics().repositoryWorkspaceLinkCount).toBe(0));
const updatesAtSuspension = updates.length;
await mkdir(workspace);
const restored = service.getSnapshot(workspace, {
force: true,
includeForge: false,
reason: "folder-returned",
});
await vi.advanceTimersByTimeAsync(0);
staleFacts.resolve({ isGit: false });
expect(await active).toEqual(cached);
expect((await restored).git.isGit).toBe(true);
expect(updates.slice(updatesAtSuspension).every((snapshot) => snapshot.git.isGit)).toBe(true);
await waitForObservationToSettle(service, 1);
subscription.unsubscribe();
await service.dispose();
});
test("workspaceFolderExists reports a missing folder and a path under a file as missing", async () => {
const folder = await createTemporaryFolder();
const file = join(folder, "file.txt");
await writeFile(file, "");
expect(await workspaceFolderExists(folder)).toBe(true);
expect(await workspaceFolderExists(join(folder, "missing"))).toBe(false);
expect(await workspaceFolderExists(join(file, "nested"))).toBe(false);
});
});
@@ -1,5 +1,5 @@
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { readFile, stat } from "node:fs/promises";
import { basename, dirname, join, resolve } from "node:path";
import { LRUCache } from "lru-cache";
import { CheckoutDiffCache } from "./checkout-diff-cache.js";
@@ -74,7 +74,7 @@ import { checkoutLiteFromGitSnapshot } from "./workspace-registry-model.js";
import { createWatcherLivenessCanary } from "./watcher-liveness-canary.js";
const WORKSPACE_GIT_WATCH_DEBOUNCE_MS = 1_000;
const BACKGROUND_GIT_FETCH_INTERVAL_MS = 180_000;
export const BACKGROUND_GIT_FETCH_INTERVAL_MS = 180_000;
const FETCH_METADATA_ECHO_TTL_MS = 5_000;
export const WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS = 60_000;
const FORGE_PR_STATUS_POLL_FAST_INTERVAL_MS = 20_000;
@@ -384,6 +384,7 @@ interface WorkspaceGitServiceDependencies {
runGitCommand: typeof runGitCommand;
getWorkspaceGitSelfHealPhaseMs: typeof getWorkspaceGitObservationReensurePhaseMs;
createWatcherLivenessCanary: typeof createWatcherLivenessCanary;
workspaceFolderExists: (cwd: string) => Promise<boolean>;
now: () => Date;
}
@@ -435,6 +436,8 @@ interface WorkspaceGitTarget {
repoGitRoot: string | null;
observationSetupPromise: Promise<void> | null;
observationSetupComplete: boolean;
folderMissing: boolean;
folderRestorePromise: Promise<void> | null;
closed: boolean;
}
@@ -571,10 +574,21 @@ function buildDefaultWorkspaceGitServiceDeps(
runGitCommand,
getWorkspaceGitSelfHealPhaseMs: getWorkspaceGitObservationReensurePhaseMs,
createWatcherLivenessCanary,
workspaceFolderExists,
now: () => new Date(),
};
}
export async function workspaceFolderExists(cwd: string): Promise<boolean> {
try {
await stat(cwd);
return true;
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
return code !== "ENOENT" && code !== "ENOTDIR";
}
}
function resolveWorkspaceGitServiceDeps(
subscribe: SubscribeToFileChanges,
deps: Partial<WorkspaceGitServiceDependencies> | undefined,
@@ -660,6 +674,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
const cwd = resolve(params.cwd);
const target = this.ensureWorkspaceTarget(cwd);
target.listeners.add(listener);
if (target.folderMissing) {
void this.resumeWorkspaceIfFolderRestored(target);
}
if (target.listeners.size === 1) {
this.startWorkspaceSubscriptionTimers(target);
}
@@ -751,6 +768,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
if (!request.force && target.latestSnapshot) {
return target.latestSnapshot;
}
if (target.folderMissing) {
await this.resumeWorkspaceIfFolderRestored(target);
}
return this.requestWorkspaceSnapshot(target, request);
}
@@ -972,6 +992,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
this.assertNotDisposed();
cwd = resolve(cwd);
const target = this.ensureWorkspaceTarget(cwd);
if (target.folderMissing) {
await this.resumeWorkspaceIfFolderRestored(target);
}
await this.refreshWorkspaceTarget(target, {
force: false,
refreshStructure: true,
@@ -1005,9 +1028,14 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
this.assertNotDisposed();
cwd = resolve(cwd);
const target = this.workspaceTargets.get(cwd);
if (target) {
this.scheduleWorkspaceRefresh(target, { queueIfBusy: false });
if (!target) {
return;
}
if (target.folderMissing) {
void this.resumeWorkspaceIfFolderRestored(target);
return;
}
this.scheduleWorkspaceRefresh(target, { queueIfBusy: false });
}
onWorkspaceStateMayHaveChanged(cwd: string): void {
@@ -1018,6 +1046,10 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
return;
}
this.invalidateForge(normalizedCwd);
if (target.folderMissing) {
void this.resumeWorkspaceIfFolderRestored(target);
return;
}
this.scheduleWorkspaceRefresh(target, {
force: true,
includeForge: true,
@@ -1227,6 +1259,8 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
repoGitRoot: null,
observationSetupPromise: null,
observationSetupComplete: false,
folderMissing: false,
folderRestorePromise: null,
closed: false,
};
@@ -1358,6 +1392,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
private isActiveObservedWorkspaceTarget(target: WorkspaceGitTarget): boolean {
return (
!target.closed &&
!target.folderMissing &&
target.listeners.size > 0 &&
this.workspaceTargets.get(target.cwd) === target
);
@@ -1612,11 +1647,110 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
workspaceTarget: WorkspaceGitTarget,
request: WorkspaceGitRefreshRequest,
): Promise<boolean> {
if (await this.suspendWorkspaceIfFolderMissing(workspaceTarget)) {
return false;
}
const fingerprintBefore = workspaceTarget.latestFingerprint;
await this.refreshWorkspaceTarget(workspaceTarget, request);
return workspaceTarget.latestFingerprint !== fingerprintBefore;
}
private async suspendWorkspaceIfFolderMissing(target: WorkspaceGitTarget): Promise<boolean> {
if (target.folderMissing) {
return true;
}
if (await this.deps.workspaceFolderExists(target.cwd)) {
return false;
}
this.suspendWorkspaceWithMissingFolder(target);
return true;
}
private suspendWorkspaceWithMissingFolder(target: WorkspaceGitTarget): void {
if (target.folderMissing || target.closed || this.workspaceTargets.get(target.cwd) !== target) {
return;
}
target.folderMissing = true;
if (target.refreshState.status === "in-flight") {
target.refreshState.queued = null;
}
this.detachWorkspaceFromRepoTarget(target);
target.repoGitRoot = null;
this.stopWorkspaceObservation(target);
target.observationSetupComplete = false;
this.scheduleMissingFolderProbe(target);
this.logger.warn(
{ cwd: target.cwd },
"Workspace folder no longer exists; suspended git observation until it is back",
);
}
private isSuspendedWorkspaceTarget(target: WorkspaceGitTarget): boolean {
return (
target.folderMissing && !target.closed && this.workspaceTargets.get(target.cwd) === target
);
}
private scheduleMissingFolderProbe(target: WorkspaceGitTarget): void {
if (
!this.isSuspendedWorkspaceTarget(target) ||
target.observationReensureTimer ||
target.folderRestorePromise
) {
return;
}
target.observationReensureTimer = setTimeout(() => {
target.observationReensureTimer = null;
void this.resumeWorkspaceIfFolderRestored(target);
}, WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS);
}
private resumeWorkspaceIfFolderRestored(target: WorkspaceGitTarget): Promise<void> {
if (!target.folderMissing) {
return Promise.resolve();
}
target.folderRestorePromise ??= this.resumeWorkspaceWithRestoredFolder(target).finally(() => {
target.folderRestorePromise = null;
this.scheduleMissingFolderProbe(target);
});
return target.folderRestorePromise;
}
private async resumeWorkspaceWithRestoredFolder(target: WorkspaceGitTarget): Promise<void> {
if (!(await this.deps.workspaceFolderExists(target.cwd))) {
return;
}
await target.observationSetupPromise;
if (target.refreshState.status === "in-flight") {
await target.refreshState.promise.catch(() => undefined);
}
if (!this.isSuspendedWorkspaceTarget(target)) {
return;
}
target.latestFacts = null;
target.folderMissing = false;
await this.refreshWorkspaceTarget(target, {
force: false,
refreshStructure: true,
refreshWorktree: true,
includeForge: false,
reason: "workspace-folder-restored",
queueIfBusy: true,
movedRemoteRefs: new Set(),
});
if (target.closed || this.workspaceTargets.get(target.cwd) !== target) {
return;
}
target.observationSetupComplete = false;
if (target.observationReensureTimer) {
clearTimeout(target.observationReensureTimer);
target.observationReensureTimer = null;
}
this.logger.info({ cwd: target.cwd }, "Workspace folder is back; resumed git observation");
this.startWorkspaceSubscriptionTimers(target);
this.scheduleWorkspaceObservationSetup(target);
}
private startWorkingTreeWatchFallback(
target: WorkingTreeWatchTarget,
reason: WorkingTreeWatchFallbackReason,
@@ -1654,7 +1788,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
return changed;
}),
);
this.notifyWorkingTreeConsumers(target);
if (await this.deps.workspaceFolderExists(target.watchPath)) {
this.notifyWorkingTreeConsumers(target);
}
if (!target.closed && (target.subscription === null || target.repoRoot === null)) {
target.fallbackPollQuietTickCount = changes.some(Boolean)
? 0
@@ -1731,6 +1867,10 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
if (target.closed || target.subscription) {
return;
}
if (!(await this.deps.workspaceFolderExists(target.watchPath))) {
this.scheduleWorkingTreeWatchRecovery(target);
return;
}
await this.refreshWorkingTreeIgnoredDirectories(target);
if (target.closed || target.subscription) {
return;
@@ -2786,7 +2926,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
}
private updateForgePrStatusPollForTarget(target: WorkspaceGitTarget): void {
if (target.listeners.size === 0) {
if (!this.isActiveObservedWorkspaceTarget(target)) {
this.stopForgePrStatusPollForTarget(target);
return;
}
@@ -3049,6 +3189,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
target: WorkspaceGitTarget,
request: WorkspaceGitRefreshRequest,
): Promise<WorkspaceGitRuntimeSnapshot> {
if (target.folderMissing) {
return Promise.resolve(target.latestSnapshot ?? buildNotGitSnapshot(target.cwd));
}
if (target.refreshState.status === "in-flight") {
const active = target.refreshState.request;
const addsWork =
@@ -3191,11 +3334,19 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
try {
const runRefreshGitCommand = createRunGitCommand(`workspace-refresh:${request.reason}`);
const admittedSnapshot = await this.workspaceRefreshLimit(() => {
if (target.closed || this.workspaceTargets.get(target.cwd) !== target) {
if (
target.closed ||
target.folderMissing ||
this.workspaceTargets.get(target.cwd) !== target
) {
return null;
}
return this.refreshSnapshot(target, request, runRefreshGitCommand);
});
if (target.folderMissing) {
failure = null;
break;
}
if (!admittedSnapshot) {
break;
}
@@ -3209,6 +3360,10 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
failure = { error };
}
if (target.folderMissing) {
failure = null;
break;
}
if (this.disposed || target.closed || this.workspaceTargets.get(target.cwd) !== target) {
break;
}
@@ -3504,7 +3659,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
snapshot: WorkspaceGitRuntimeSnapshot,
options?: { forceEmit?: boolean },
): void {
if (target.closed || this.workspaceTargets.get(target.cwd) !== target) {
if (target.closed || target.folderMissing || this.workspaceTargets.get(target.cwd) !== target) {
return;
}
target.latestSnapshot = snapshot;
@@ -3660,21 +3815,25 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
}
private removeWorkspaceTarget(target: WorkspaceGitTarget): void {
if (target.repoGitRoot) {
const repoTarget = this.repoTargets.get(target.repoGitRoot);
repoTarget?.workspaceKeys.delete(target.cwd);
if (repoTarget && repoTarget.workspaceKeys.size === 0) {
this.closeRepoTarget(repoTarget);
this.repoTargets.delete(target.repoGitRoot);
} else if (repoTarget?.cwd === target.cwd) {
repoTarget.cwd = repoTarget.workspaceKeys.values().next().value ?? repoTarget.cwd;
}
}
this.detachWorkspaceFromRepoTarget(target);
this.closeWorkspaceTarget(target);
this.workspaceTargets.delete(target.cwd);
}
private detachWorkspaceFromRepoTarget(target: WorkspaceGitTarget): void {
if (!target.repoGitRoot) {
return;
}
const repoTarget = this.repoTargets.get(target.repoGitRoot);
repoTarget?.workspaceKeys.delete(target.cwd);
if (repoTarget && repoTarget.workspaceKeys.size === 0) {
this.closeRepoTarget(repoTarget);
this.repoTargets.delete(target.repoGitRoot);
} else if (repoTarget?.cwd === target.cwd) {
repoTarget.cwd = repoTarget.workspaceKeys.values().next().value ?? repoTarget.cwd;
}
}
private removeWorkingTreeWatchListener(cwd: string, listener: () => void): void {
const target = this.workingTreeWatchTargets.get(cwd);
if (!target) {
@@ -3719,6 +3878,11 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
private closeWorkspaceTarget(target: WorkspaceGitTarget): void {
target.closed = true;
this.stopWorkspaceObservation(target);
target.listeners.clear();
}
private stopWorkspaceObservation(target: WorkspaceGitTarget): void {
if (target.workingTreeWatchTarget) {
this.removeWorkspaceWorkingTreeLink(target.workingTreeWatchTarget, target.cwd);
target.workingTreeWatchTarget = null;
@@ -3727,12 +3891,12 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService {
clearTimeout(target.debounceTimer);
target.debounceTimer = null;
}
target.pendingDebounceRequest = null;
if (target.observationReensureTimer) {
clearTimeout(target.observationReensureTimer);
target.observationReensureTimer = null;
}
this.stopForgePrStatusPollForTarget(target);
target.listeners.clear();
}
private closeWorkingTreeWatchTarget(target: WorkingTreeWatchTarget): void {