From bfac1329204f9aaeb6d47742885c51551d64f70b Mon Sep 17 00:00:00 2001 From: "paseo-bot[bot]" <266920839+paseo-bot[bot]@users.noreply.github.com> Date: Sun, 1 Nov 2026 18:02:26 +0000 Subject: [PATCH] Stop Git polling while workspace folders are missing (#8462) * fix(server): stop polling git for a workspace whose folder is gone When a workspace's folder disappears together with its project (the repository deleted, a volume unmounted), reconciliation leaves the workspace registered, and the daemon kept polling it: a failed git command and a warning about once a minute, and a working-tree watcher retry every minute or two, for as long as the daemon ran. The degraded poll now checks that the workspace folder exists before refreshing. A workspace whose folder is gone is suspended once: it is detached from its repository and working-tree targets, its timers stop, and one warning is logged. A worktree that shared a repository target hands the repository's polling and background fetch to a surviving worktree. Nothing is archived. A working-tree watch that an open diff still holds skips its consumer notifications and watcher recovery while its folder is missing. While suspended, the workspace runs no git. A stat every 60 seconds, a new subscription, a forced snapshot or a refresh request checks whether the folder is back; when it is, the workspace reloads its git facts, refreshes, and rebuilds its observation. Fixes #7856 * fix(server): keep missing workspaces suspended during refresh Admit Git refreshes only after the folder has returned. Cover missing-folder polling, explicit refreshes, and restoration with real Git and the Linux observer. Co-authored-by: Dominic Anaeto * fix(server): preserve workspace snapshots through suspension Drop queued refreshes and withhold publication when a folder disappears during an active read. Wait for that read before restoring observation so its stale result cannot overwrite the recovered snapshot. --------- Co-authored-by: Dominic Anaeto Co-authored-by: paseo-bot[bot] <266920839+paseo-bot[bot]@users.noreply.github.com> --- ...it-service.observation.integration.test.ts | 101 +++ .../workspace-git-service.observation.test.ts | 1 + .../src/server/workspace-git-service.test.ts | 606 +++++++++++++++++- .../src/server/workspace-git-service.ts | 204 +++++- 4 files changed, 891 insertions(+), 21 deletions(-) diff --git a/packages/server/src/server/workspace-git-service.observation.integration.test.ts b/packages/server/src/server/workspace-git-service.observation.integration.test.ts index 371502748d..1ae04bfc2f 100644 --- a/packages/server/src/server/workspace-git-service.observation.integration.test.ts +++ b/packages/server/src/server/workspace-git-service.observation.integration.test.ts @@ -1,3 +1,4 @@ +import { flushGitCommandTrace } from "../utils/git-command-trace.js"; import { createWorkspaceRuntimeStore } from "./workspace-runtime-store/index.js"; import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { execFileSync } from "node:child_process"; @@ -730,3 +731,103 @@ test("Git event identity retains a symlink's literal target before canonicalizin rmSync(root, { recursive: true, force: true }); } }); + +test.skipIf(process.platform !== "linux")( + "missing folders stop degraded Git polling and explicit refreshes, then recover", + async () => { + const artifacts = path.resolve(".artifacts"); + mkdirSync(artifacts, { recursive: true }); + const root = mkdtempSync(path.join(artifacts, "workspace-git-missing-")); + const cwd = path.join(root, "repo"); + const initializeRepo = (branch: string) => { + mkdirSync(cwd); + execFileSync("git", ["init", "-b", branch], { cwd, stdio: "pipe" }); + writeFileSync(path.join(cwd, "README.md"), "fixture\n"); + execFileSync("git", ["add", "."], { cwd, stdio: "pipe" }); + execFileSync( + "git", + [ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.invalid", + "-c", + "commit.gpgsign=false", + "commit", + "-m", + "fixture", + ], + { cwd, stdio: "pipe" }, + ); + }; + initializeRepo("main"); + // Exceed the real Linux observer's directory limit to enter degraded polling. + // No watcher errors or Git responses are injected. + for (let index = 0; index < 5100; index++) { + mkdirSync(path.join(cwd, `generated-${index}`)); + } + const trace = path.join(root, "git.jsonl"); + writeFileSync(trace, ""); + const previousTrace = process.env.PASEO_GIT_TRACE_FILE; + process.env.PASEO_GIT_TRACE_FILE = trace; + let degraded = false; + let suspended = false; + const logger = pino( + { level: "warn" }, + { + write(line) { + const message = JSON.parse(line).msg; + if (message === "Failed to start working tree watcher; using degraded polling") + degraded = true; + if ( + message === + "Workspace folder no longer exists; suspended git observation until it is back" + ) + suspended = true; + }, + }, + ); + const service = new WorkspaceGitServiceImpl({ logger, paseoHome: path.join(root, "home") }); + const subscription = service.registerWorkspace({ cwd }, () => {}); + const commandCount = async () => { + await flushGitCommandTrace(); + return readFileSync(trace, "utf8").trim().split("\n").filter(Boolean).length; + }; + try { + await expect + .poll(() => degraded && service.getMetrics().workspaceRefreshInFlightCount === 0, { + timeout: 15_000, + }) + .toBe(true); + rmSync(cwd, { recursive: true, force: true }); + await expect.poll(() => suspended, { timeout: 15_000 }).toBe(true); + await expect.poll(() => service.getMetrics().workspaceRefreshInFlightCount).toBe(0); + const commandsWhileMissing = await commandCount(); + await service.getSnapshot(cwd, { + force: true, + includeForge: false, + reason: "refresh-button", + }); + await service.refresh(cwd); + expect(await commandCount()).toBe(commandsWhileMissing); + await new Promise((resolve) => setTimeout(resolve, 16_000)); + expect(await commandCount()).toBe(commandsWhileMissing); + + initializeRepo("restored"); + const snapshot = await service.getSnapshot(cwd, { + force: true, + includeForge: false, + reason: "refresh-button", + }); + expect(snapshot.git).toMatchObject({ isGit: true, currentBranch: "restored" }); + await expect.poll(() => service.getMetrics().repositoryWorkspaceLinkCount).toBe(1); + } finally { + subscription.unsubscribe(); + await service.dispose(); + if (previousTrace === undefined) delete process.env.PASEO_GIT_TRACE_FILE; + else process.env.PASEO_GIT_TRACE_FILE = previousTrace; + rmSync(root, { recursive: true, force: true }); + } + }, + 60_000, +); diff --git a/packages/server/src/server/workspace-git-service.observation.test.ts b/packages/server/src/server/workspace-git-service.observation.test.ts index ab93558793..04604cd1b5 100644 --- a/packages/server/src/server/workspace-git-service.observation.test.ts +++ b/packages/server/src/server/workspace-git-service.observation.test.ts @@ -244,6 +244,7 @@ function createService( filterEvents: (events) => events, verify: vi.fn(async () => {}), })), + workspaceFolderExists: vi.fn(async () => true), ...overrides, } as never, }); diff --git a/packages/server/src/server/workspace-git-service.test.ts b/packages/server/src/server/workspace-git-service.test.ts index 4e6acff9cd..a7a3b49f06 100644 --- a/packages/server/src/server/workspace-git-service.test.ts +++ b/packages/server/src/server/workspace-git-service.test.ts @@ -1,5 +1,7 @@ import { createWorkspaceRuntimeStore } from "./workspace-runtime-store/index.js"; import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path, { join } from "node:path"; import type pino from "pino"; @@ -10,11 +12,14 @@ import type { PullRequestStatusResult, } from "../utils/checkout-git.js"; import { + BACKGROUND_GIT_FETCH_INTERVAL_MS, + WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS, WORKSPACE_GIT_OBSERVATION_SETUP_CONCURRENCY, WORKSPACE_GIT_REFRESH_CONCURRENCY, WORKSPACE_GIT_WATCHER_SUBSCRIBE_TIMEOUT_MS, WorkspaceGitServiceImpl, type WorkspaceGitRuntimeSnapshot, + workspaceFolderExists, } from "./workspace-git-service.js"; const REPO_CWD = path.resolve("/tmp/repo"); @@ -23,6 +28,7 @@ function createLogger() { const logger = { child: () => logger, debug: vi.fn(), + info: vi.fn(), warn: vi.fn(), }; return logger; @@ -251,7 +257,7 @@ function createGitHubServiceStub(): ForgeService { interface CreateServiceTestOptions { subscribe?: ReturnType; getCheckoutStatus?: ReturnType; - getCheckoutSnapshotFacts?: ReturnType; + getCheckoutSnapshotFacts?: (cwd: string) => Promise; getCheckoutShortstat?: ReturnType; getCheckoutWorktreeState?: ReturnType; getPullRequestStatus?: ReturnType; @@ -261,6 +267,7 @@ interface CreateServiceTestOptions { runGitFetch?: ReturnType; runGitCommand?: ReturnType; getWorkspaceGitSelfHealPhaseMs?: (cwd: string) => number; + workspaceFolderExists?: (cwd: string) => Promise; now?: () => Date; } @@ -294,6 +301,7 @@ function buildDefaultTestServiceDeps() { signal: null, })), getWorkspaceGitSelfHealPhaseMs: vi.fn(() => 30_000), + workspaceFolderExists: vi.fn(async () => true), createWatcherLivenessCanary: vi.fn(() => ({ path: "", filterEvents: (events) => events, @@ -324,6 +332,145 @@ function createService(options?: CreateServiceTestOptions) { }); } +type WatchCallback = (error: Error | null, events: Array<{ path: string; type: "update" }>) => void; + +interface FolderBackedGitOptions { + facts?: (cwd: string) => CheckoutSnapshotFacts; + unwatchableDirectories?: string[]; + heldWatch?: { directory: string; release: Promise }; +} + +function createFolderBackedGit(options?: FolderBackedGitOptions) { + const watchCallbacks = new Map(); + const subscribe = vi.fn(async (directory: string, callback: WatchCallback) => { + if (!existsSync(directory) || options?.unwatchableDirectories?.includes(directory)) { + throw new Error(`Cannot watch ${directory}`); + } + if (options?.heldWatch?.directory === directory) { + await options.heldWatch.release; + } + watchCallbacks.set(directory, callback); + return createAsyncSubscription(); + }); + const getCheckoutSnapshotFacts = vi.fn( + async (cwd: string): Promise => + existsSync(cwd) ? (options?.facts ?? createCheckoutSnapshotFacts)(cwd) : { isGit: false }, + ); + const getCheckoutStatus = vi.fn(async (cwd: string) => + existsSync(cwd) ? createCheckoutStatus(cwd) : ({ isGit: false } as const), + ); + const getCheckoutShortstat = vi.fn(async (_cwd: string) => ({ additions: 1, deletions: 0 })); + const getCheckoutWorktreeState = vi.fn(async (_cwd: string) => ({ + isDirty: false, + diffStat: { additions: 1, deletions: 0 }, + })); + const runGitCommand = vi.fn(async (args: string[], runOptions: { cwd: string }) => ({ + stdout: args[0] === "rev-parse" ? `${runOptions.cwd}\n` : "", + stderr: "", + truncated: false, + exitCode: 0, + signal: null, + })); + const runGitFetch = vi.fn(async (_cwd: string) => ({ changes: [], error: null })); + const callsIn = (fake: ReturnType, cwd: string) => + fake.mock.calls.filter(([calledCwd]) => calledCwd === cwd).length; + + return { + deps: { + subscribe, + getCheckoutSnapshotFacts, + getCheckoutStatus, + getCheckoutShortstat, + getCheckoutWorktreeState, + runGitCommand, + runGitFetch, + workspaceFolderExists: async (cwd: string) => existsSync(cwd), + now: () => new Date(), + }, + watchCallbackFor(directory: string): WatchCallback { + const callback = watchCallbacks.get(directory); + if (!callback) { + throw new Error(`No watcher was started for ${directory}`); + } + return callback; + }, + watchAttemptsFor: (directory: string) => callsIn(subscribe, directory), + pollsOf: (cwd: string) => callsIn(getCheckoutShortstat, cwd), + fetchesIn: (cwd: string) => callsIn(runGitFetch, cwd), + gitCommandsIn(cwd: string): number { + const cwdCommands = [ + getCheckoutSnapshotFacts, + getCheckoutStatus, + getCheckoutShortstat, + getCheckoutWorktreeState, + runGitFetch, + ].reduce((count, fake) => count + callsIn(fake, cwd), 0); + const commandsRunInCwd = runGitCommand.mock.calls.filter( + ([, runOptions]) => runOptions.cwd === cwd, + ).length; + return cwdCommands + commandsRunInCwd; + }, + }; +} + +function linkedWorktreeFacts(gitCommonDir: string) { + return (cwd: string): CheckoutSnapshotFacts => ({ + ...createCheckoutSnapshotFacts(cwd), + absoluteGitDir: join(gitCommonDir, "worktrees", path.basename(cwd)), + gitCommonDir, + }); +} + +const SETTLE_ATTEMPTS = 1_000; + +async function settle(assertion: () => void): Promise { + let lastFailure: unknown = null; + for (let attempt = 0; attempt < SETTLE_ATTEMPTS; attempt += 1) { + try { + assertion(); + return; + } catch (failure) { + lastFailure = failure; + } + await vi.advanceTimersByTimeAsync(0); + } + throw lastFailure; +} + +async function waitForRefreshesToSettle(service: WorkspaceGitServiceImpl): Promise { + await settle(() => { + expect(service.getMetrics().workspaceRefreshInFlightCount).toBe(0); + }); +} + +async function waitForObservationToSettle( + service: WorkspaceGitServiceImpl, + repositoryWorkspaceLinkCount: number, +): Promise { + await settle(() => { + const metrics = service.getMetrics(); + expect(metrics.repositoryWorkspaceLinkCount).toBe(repositoryWorkspaceLinkCount); + expect(metrics.workspaceObservationSetupInFlightCount).toBe(0); + expect(metrics.workspaceRefreshInFlightCount).toBe(0); + }); +} + +async function advanceThroughDegradedPolls(service: WorkspaceGitServiceImpl): Promise { + for (const delayMs of [5_000, 10_000, 20_000, 40_000, 60_000, 60_000]) { + await vi.advanceTimersByTimeAsync(delayMs); + await waitForRefreshesToSettle(service); + } +} + +async function advanceThroughWorkingTreeRecoveryAttempts( + service: WorkspaceGitServiceImpl, +): Promise { + for (const delayMs of [30_000, 60_000, 120_000]) { + await vi.advanceTimersByTimeAsync(delayMs); + await waitForRefreshesToSettle(service); + } +} + describe("WorkspaceGitServiceImpl", () => { beforeEach(() => { vi.useFakeTimers(); @@ -1553,3 +1700,460 @@ describe("WorkspaceGitServiceImpl", () => { service.dispose(); }); }); + +describe("WorkspaceGitServiceImpl with workspace folders that no longer exist", () => { + const temporaryFolders: string[] = []; + + async function createTemporaryFolder(): Promise { + const folder = await mkdtemp(join(os.tmpdir(), "paseo-workspace-git-")); + temporaryFolders.push(folder); + return folder; + } + + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all( + temporaryFolders.splice(0).map((folder) => rm(folder, { recursive: true, force: true })), + ); + }); + + test("polling stops for a workspace whose folder no longer exists", async () => { + const workspace = await createTemporaryFolder(); + const git = createFolderBackedGit(); + const service = createService(git.deps); + const subscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + + await rm(workspace, { recursive: true, force: true }); + const gitCommandsAtDeletion = git.gitCommandsIn(workspace); + await advanceThroughDegradedPolls(service); + + expect(git.gitCommandsIn(workspace)).toBe(gitCommandsAtDeletion); + expect(service.getMetrics()).toMatchObject({ + repositoryTargetCount: 0, + workingTreeWatchTargetCount: 0, + }); + expect(vi.getTimerCount()).toBe(1); + + subscription.unsubscribe(); + await service.dispose(); + }); + + test("a deleted worktree leaves its shared repository polling and fetch to the surviving worktree", async () => { + const gitCommonDir = join(await createTemporaryFolder(), ".git"); + const deletedWorktree = await createTemporaryFolder(); + const survivingWorktree = await createTemporaryFolder(); + const git = createFolderBackedGit({ facts: linkedWorktreeFacts(gitCommonDir) }); + let survivingWorktreeEdits = 0; + git.deps.getCheckoutShortstat.mockImplementation(async (cwd: string) => ({ + additions: cwd === survivingWorktree ? ++survivingWorktreeEdits : 1, + deletions: 0, + })); + const service = createService(git.deps); + const deletedSubscription = service.registerWorkspace({ cwd: deletedWorktree }, vi.fn()); + await waitForObservationToSettle(service, 1); + const survivingSubscription = service.registerWorkspace({ cwd: survivingWorktree }, vi.fn()); + await waitForObservationToSettle(service, 2); + + await rm(deletedWorktree, { recursive: true, force: true }); + await vi.advanceTimersByTimeAsync(60_000); + await waitForRefreshesToSettle(service); + const deletedWorktreeCommands = git.gitCommandsIn(deletedWorktree); + + for (let tick = 0; tick < 3; tick += 1) { + const survivingWorktreePolls = git.pollsOf(survivingWorktree); + await vi.advanceTimersByTimeAsync(5_000); + await settle(() => { + expect(git.pollsOf(survivingWorktree)).toBe(survivingWorktreePolls + 1); + }); + } + expect(git.gitCommandsIn(deletedWorktree)).toBe(deletedWorktreeCommands); + + const fetchesBefore = git.deps.runGitFetch.mock.calls.length; + await vi.advanceTimersByTimeAsync(2 * BACKGROUND_GIT_FETCH_INTERVAL_MS); + expect(git.deps.runGitFetch.mock.calls.slice(fetchesBefore).map(([cwd]) => cwd)).toEqual([ + survivingWorktree, + survivingWorktree, + ]); + expect(git.gitCommandsIn(deletedWorktree)).toBe(deletedWorktreeCommands); + + deletedSubscription.unsubscribe(); + survivingSubscription.unsubscribe(); + await service.dispose(); + }); + + test("deleting every worktree of a repository stops its polling and background fetch", async () => { + const gitCommonDir = join(await createTemporaryFolder(), ".git"); + const firstWorktree = await createTemporaryFolder(); + const secondWorktree = await createTemporaryFolder(); + const git = createFolderBackedGit({ facts: linkedWorktreeFacts(gitCommonDir) }); + const service = createService(git.deps); + const firstSubscription = service.registerWorkspace({ cwd: firstWorktree }, vi.fn()); + await waitForObservationToSettle(service, 1); + const secondSubscription = service.registerWorkspace({ cwd: secondWorktree }, vi.fn()); + await waitForObservationToSettle(service, 2); + + await rm(firstWorktree, { recursive: true, force: true }); + await rm(secondWorktree, { recursive: true, force: true }); + const commandsAtDeletion = [ + git.gitCommandsIn(firstWorktree), + git.gitCommandsIn(secondWorktree), + ]; + await advanceThroughDegradedPolls(service); + + expect([git.gitCommandsIn(firstWorktree), git.gitCommandsIn(secondWorktree)]).toEqual( + commandsAtDeletion, + ); + expect(service.getMetrics()).toMatchObject({ + repositoryTargetCount: 0, + workingTreeWatchTargetCount: 0, + }); + expect(vi.getTimerCount()).toBe(2); + + firstSubscription.unsubscribe(); + secondSubscription.unsubscribe(); + await service.dispose(); + }); + + test("working tree fallback polling and watcher recovery stop for a workspace whose folder no longer exists", async () => { + const workspace = await createTemporaryFolder(); + await mkdir(join(workspace, ".git")); + const git = createFolderBackedGit(); + const service = createService(git.deps); + const subscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + + await rm(workspace, { recursive: true, force: true }); + git.watchCallbackFor(workspace)(new Error("watched directory was deleted"), []); + await vi.advanceTimersByTimeAsync(1_000); + await waitForRefreshesToSettle(service); + const commandsAfterWatcherError = git.gitCommandsIn(workspace); + const watchAttemptsAfterWatcherError = git.watchAttemptsFor(workspace); + await advanceThroughDegradedPolls(service); + + expect(git.gitCommandsIn(workspace)).toBe(commandsAfterWatcherError); + expect(git.watchAttemptsFor(workspace)).toBe(watchAttemptsAfterWatcherError); + expect(vi.getTimerCount()).toBe(1); + + subscription.unsubscribe(); + await service.dispose(); + }); + + test("a workspace whose folder comes back resumes polling on its next subscription", async () => { + const workspace = await createTemporaryFolder(); + const git = createFolderBackedGit(); + const service = createService(git.deps); + const firstSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + await rm(workspace, { recursive: true, force: true }); + const commandsWhileMissing = git.gitCommandsIn(workspace); + await advanceThroughDegradedPolls(service); + expect(git.gitCommandsIn(workspace)).toBe(commandsWhileMissing); + + await mkdir(workspace); + const secondSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + const pollsAfterResume = git.pollsOf(workspace); + await vi.advanceTimersByTimeAsync(5_000); + await waitForRefreshesToSettle(service); + + expect(git.pollsOf(workspace)).toBe(pollsAfterResume + 1); + + firstSubscription.unsubscribe(); + secondSubscription.unsubscribe(); + await service.dispose(); + }); + + test("a suspended workspace costs no git while its folder is missing, even when a refresh is requested", async () => { + const workspace = await createTemporaryFolder(); + const git = createFolderBackedGit(); + const service = createService(git.deps); + const subscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + await rm(workspace, { recursive: true, force: true }); + await advanceThroughDegradedPolls(service); + const commandsWhileMissing = git.gitCommandsIn(workspace); + + for (let probe = 0; probe < 10; probe += 1) { + await vi.advanceTimersByTimeAsync(WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS); + await waitForRefreshesToSettle(service); + } + service.scheduleRefreshForCwd(workspace); + service.onWorkspaceStateMayHaveChanged(workspace); + await vi.advanceTimersByTimeAsync(5_000); + await waitForRefreshesToSettle(service); + + expect(git.gitCommandsIn(workspace)).toBe(commandsWhileMissing); + expect(vi.getTimerCount()).toBe(1); + + subscription.unsubscribe(); + await service.dispose(); + }); + + test("a suspended worktree resumes polling and its repository fetch once its folder is back, without a new subscription", async () => { + const gitCommonDir = join(await createTemporaryFolder(), ".git"); + const worktree = await createTemporaryFolder(); + const git = createFolderBackedGit({ facts: linkedWorktreeFacts(gitCommonDir) }); + const service = createService(git.deps); + const subscription = service.registerWorkspace({ cwd: worktree }, vi.fn()); + await waitForObservationToSettle(service, 1); + await rm(worktree, { recursive: true, force: true }); + await advanceThroughDegradedPolls(service); + await waitForObservationToSettle(service, 0); + + await mkdir(worktree); + const fetchesBeforeRestore = git.fetchesIn(worktree); + await vi.advanceTimersByTimeAsync(WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS); + await waitForObservationToSettle(service, 1); + expect(git.fetchesIn(worktree)).toBe(fetchesBeforeRestore + 1); + + const pollsAfterRestore = git.pollsOf(worktree); + await vi.advanceTimersByTimeAsync(5_000); + await waitForRefreshesToSettle(service); + expect(git.pollsOf(worktree)).toBe(pollsAfterRestore + 1); + + subscription.unsubscribe(); + await service.dispose(); + }); + + test("a forced snapshot resumes a suspended workspace whose folder is back", async () => { + const workspace = await createTemporaryFolder(); + const git = createFolderBackedGit(); + const service = createService(git.deps); + const subscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + await rm(workspace, { recursive: true, force: true }); + await advanceThroughDegradedPolls(service); + await waitForObservationToSettle(service, 0); + + await mkdir(workspace); + const snapshot = await service.getSnapshot(workspace, { + force: true, + reason: "refresh-button", + }); + await waitForObservationToSettle(service, 1); + + expect(snapshot.git.isGit).toBe(true); + + subscription.unsubscribe(); + await service.dispose(); + }); + + test("a resumed workspace rebuilds observation from fresh facts, not the non-git facts cached while its folder was gone", async () => { + const workspace = await createTemporaryFolder(); + const git = createFolderBackedGit(); + const service = createService(git.deps); + const firstSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + await rm(workspace, { recursive: true, force: true }); + service.onWorkspaceStateMayHaveChanged(workspace); + await vi.advanceTimersByTimeAsync(1_000); + await waitForRefreshesToSettle(service); + expect((await service.getSnapshot(workspace)).git.isGit).toBe(false); + await advanceThroughDegradedPolls(service); + await waitForObservationToSettle(service, 0); + + await mkdir(workspace); + const secondSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + + expect((await service.getSnapshot(workspace)).git.isGit).toBe(true); + + firstSubscription.unsubscribe(); + secondSubscription.unsubscribe(); + await service.dispose(); + }); + + test("resuming waits for an observation setup that was still running when the folder vanished", async () => { + const workspace = await createTemporaryFolder(); + const gitDir = join(workspace, ".git"); + await mkdir(gitDir); + const metadataWatchRelease = createDeferred(); + const git = createFolderBackedGit({ + unwatchableDirectories: [workspace], + heldWatch: { directory: gitDir, release: metadataWatchRelease.promise }, + }); + const service = createService(git.deps); + const firstSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await settle(() => { + expect(git.watchAttemptsFor(gitDir)).toBe(1); + }); + + await rm(workspace, { recursive: true, force: true }); + await vi.advanceTimersByTimeAsync(5_000); + await settle(() => { + expect(service.getMetrics()).toMatchObject({ + repositoryTargetCount: 0, + workingTreeWatchTargetCount: 0, + }); + }); + await mkdir(workspace); + const secondSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + metadataWatchRelease.resolve(); + await waitForObservationToSettle(service, 1); + + expect(service.getMetrics().workingTreeWatchTargetCount).toBe(1); + + firstSubscription.unsubscribe(); + secondSubscription.unsubscribe(); + await service.dispose(); + }); + + test("an open working tree watch neither notifies nor recovers while its folder is missing, and picks up once it is back", async () => { + const workspace = await createTemporaryFolder(); + await mkdir(join(workspace, ".git")); + const git = createFolderBackedGit(); + const service = createService(git.deps); + const workspaceSubscription = service.registerWorkspace({ cwd: workspace }, vi.fn()); + await waitForObservationToSettle(service, 1); + const diffListener = vi.fn(); + const diffWatch = await service.requestWorkingTreeWatch(workspace, diffListener); + + await rm(workspace, { recursive: true, force: true }); + git.watchCallbackFor(workspace)(new Error("watched directory was deleted"), []); + await vi.advanceTimersByTimeAsync(1_000); + await waitForRefreshesToSettle(service); + const commandsAfterWatcherError = git.gitCommandsIn(workspace); + const notificationsAfterWatcherError = diffListener.mock.calls.length; + const watchAttemptsAfterWatcherError = git.watchAttemptsFor(workspace); + await advanceThroughWorkingTreeRecoveryAttempts(service); + + expect(git.gitCommandsIn(workspace)).toBe(commandsAfterWatcherError); + expect(diffListener.mock.calls.length).toBe(notificationsAfterWatcherError); + expect(git.watchAttemptsFor(workspace)).toBe(watchAttemptsAfterWatcherError); + expect(service.getMetrics().workingTreeWatchTargetCount).toBe(1); + + await mkdir(join(workspace, ".git"), { recursive: true }); + await vi.advanceTimersByTimeAsync(60_000); + await waitForRefreshesToSettle(service); + expect(diffListener.mock.calls.length).toBe(notificationsAfterWatcherError + 1); + + await vi.advanceTimersByTimeAsync(240_000); + await waitForRefreshesToSettle(service); + expect(git.watchAttemptsFor(workspace)).toBe(watchAttemptsAfterWatcherError + 1); + const notificationsAfterResubscribe = diffListener.mock.calls.length; + await vi.advanceTimersByTimeAsync(120_000); + await waitForRefreshesToSettle(service); + expect(diffListener.mock.calls.length).toBe(notificationsAfterResubscribe); + + diffWatch.unsubscribe(); + workspaceSubscription.unsubscribe(); + await service.dispose(); + }); + + test("a refresh crossing suspension preserves the cached snapshot and drops queued work", async () => { + const workspace = await createTemporaryFolder(); + const git = createFolderBackedGit(); + const started = createDeferred(); + const release = createDeferred(); + let holdNextRead = false; + let factsReads = 0; + const service = createService({ + ...git.deps, + getCheckoutSnapshotFacts: async (cwd: string) => { + factsReads++; + if (holdNextRead) { + holdNextRead = false; + started.resolve(); + await release.promise; + } + return git.deps.getCheckoutSnapshotFacts(cwd); + }, + }); + const updates: WorkspaceGitRuntimeSnapshot[] = []; + const subscription = service.registerWorkspace({ cwd: workspace }, (snapshot) => + updates.push(snapshot), + ); + await waitForObservationToSettle(service, 1); + const cached = await service.getSnapshot(workspace); + holdNextRead = true; + const active = service.getSnapshot(workspace, { + force: true, + includeForge: false, + reason: "overlapping-refresh", + }); + await started.promise; + const queued = service.getSnapshot(workspace, { + force: true, + includeForge: true, + reason: "queued-refresh", + }); + await rm(workspace, { recursive: true, force: true }); + await vi.advanceTimersByTimeAsync(5_000); + await settle(() => expect(service.getMetrics().repositoryWorkspaceLinkCount).toBe(0)); + const readsAtSuspension = factsReads; + const updatesAtSuspension = updates.length; + release.resolve(); + expect(await active).toEqual(cached); + expect(await queued).toEqual(cached); + expect(await service.getSnapshot(workspace)).toEqual(cached); + expect(updates).toHaveLength(updatesAtSuspension); + expect(factsReads).toBe(readsAtSuspension); + subscription.unsubscribe(); + await service.dispose(); + }); + + test("restoration settles a suspended refresh before reloading facts", async () => { + const workspace = await createTemporaryFolder(); + const git = createFolderBackedGit(); + const staleFacts = createDeferred(); + const started = createDeferred(); + let holdNextRead = false; + const service = createService({ + ...git.deps, + getCheckoutSnapshotFacts: async (cwd: string) => { + if (holdNextRead) { + holdNextRead = false; + started.resolve(); + return staleFacts.promise; + } + return git.deps.getCheckoutSnapshotFacts(cwd); + }, + }); + const updates: WorkspaceGitRuntimeSnapshot[] = []; + const subscription = service.registerWorkspace({ cwd: workspace }, (snapshot) => + updates.push(snapshot), + ); + await waitForObservationToSettle(service, 1); + const cached = await service.getSnapshot(workspace); + holdNextRead = true; + const active = service.getSnapshot(workspace, { + force: true, + includeForge: false, + reason: "overlapping-refresh", + }); + await started.promise; + await rm(workspace, { recursive: true, force: true }); + await vi.advanceTimersByTimeAsync(5_000); + await settle(() => expect(service.getMetrics().repositoryWorkspaceLinkCount).toBe(0)); + const updatesAtSuspension = updates.length; + await mkdir(workspace); + const restored = service.getSnapshot(workspace, { + force: true, + includeForge: false, + reason: "folder-returned", + }); + await vi.advanceTimersByTimeAsync(0); + staleFacts.resolve({ isGit: false }); + expect(await active).toEqual(cached); + expect((await restored).git.isGit).toBe(true); + expect(updates.slice(updatesAtSuspension).every((snapshot) => snapshot.git.isGit)).toBe(true); + await waitForObservationToSettle(service, 1); + subscription.unsubscribe(); + await service.dispose(); + }); + + test("workspaceFolderExists reports a missing folder and a path under a file as missing", async () => { + const folder = await createTemporaryFolder(); + const file = join(folder, "file.txt"); + await writeFile(file, ""); + + expect(await workspaceFolderExists(folder)).toBe(true); + expect(await workspaceFolderExists(join(folder, "missing"))).toBe(false); + expect(await workspaceFolderExists(join(file, "nested"))).toBe(false); + }); +}); diff --git a/packages/server/src/server/workspace-git-service.ts b/packages/server/src/server/workspace-git-service.ts index b5672846bd..26a7f16419 100644 --- a/packages/server/src/server/workspace-git-service.ts +++ b/packages/server/src/server/workspace-git-service.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; +import { readFile, stat } from "node:fs/promises"; import { basename, dirname, join, resolve } from "node:path"; import { LRUCache } from "lru-cache"; import { CheckoutDiffCache } from "./checkout-diff-cache.js"; @@ -74,7 +74,7 @@ import { checkoutLiteFromGitSnapshot } from "./workspace-registry-model.js"; import { createWatcherLivenessCanary } from "./watcher-liveness-canary.js"; const WORKSPACE_GIT_WATCH_DEBOUNCE_MS = 1_000; -const BACKGROUND_GIT_FETCH_INTERVAL_MS = 180_000; +export const BACKGROUND_GIT_FETCH_INTERVAL_MS = 180_000; const FETCH_METADATA_ECHO_TTL_MS = 5_000; export const WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS = 60_000; const FORGE_PR_STATUS_POLL_FAST_INTERVAL_MS = 20_000; @@ -384,6 +384,7 @@ interface WorkspaceGitServiceDependencies { runGitCommand: typeof runGitCommand; getWorkspaceGitSelfHealPhaseMs: typeof getWorkspaceGitObservationReensurePhaseMs; createWatcherLivenessCanary: typeof createWatcherLivenessCanary; + workspaceFolderExists: (cwd: string) => Promise; now: () => Date; } @@ -435,6 +436,8 @@ interface WorkspaceGitTarget { repoGitRoot: string | null; observationSetupPromise: Promise | null; observationSetupComplete: boolean; + folderMissing: boolean; + folderRestorePromise: Promise | null; closed: boolean; } @@ -571,10 +574,21 @@ function buildDefaultWorkspaceGitServiceDeps( runGitCommand, getWorkspaceGitSelfHealPhaseMs: getWorkspaceGitObservationReensurePhaseMs, createWatcherLivenessCanary, + workspaceFolderExists, now: () => new Date(), }; } +export async function workspaceFolderExists(cwd: string): Promise { + try { + await stat(cwd); + return true; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + return code !== "ENOENT" && code !== "ENOTDIR"; + } +} + function resolveWorkspaceGitServiceDeps( subscribe: SubscribeToFileChanges, deps: Partial | undefined, @@ -660,6 +674,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { const cwd = resolve(params.cwd); const target = this.ensureWorkspaceTarget(cwd); target.listeners.add(listener); + if (target.folderMissing) { + void this.resumeWorkspaceIfFolderRestored(target); + } if (target.listeners.size === 1) { this.startWorkspaceSubscriptionTimers(target); } @@ -751,6 +768,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { if (!request.force && target.latestSnapshot) { return target.latestSnapshot; } + if (target.folderMissing) { + await this.resumeWorkspaceIfFolderRestored(target); + } return this.requestWorkspaceSnapshot(target, request); } @@ -972,6 +992,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { this.assertNotDisposed(); cwd = resolve(cwd); const target = this.ensureWorkspaceTarget(cwd); + if (target.folderMissing) { + await this.resumeWorkspaceIfFolderRestored(target); + } await this.refreshWorkspaceTarget(target, { force: false, refreshStructure: true, @@ -1005,9 +1028,14 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { this.assertNotDisposed(); cwd = resolve(cwd); const target = this.workspaceTargets.get(cwd); - if (target) { - this.scheduleWorkspaceRefresh(target, { queueIfBusy: false }); + if (!target) { + return; } + if (target.folderMissing) { + void this.resumeWorkspaceIfFolderRestored(target); + return; + } + this.scheduleWorkspaceRefresh(target, { queueIfBusy: false }); } onWorkspaceStateMayHaveChanged(cwd: string): void { @@ -1018,6 +1046,10 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { return; } this.invalidateForge(normalizedCwd); + if (target.folderMissing) { + void this.resumeWorkspaceIfFolderRestored(target); + return; + } this.scheduleWorkspaceRefresh(target, { force: true, includeForge: true, @@ -1227,6 +1259,8 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { repoGitRoot: null, observationSetupPromise: null, observationSetupComplete: false, + folderMissing: false, + folderRestorePromise: null, closed: false, }; @@ -1358,6 +1392,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { private isActiveObservedWorkspaceTarget(target: WorkspaceGitTarget): boolean { return ( !target.closed && + !target.folderMissing && target.listeners.size > 0 && this.workspaceTargets.get(target.cwd) === target ); @@ -1612,11 +1647,110 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { workspaceTarget: WorkspaceGitTarget, request: WorkspaceGitRefreshRequest, ): Promise { + if (await this.suspendWorkspaceIfFolderMissing(workspaceTarget)) { + return false; + } const fingerprintBefore = workspaceTarget.latestFingerprint; await this.refreshWorkspaceTarget(workspaceTarget, request); return workspaceTarget.latestFingerprint !== fingerprintBefore; } + private async suspendWorkspaceIfFolderMissing(target: WorkspaceGitTarget): Promise { + if (target.folderMissing) { + return true; + } + if (await this.deps.workspaceFolderExists(target.cwd)) { + return false; + } + this.suspendWorkspaceWithMissingFolder(target); + return true; + } + + private suspendWorkspaceWithMissingFolder(target: WorkspaceGitTarget): void { + if (target.folderMissing || target.closed || this.workspaceTargets.get(target.cwd) !== target) { + return; + } + target.folderMissing = true; + if (target.refreshState.status === "in-flight") { + target.refreshState.queued = null; + } + this.detachWorkspaceFromRepoTarget(target); + target.repoGitRoot = null; + this.stopWorkspaceObservation(target); + target.observationSetupComplete = false; + this.scheduleMissingFolderProbe(target); + this.logger.warn( + { cwd: target.cwd }, + "Workspace folder no longer exists; suspended git observation until it is back", + ); + } + + private isSuspendedWorkspaceTarget(target: WorkspaceGitTarget): boolean { + return ( + target.folderMissing && !target.closed && this.workspaceTargets.get(target.cwd) === target + ); + } + + private scheduleMissingFolderProbe(target: WorkspaceGitTarget): void { + if ( + !this.isSuspendedWorkspaceTarget(target) || + target.observationReensureTimer || + target.folderRestorePromise + ) { + return; + } + target.observationReensureTimer = setTimeout(() => { + target.observationReensureTimer = null; + void this.resumeWorkspaceIfFolderRestored(target); + }, WORKSPACE_GIT_OBSERVATION_REENSURE_INTERVAL_MS); + } + + private resumeWorkspaceIfFolderRestored(target: WorkspaceGitTarget): Promise { + if (!target.folderMissing) { + return Promise.resolve(); + } + target.folderRestorePromise ??= this.resumeWorkspaceWithRestoredFolder(target).finally(() => { + target.folderRestorePromise = null; + this.scheduleMissingFolderProbe(target); + }); + return target.folderRestorePromise; + } + + private async resumeWorkspaceWithRestoredFolder(target: WorkspaceGitTarget): Promise { + if (!(await this.deps.workspaceFolderExists(target.cwd))) { + return; + } + await target.observationSetupPromise; + if (target.refreshState.status === "in-flight") { + await target.refreshState.promise.catch(() => undefined); + } + if (!this.isSuspendedWorkspaceTarget(target)) { + return; + } + target.latestFacts = null; + target.folderMissing = false; + await this.refreshWorkspaceTarget(target, { + force: false, + refreshStructure: true, + refreshWorktree: true, + includeForge: false, + reason: "workspace-folder-restored", + queueIfBusy: true, + movedRemoteRefs: new Set(), + }); + if (target.closed || this.workspaceTargets.get(target.cwd) !== target) { + return; + } + target.observationSetupComplete = false; + if (target.observationReensureTimer) { + clearTimeout(target.observationReensureTimer); + target.observationReensureTimer = null; + } + this.logger.info({ cwd: target.cwd }, "Workspace folder is back; resumed git observation"); + this.startWorkspaceSubscriptionTimers(target); + this.scheduleWorkspaceObservationSetup(target); + } + private startWorkingTreeWatchFallback( target: WorkingTreeWatchTarget, reason: WorkingTreeWatchFallbackReason, @@ -1654,7 +1788,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { return changed; }), ); - this.notifyWorkingTreeConsumers(target); + if (await this.deps.workspaceFolderExists(target.watchPath)) { + this.notifyWorkingTreeConsumers(target); + } if (!target.closed && (target.subscription === null || target.repoRoot === null)) { target.fallbackPollQuietTickCount = changes.some(Boolean) ? 0 @@ -1731,6 +1867,10 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { if (target.closed || target.subscription) { return; } + if (!(await this.deps.workspaceFolderExists(target.watchPath))) { + this.scheduleWorkingTreeWatchRecovery(target); + return; + } await this.refreshWorkingTreeIgnoredDirectories(target); if (target.closed || target.subscription) { return; @@ -2786,7 +2926,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { } private updateForgePrStatusPollForTarget(target: WorkspaceGitTarget): void { - if (target.listeners.size === 0) { + if (!this.isActiveObservedWorkspaceTarget(target)) { this.stopForgePrStatusPollForTarget(target); return; } @@ -3049,6 +3189,9 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { target: WorkspaceGitTarget, request: WorkspaceGitRefreshRequest, ): Promise { + if (target.folderMissing) { + return Promise.resolve(target.latestSnapshot ?? buildNotGitSnapshot(target.cwd)); + } if (target.refreshState.status === "in-flight") { const active = target.refreshState.request; const addsWork = @@ -3191,11 +3334,19 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { try { const runRefreshGitCommand = createRunGitCommand(`workspace-refresh:${request.reason}`); const admittedSnapshot = await this.workspaceRefreshLimit(() => { - if (target.closed || this.workspaceTargets.get(target.cwd) !== target) { + if ( + target.closed || + target.folderMissing || + this.workspaceTargets.get(target.cwd) !== target + ) { return null; } return this.refreshSnapshot(target, request, runRefreshGitCommand); }); + if (target.folderMissing) { + failure = null; + break; + } if (!admittedSnapshot) { break; } @@ -3209,6 +3360,10 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { failure = { error }; } + if (target.folderMissing) { + failure = null; + break; + } if (this.disposed || target.closed || this.workspaceTargets.get(target.cwd) !== target) { break; } @@ -3504,7 +3659,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { snapshot: WorkspaceGitRuntimeSnapshot, options?: { forceEmit?: boolean }, ): void { - if (target.closed || this.workspaceTargets.get(target.cwd) !== target) { + if (target.closed || target.folderMissing || this.workspaceTargets.get(target.cwd) !== target) { return; } target.latestSnapshot = snapshot; @@ -3660,21 +3815,25 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { } private removeWorkspaceTarget(target: WorkspaceGitTarget): void { - if (target.repoGitRoot) { - const repoTarget = this.repoTargets.get(target.repoGitRoot); - repoTarget?.workspaceKeys.delete(target.cwd); - if (repoTarget && repoTarget.workspaceKeys.size === 0) { - this.closeRepoTarget(repoTarget); - this.repoTargets.delete(target.repoGitRoot); - } else if (repoTarget?.cwd === target.cwd) { - repoTarget.cwd = repoTarget.workspaceKeys.values().next().value ?? repoTarget.cwd; - } - } - + this.detachWorkspaceFromRepoTarget(target); this.closeWorkspaceTarget(target); this.workspaceTargets.delete(target.cwd); } + private detachWorkspaceFromRepoTarget(target: WorkspaceGitTarget): void { + if (!target.repoGitRoot) { + return; + } + const repoTarget = this.repoTargets.get(target.repoGitRoot); + repoTarget?.workspaceKeys.delete(target.cwd); + if (repoTarget && repoTarget.workspaceKeys.size === 0) { + this.closeRepoTarget(repoTarget); + this.repoTargets.delete(target.repoGitRoot); + } else if (repoTarget?.cwd === target.cwd) { + repoTarget.cwd = repoTarget.workspaceKeys.values().next().value ?? repoTarget.cwd; + } + } + private removeWorkingTreeWatchListener(cwd: string, listener: () => void): void { const target = this.workingTreeWatchTargets.get(cwd); if (!target) { @@ -3719,6 +3878,11 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { private closeWorkspaceTarget(target: WorkspaceGitTarget): void { target.closed = true; + this.stopWorkspaceObservation(target); + target.listeners.clear(); + } + + private stopWorkspaceObservation(target: WorkspaceGitTarget): void { if (target.workingTreeWatchTarget) { this.removeWorkspaceWorkingTreeLink(target.workingTreeWatchTarget, target.cwd); target.workingTreeWatchTarget = null; @@ -3727,12 +3891,12 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { clearTimeout(target.debounceTimer); target.debounceTimer = null; } + target.pendingDebounceRequest = null; if (target.observationReensureTimer) { clearTimeout(target.observationReensureTimer); target.observationReensureTimer = null; } this.stopForgePrStatusPollForTarget(target); - target.listeners.clear(); } private closeWorkingTreeWatchTarget(target: WorkingTreeWatchTarget): void {