From e998e0a08237290dbecb3960615ef1ca8701c17e Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 23 Sep 2026 22:20:17 +0200 Subject: [PATCH] Stop the daemon holding a CPU core on a repository the watcher can't observe (#5170) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(server): back off degraded git polling on an unchanged workspace When the recursive watcher declines a repository (over 250,000 entries on macOS/Windows, or any watcher setup failure), WorkspaceGit falls back to polling. That fallback rescheduled itself every 5 seconds for as long as the workspace existed, with no client or agent involved. On a repository large enough to defeat the watcher each tick is an expensive `git status`, so the daemon held about a core indefinitely. Double the gap after every tick whose workspace snapshot came back identical, up to a 60-second ceiling, and reset to the 5-second base on the first change. Both degraded loops — working tree and repository metadata — share the rule. Explicit reads still force a refresh, so the ceiling only bounds how long an edit made outside Paseo stays undiscovered. Fixes #5102 * test(server): cover repository-metadata degraded backoff, document failed-tick semantics * test(server): drive the recovery-refresh assertion off the fake clock, not waitFor * test(server): declare metadata poll counter as a function --- docs/file-observation.md | 2 +- .../workspace-git-service.observation.test.ts | 91 ++++++++++++++++++- .../src/server/workspace-git-service.ts | 62 +++++++++++-- 3 files changed, 141 insertions(+), 14 deletions(-) diff --git a/docs/file-observation.md b/docs/file-observation.md index 7e46e3342e..d7d1d8f2f0 100644 --- a/docs/file-observation.md +++ b/docs/file-observation.md @@ -10,7 +10,7 @@ Closing an observation must remain safe after its root is renamed or removed. Wo Linux topology repair is trailing-debounced and scoped to the renamed path. A file rename needs one classification stat; a new or moved directory scans only its subtree. Full-tree scans are reserved for startup and ignore-set replacement. The initial Linux scan conservatively emits existing files discovered before their directory watcher was installed; WorkspaceGit batches these into one refresh. Benchmark this startup cost with pre-populated trees. -Windows and macOS use one native recursive watcher plus an ignored-pruned, directory-indexed inventory. Native events deliver immediately. Named events trigger serialized shallow scans of their parent directories; known directory events also reconcile that subtree. This recovers concrete create and delete paths omitted by coalescing without walking the whole root for each edit. Classification is admission-controlled to 32 concurrent `fs.stat` calls, since `fs.stat` runs on the libuv threadpool the whole daemon shares. Startup and exclusion changes use an awaited full inventory barrier. Activity arms a full safety audit 30 seconds after it settles, with a 30-second minimum interval and a five-minute starvation bound; pathless native events also trigger an immediate shallow root scan. Observation fails into polling rather than retaining more than 250,000 file and directory entries per root. +Windows and macOS use one native recursive watcher plus an ignored-pruned, directory-indexed inventory. Native events deliver immediately. Named events trigger serialized shallow scans of their parent directories; known directory events also reconcile that subtree. This recovers concrete create and delete paths omitted by coalescing without walking the whole root for each edit. Classification is admission-controlled to 32 concurrent `fs.stat` calls, since `fs.stat` runs on the libuv threadpool the whole daemon shares. Startup and exclusion changes use an awaited full inventory barrier. Activity arms a full safety audit 30 seconds after it settles, with a 30-second minimum interval and a five-minute starvation bound; pathless native events also trigger an immediate shallow root scan. Observation fails into polling rather than retaining more than 250,000 file and directory entries per root. A repository that large is also expensive to poll, so degraded polling doubles its gap from 5 seconds to a 60-second ceiling while the workspace snapshot comes back unchanged, and resets to 5 seconds on the first change. Explicit reads still force a refresh; the ceiling only bounds how long an edit made outside Paseo stays undiscovered. Read aggregate health from the owning observer service. Runtime metrics include active observations, native handles, tracked native files, pending events and immediate reconciliation work, scoped and full reconciliation activity and latency, and failure counts. `nativeTrackedFileCount` and other per-file counters are native-backend diagnostics only — the Linux backend keeps no per-file inventory and reports zero — so a cross-platform assertion belongs on delivered events, not those counters. Delayed safety audits do not count as pending work. Closing the service releases every subscription and clears its diagnostics. Do not add path lists, watcher handles, or platform-specific controls to this interface. diff --git a/packages/server/src/server/workspace-git-service.observation.test.ts b/packages/server/src/server/workspace-git-service.observation.test.ts index c0534aceb6..f5bfbc344c 100644 --- a/packages/server/src/server/workspace-git-service.observation.test.ts +++ b/packages/server/src/server/workspace-git-service.observation.test.ts @@ -1539,6 +1539,87 @@ describe("WorkspaceGitService checkout observation", () => { service.dispose(); }); + test("degraded polling backs off while the snapshot is unchanged and resets on a change", async () => { + // A repository large enough to defeat the recursive watcher makes every degraded refresh + // expensive, so a fixed cadence keeps the daemon shelling out Git on an untouched workspace. + const watcher = createWatcherHarness({ failDirectories: new Set([REPO_CWD]) }); + let isDirty = false; + const getCheckoutStatus = vi.fn(async (cwd: string) => createCheckoutStatus(cwd, { isDirty })); + const service = createService(watcher, { getCheckoutStatus }); + const subscription = service.registerWorkspace({ cwd: REPO_CWD }, vi.fn()); + + await vi.waitFor(() => { + expect(service.getMetrics().workingTreeWatchTargetCount).toBe(1); + expect(getCheckoutStatus).toHaveBeenCalledTimes(1); + }); + + // Quiet ticks at 5s, 10s, 20s, 40s, then the 60s ceiling. + for (const [elapsedMs, expectedCalls] of [ + [5_000, 2], + [10_000, 3], + [20_000, 4], + [40_000, 5], + [60_000, 6], + [60_000, 7], + ] as const) { + await vi.advanceTimersByTimeAsync(elapsedMs); + expect(getCheckoutStatus).toHaveBeenCalledTimes(expectedCalls); + } + + // A fixed 5s cadence would have run 39 polls over the same 195s. + expect(getCheckoutStatus.mock.calls.length).toBeLessThan(10); + + // A real change snaps the loop back to the base interval. + isDirty = true; + await vi.advanceTimersByTimeAsync(60_000); + expect(getCheckoutStatus).toHaveBeenCalledTimes(8); + await vi.advanceTimersByTimeAsync(5_000); + expect(getCheckoutStatus).toHaveBeenCalledTimes(9); + + subscription.unsubscribe(); + service.dispose(); + }); + + test("degraded repository-metadata polling backs off and resets on a change", async () => { + // Only the Git-directory watcher fails, so the metadata fallback is the one poll loop running. + const watcher = createWatcherHarness({ failDirectories: new Set([GIT_DIR]) }); + let isDirty = false; + const getCheckoutStatus = vi.fn(async (cwd: string) => createCheckoutStatus(cwd, { isDirty })); + const service = createService(watcher, { getCheckoutStatus }); + const subscription = service.registerWorkspace({ cwd: REPO_CWD }, vi.fn()); + + await vi.waitFor(() => { + expect(service.getMetrics().workspaceObservationSetupInFlightCount).toBe(0); + expect(service.getMetrics().workspaceRefreshInFlightCount).toBe(0); + }); + const callsBeforePolling = getCheckoutStatus.mock.calls.length; + function pollCount(): number { + return getCheckoutStatus.mock.calls.length - callsBeforePolling; + } + + // Quiet ticks at 5s, 10s, 20s, 40s, then the 60s ceiling. + for (const [elapsedMs, expectedPolls] of [ + [5_000, 1], + [10_000, 2], + [20_000, 3], + [40_000, 4], + [60_000, 5], + [60_000, 6], + ] as const) { + await vi.advanceTimersByTimeAsync(elapsedMs); + expect(pollCount()).toBe(expectedPolls); + } + + isDirty = true; + await vi.advanceTimersByTimeAsync(60_000); + expect(pollCount()).toBe(7); + await vi.advanceTimersByTimeAsync(5_000); + expect(pollCount()).toBe(8); + + subscription.unsubscribe(); + service.dispose(); + }); + test("non-Git fallback promotes an externally initialized checkout", async () => { const watcher = createWatcherHarness(); let isGit = false; @@ -1752,17 +1833,17 @@ describe("WorkspaceGitService checkout observation", () => { expect(service.getMetrics().workspaceObservationSetupInFlightCount).toBe(0); }); const statusCallsAfterSetup = getCheckoutStatus.mock.calls.length; - await vi.advanceTimersByTimeAsync(5_000); - await vi.waitFor(() => { - expect(getCheckoutStatus.mock.calls.length).toBeGreaterThan(statusCallsAfterSetup); - }); - await vi.advanceTimersByTimeAsync(24_000); + await vi.advanceTimersByTimeAsync(29_000); expect(getWatcherSubscribeCallCount(watcher, REPO_CWD)).toBe(1); await vi.advanceTimersByTimeAsync(1_000); await vi.waitFor(() => { expect(getWatcherSubscribeCallCount(watcher, REPO_CWD)).toBe(2); }); expect(erroredUnsubscribe).toHaveBeenCalledTimes(1); + // Recovery schedules a debounced refresh. Advance the fake clock past the debounce + // explicitly; vi.waitFor alone runs out of real time before it on a slow runner. + await vi.advanceTimersByTimeAsync(2_000); + expect(getCheckoutStatus.mock.calls.length).toBeGreaterThan(statusCallsAfterSetup); subscription.unsubscribe(); service.dispose(); diff --git a/packages/server/src/server/workspace-git-service.ts b/packages/server/src/server/workspace-git-service.ts index eed5fca2ec..e62720e1c1 100644 --- a/packages/server/src/server/workspace-git-service.ts +++ b/packages/server/src/server/workspace-git-service.ts @@ -86,6 +86,13 @@ const FORGE_PR_STATUS_POLL_FAST_INTERVAL_MS = 20_000; const FORGE_PR_STATUS_POLL_SLOW_INTERVAL_MS = 120_000; const FORGE_PR_STATUS_POLL_ERROR_BACKOFF_CAP_MS = 300_000; const DEGRADED_GIT_POLL_INTERVAL_MS = 5_000; +// Degraded polling shells out a full Git refresh per tick, and the repositories that defeat the +// recursive watcher are the ones where that refresh is expensive. At a fixed cadence the daemon +// therefore burns a core forever on a workspace nobody has touched. Double the gap after every +// tick that produced an identical snapshot, and drop back to the base interval as soon as one +// does not. Explicit reads still force a refresh, so this only bounds background discovery of +// changes made outside Paseo. +const DEGRADED_GIT_POLL_MAX_INTERVAL_MS = 60_000; // A dependency install creates directories for minutes. Collapse the burst into // a couple of `git ls-files` runs rather than one per event. const WORKING_TREE_IGNORE_REFRESH_QUIET_MS = 300; @@ -442,6 +449,7 @@ interface RepoGitTarget { subscription: FileObserverSubscription | null; fallbackPolling: boolean; fallbackPollTimer: NodeJS.Timeout | null; + fallbackPollQuietTickCount: number; recovery: WatchRecoveryState; intervalId: NodeJS.Timeout | null; fetchInFlight: boolean; @@ -501,6 +509,7 @@ interface WorkingTreeWatchTarget { workspaceKeys: Set; fallbackPolling: boolean; fallbackPollTimer: NodeJS.Timeout | null; + fallbackPollQuietTickCount: number; recovery: WatchRecoveryState; listeners: Set<() => void>; closed: boolean; @@ -1352,6 +1361,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { workspaceKeys: new Set(), fallbackPolling: false, fallbackPollTimer: null, + fallbackPollQuietTickCount: 0, recovery: { attemptCount: 0, timer: null, establishedAt: null }, listeners: new Set(), closed: false, @@ -1558,6 +1568,25 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { } } + private degradedPollDelayMs(quietTickCount: number): number { + return Math.min( + DEGRADED_GIT_POLL_MAX_INTERVAL_MS, + DEGRADED_GIT_POLL_INTERVAL_MS * 2 ** quietTickCount, + ); + } + + // A refresh that throws is swallowed by refreshWorkspaceTarget and leaves the fingerprint + // alone, so it reads as a quiet tick and widens the gap. That is the intended outcome: a + // repository whose Git commands keep failing is the last one to retry every 5 seconds. + private async refreshDegradedPollTarget( + workspaceTarget: WorkspaceGitTarget, + request: WorkspaceGitRefreshRequest, + ): Promise { + const fingerprintBefore = workspaceTarget.latestFingerprint; + await this.refreshWorkspaceTarget(workspaceTarget, request); + return workspaceTarget.latestFingerprint !== fingerprintBefore; + } + private startWorkingTreeWatchFallback( target: WorkingTreeWatchTarget, reason: WorkingTreeWatchFallbackReason, @@ -1566,19 +1595,20 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { return; } target.fallbackPolling = true; + target.fallbackPollQuietTickCount = 0; const { cwd } = target; const poll = async () => { target.fallbackPollTimer = null; if (target.closed || this.workingTreeWatchTargets.get(target.cwd) !== target) { return; } - await Promise.all( + const changes = await Promise.all( Array.from(target.workspaceKeys, async (workspaceKey) => { const workspaceTarget = this.workspaceTargets.get(workspaceKey); if (!workspaceTarget) { - return; + return false; } - await this.refreshWorkspaceTarget(workspaceTarget, { + const changed = await this.refreshDegradedPollTarget(workspaceTarget, { force: false, refreshStructure: target.repoRoot === null, refreshWorktree: true, @@ -1592,11 +1622,18 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { workspaceTarget.observationSetupComplete = false; this.scheduleWorkspaceObservationSetup(workspaceTarget); } + return changed; }), ); this.notifyWorkingTreeConsumers(target); if (!target.closed && (target.subscription === null || target.repoRoot === null)) { - target.fallbackPollTimer = setTimeout(poll, DEGRADED_GIT_POLL_INTERVAL_MS); + target.fallbackPollQuietTickCount = changes.some(Boolean) + ? 0 + : target.fallbackPollQuietTickCount + 1; + target.fallbackPollTimer = setTimeout( + poll, + this.degradedPollDelayMs(target.fallbackPollQuietTickCount), + ); } else { target.fallbackPolling = false; } @@ -1606,6 +1643,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { { cwd, intervalMs: DEGRADED_GIT_POLL_INTERVAL_MS, + maxIntervalMs: DEGRADED_GIT_POLL_MAX_INTERVAL_MS, reason, }, "Working tree watcher unavailable; using bounded polling fallback", @@ -2107,6 +2145,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { subscription: null, fallbackPolling: false, fallbackPollTimer: null, + fallbackPollQuietTickCount: 0, recovery: { attemptCount: 0, timer: null, establishedAt: null }, intervalId: null, fetchInFlight: false, @@ -2607,17 +2646,18 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { return; } target.fallbackPolling = true; + target.fallbackPollQuietTickCount = 0; const poll = async () => { target.fallbackPollTimer = null; if (target.closed || this.repoTargets.get(target.repoGitRoot) !== target) { return; } const workingTreeTargets = new Set(); - await Promise.all( + const changes = await Promise.all( Array.from(target.workspaceKeys, async (workspaceKey) => { const workspaceTarget = this.workspaceTargets.get(workspaceKey); if (!workspaceTarget) { - return; + return false; } this.invalidateCheckoutDiffCache(workspaceTarget.cwd, "base"); if (workspaceTarget.latestFacts?.isGit) { @@ -2627,7 +2667,7 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { if (workingTreeTarget) { workingTreeTargets.add(workingTreeTarget); } - await this.refreshWorkspaceTarget(workspaceTarget, { + return await this.refreshDegradedPollTarget(workspaceTarget, { force: false, refreshStructure: true, refreshWorktree: true, @@ -2644,7 +2684,13 @@ export class WorkspaceGitServiceImpl implements WorkspaceGitService { this.notifyWorkingTreeConsumers(workingTreeTarget); } if (!target.closed && target.subscription === null) { - target.fallbackPollTimer = setTimeout(poll, DEGRADED_GIT_POLL_INTERVAL_MS); + target.fallbackPollQuietTickCount = changes.some(Boolean) + ? 0 + : target.fallbackPollQuietTickCount + 1; + target.fallbackPollTimer = setTimeout( + poll, + this.degradedPollDelayMs(target.fallbackPollQuietTickCount), + ); } else { target.fallbackPolling = false; }