diff --git a/docs/data-model.md b/docs/data-model.md index 3679d834eb..f438ffd8e8 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -582,12 +582,17 @@ host, kinds, and ids. Opening a cached workspace uses exact workspace and projec directory scan. One invalid row is deleted and returned as a miss without affecting other rows. An invalid directory row and its affected checkpoint cursor are repaired in one transaction, so a later launch cannot accept a checkpoint for a partial baseline. Directory changes and their -checkpoint are also applied in one transaction. Reads wait for accepted changes to reach storage, +checkpoint are also applied in one transaction. A stored checkpoint carries a version. When a writer +bug could have stored a checkpoint without its directory, bump the version: older checkpoints are +rejected on read, cached rows still paint, and each host re-syncs its directory once. Reads wait for +accepted changes to reach storage, then retry only when a requested row changes during the read. Unrelated agent streams do not invalidate the read. Directory baseline replacements and host removal or renaming still invalidate the affected reads. -The cache is capped at 32 MiB and evicts whole hosts in least-recently-written order. Budget +The cache is capped at 32 MiB and evicts whole hosts in least-recently-written order. After an +eviction, only a baseline replacement stores that host's directory again. Mutations queued around +the eviction are dropped when they are written, so a checkpoint never outlives its rows. Budget bookkeeping may scan opaque row sizes during a deferred write, never during host registry startup or before a requested cache row can paint. The row store is not encrypted. A cached timeline can contain source code, prompts, and tool output; encrypted-at-rest storage is a separate security decision. diff --git a/packages/app/e2e/browser/sidebar-workspace.spec.ts b/packages/app/e2e/browser/sidebar-workspace.spec.ts index a307002c3b..b445744046 100644 --- a/packages/app/e2e/browser/sidebar-workspace.spec.ts +++ b/packages/app/e2e/browser/sidebar-workspace.spec.ts @@ -206,13 +206,14 @@ async function simulateDamagedLegacyDirectoryCache( return; } const stored = JSON.parse(checkpoint.payload); - // Accept either writer so this fixture also reproduces on the unfixed client. - const cursors = stored.version === 1 ? stored.cursors : stored; - if (!cursors.workspaces?.generation) { + // Read the cursors from whichever version wrote them, then store them as version 1: + // the format whose writer could keep a checkpoint after evicting the rows it describes. + const cursors = typeof stored.version === "number" ? stored.cursors : stored; + if (!cursors?.workspaces?.generation) { transaction.abort(); return; } - store.put({ ...checkpoint, payload: JSON.stringify(cursors) }); + store.put({ ...checkpoint, payload: JSON.stringify({ version: 1, cursors }) }); for (const row of rows) { if ( row.serverId === serverId && diff --git a/packages/app/src/runtime/directory-sync/index.test.ts b/packages/app/src/runtime/directory-sync/index.test.ts index e4db5dfb71..61b15ad5ec 100644 --- a/packages/app/src/runtime/directory-sync/index.test.ts +++ b/packages/app/src/runtime/directory-sync/index.test.ts @@ -10,7 +10,10 @@ import type { DaemonClient, WorkspaceLabelListPayload, } from "@getpaseo/client/internal/daemon-client"; -import type { SessionOutboundMessage } from "@getpaseo/protocol/messages"; +import type { + SessionOutboundMessage, + WorkspaceDescriptorPayload, +} from "@getpaseo/protocol/messages"; import { normalizeProjectDescriptor, normalizeWorkspaceDescriptor, @@ -1667,6 +1670,187 @@ it("fills every cached workspace beneath live updates received during the SQLite database.close(); }); +/** + * A host whose cache was damaged before the eviction fix: workspaces survived, project rows did not, + * and a version 1 checkpoint still claims the project directory is current. The daemon holds + * projects A and B; renaming A is its update at `renameSeq`. + */ +async function createDamagedLegacyProjectDirectory(serverId: string, renameSeq: number) { + serverIds.add(serverId); + const { cache, database, storage, reopen, holdRead } = createSqliteCache(); + cache.setHosts([serverId]); + const workspacePayload = (id: string, projectId: string): WorkspaceDescriptorPayload => ({ + id, + projectId, + projectDisplayName: projectId, + projectRootPath: `/repo/${projectId}`, + workspaceDirectory: `/repo/${projectId}`, + projectKind: "git", + workspaceKind: "local_checkout", + name: id, + status: "done", + statusEnteredAt: null, + activityAt: null, + archivingAt: null, + diffStat: null, + scripts: [], + }); + const project = (projectId: string, projectDisplayName: string) => ({ + projectId, + projectDisplayName, + projectRootPath: `/repo/${projectId}`, + projectKind: "git" as const, + }); + const workspaceEntries = [workspacePayload("wa", "A"), workspacePayload("wb", "B")]; + cache.replaceDirectoryBaseline(serverId, { + agents: new Map(), + workspaces: new Map( + workspaceEntries.map((entry) => [entry.id, normalizeWorkspaceDescriptor(entry)]), + ), + projects: new Map(), + }); + await cache.flush(); + await storage.apply({ + deletes: [], + upserts: [ + { + serverId, + kind: "checkpoint", + id: "singleton", + payload: JSON.stringify({ + version: 1, + cursors: { projects: { generation: "g", afterSeq: renameSeq - 1 } }, + }), + }, + ], + }); + const reopened = reopen(); + reopened.setHosts([serverId]); + const client = new FakeDirectoryClient(); + let failNextProjectRequest = false; + client.listProjects = async (options?: unknown): Promise => { + client.lastProjectOptions = options; + if (failNextProjectRequest) { + failNextProjectRequest = false; + throw new Error("Request timed out"); + } + const resumesAfterRename = + (options as { sync?: { afterSeq?: number } } | undefined)?.sync?.afterSeq === renameSeq; + return resumesAfterRename + ? { + requestId: "projects", + projects: [], + sync: { generation: "g", headSeq: renameSeq, mode: "changes", removals: [] }, + } + : { + requestId: "projects", + projects: [project("A", "Renamed"), project("B", "B")], + sync: { generation: "g", headSeq: renameSeq, mode: "snapshot", removals: [] }, + }; + }; + client.fetchWorkspaces = async (): Promise => ({ + requestId: "workspaces", + entries: workspaceEntries, + emptyProjects: [], + pageInfo: { hasMore: false, nextCursor: null, prevCursor: null }, + sync: { generation: "g", headSeq: 2, mode: "snapshot", removals: [] }, + }); + const directory = new DirectorySync( + serverId, + { + onAgentStoppedRunning: () => {}, + markAgentLoading: () => {}, + markAgentReady: () => {}, + markAgentError: () => {}, + }, + reopened, + ); + const store = useSessionStore.getState(); + store.initializeSession(serverId, client as unknown as DaemonClient, 1); + store.updateSessionServerInfo(serverId, { + serverId, + hostname: null, + version: "test", + features: { workspaceMultiplicity: true, directorySync: true, projectList: true }, + }); + directory.connectionChanged({ + client: client as unknown as DaemonClient, + status: "online", + source: { clientGeneration: 1, connectionEpoch: 1 }, + }); + return { + client, + directory, + holdRead, + failNextProjectRequest: () => { + failNextProjectRequest = true; + }, + renameProjectA: () => + client.emit({ + type: "project.update", + payload: { + kind: "upsert", + project: project("A", "Renamed"), + generation: "g", + seq: renameSeq, + }, + }), + sessionProjectIds: () => + [...useSessionStore.getState().sessions[serverId].projects.keys()].sort(), + persistedProjectIds: async () => { + await reopened.flush(); + const restarted = reopen(); + restarted.setHosts([serverId]); + return [...(await restarted.readDirectory(serverId)).projects.keys()].sort(); + }, + close: async () => { + directory.dispose(); + await reopened.flush(); + database.close(); + }, + }; +} + +it("re-syncs every project when a project update lands while an untrusted cache loads", async () => { + const damaged = await createDamagedLegacyProjectDirectory("untrusted-checkpoint-race", 6); + try { + let renamed = false; + damaged.holdRead(async () => { + if (renamed) return; + renamed = true; + damaged.renameProjectA(); + }); + + damaged.directory.setDemand({}, true); + await damaged.directory.refreshDemand(); + + expect(renamed).toBe(true); + expect(damaged.client.lastProjectOptions).toEqual({ sync: {} }); + expect(damaged.sessionProjectIds()).toEqual(["A", "B"]); + expect(await damaged.persistedProjectIds()).toEqual(["A", "B"]); + } finally { + await damaged.close(); + } +}); + +it("re-syncs every project when a project update lands after the first sync fails", async () => { + const damaged = await createDamagedLegacyProjectDirectory("failed-first-sync", 3); + try { + damaged.failNextProjectRequest(); + damaged.directory.setDemand({}, true); + await expect(damaged.directory.refreshDemand()).rejects.toThrow("Request timed out"); + + damaged.renameProjectA(); + await damaged.directory.refreshDemand(); + + expect(damaged.client.lastProjectOptions).toEqual({ sync: {} }); + expect(damaged.sessionProjectIds()).toEqual(["A", "B"]); + expect(await damaged.persistedProjectIds()).toEqual(["A", "B"]); + } finally { + await damaged.close(); + } +}); + function createWorkspaceLabelDirectory(serverId: string) { const { client, directory } = createDirectory(serverId, { workspaceLabels: true }); const store = useSessionStore.getState(); diff --git a/packages/app/src/runtime/directory-sync/index.ts b/packages/app/src/runtime/directory-sync/index.ts index 0f539a2b1d..b9114deb21 100644 --- a/packages/app/src/runtime/directory-sync/index.ts +++ b/packages/app/src/runtime/directory-sync/index.ts @@ -288,7 +288,7 @@ export class DirectorySync { this.revision += 1; if (this.agentTransactions.record(source, delta)) return; this.agents.applyDelta(delta); - this.noteLiveCursor("agents", delta); + if (this.extendsRefreshedBaseline(source)) this.noteLiveCursor("agents", delta); this.persistCheckpoint(); } @@ -300,7 +300,7 @@ export class DirectorySync { this.workspaceRevision += 1; if (this.workspaceTransactions.record(source, delta)) return; this.applyWorkspaceDelta(delta); - if (delta.kind !== "script_status") + if (delta.kind !== "script_status" && this.extendsRefreshedBaseline(source)) this.noteLiveCursor( "projectId" in delta || "project" in delta ? "projects" : "workspaces", delta, @@ -308,6 +308,20 @@ export class DirectorySync { this.persistCheckpoint(); } + /** + * A live event's sequence covers everything before it only when a refresh completed on the + * subscriptions that delivered it. Before that (first sync still pending or failed) or after + * they lapse, the client may have missed earlier changes, so the event updates rows but leaves + * the cursor where the next refresh can still fetch what was missed. + */ + private extendsRefreshedBaseline(source: DirectorySourceToken): boolean { + const refreshed = this.satisfiedDemandSource; + return ( + refreshed?.clientGeneration === source.clientGeneration && + refreshed.connectionEpoch === source.connectionEpoch + ); + } + private observeDirectoryEvents(): void { if (this.eventSubscription) return; const { client, source } = this.requireOnline(); @@ -566,24 +580,16 @@ export class DirectorySync { input: { subscribe?: boolean } | undefined, loadDirectoryCache: boolean, ): Promise { - if (loadDirectoryCache) await this.loadCachedDirectory(); + if (loadDirectoryCache) { + this.primeWorkspaceTransactionForCacheLoad(); + await this.loadCachedDirectory(); + } const onlineConnection = this.getOnlineConnection(); if (!onlineConnection) return; const { client, source } = onlineConnection; - const transaction = this.workspaceTransactions.begin(source, () => { - const baseline = this.workspaces.snapshot(); - return { - workspaces: new Map(baseline.workspaces), - projects: new Map(baseline.projects), - // Cache hydration can advance the live replica while these requests await responses. - // Every page must use the cursors belonging to the maps captured here. - requestCursors: { ...this.cursors }, - syncCursors: {}, - syncModes: {}, - touchedWorkspaceIds: new Set(), - touchedProjectIds: new Set(), - }; - }); + const transaction = this.workspaceTransactions.begin(source, () => + this.createWorkspaceRefreshSnapshot(), + ); try { await this.waitForSessionMetadata(client, source); const serverInfo = useSessionStore.getState().sessions[this.serverId]?.serverInfo; @@ -807,6 +813,31 @@ export class DirectorySync { })); } + // Updates that arrive before a refresh completes have no trusted baseline to advance from. + // Holding them keeps their cursor out of the cache until the refresh they are applied over. + private primeWorkspaceTransactionForCacheLoad(): void { + const connection = this.getOnlineConnection(); + if (!connection) return; + this.workspaceTransactions.begin(connection.source, () => + this.createWorkspaceRefreshSnapshot(), + ); + } + + private createWorkspaceRefreshSnapshot(): WorkspaceRefreshSnapshot { + const baseline = this.workspaces.snapshot(); + return { + workspaces: new Map(baseline.workspaces), + projects: new Map(baseline.projects), + // Cache hydration can advance the live replica while these requests await responses. + // Every page must use the cursors belonging to the maps captured here. + requestCursors: { ...this.cursors }, + syncCursors: {}, + syncModes: {}, + touchedWorkspaceIds: new Set(), + touchedProjectIds: new Set(), + }; + } + private supportsDirectorySync(): boolean { return ( useSessionStore.getState().sessions[this.serverId]?.serverInfo?.features?.directorySync === diff --git a/packages/app/src/runtime/replica-cache/index.test.ts b/packages/app/src/runtime/replica-cache/index.test.ts index f0187e8319..6c45f2fdf0 100644 --- a/packages/app/src/runtime/replica-cache/index.test.ts +++ b/packages/app/src/runtime/replica-cache/index.test.ts @@ -37,6 +37,9 @@ class MemoryStorage implements ReplicaRowStore { onRead: (() => void) | null = null; /** Throws once a read goes past this count, so a non-terminating read loop fails the test. */ readLimit: number | null = null; + /** Awaited before the store lists every row or deletes a host, so a test can interleave there. */ + beforeReadAll: (() => Promise) | null = null; + beforeDeleteHost: (() => Promise) | null = null; private key(row: Pick): string { return `${row.serverId}:${row.kind}:${row.id}`; @@ -68,6 +71,7 @@ class MemoryStorage implements ReplicaRowStore { } async readAll(): Promise { + await this.beforeReadAll?.(); const hosts = new Map(); for (const row of this.rows.values()) { const rows = hosts.get(row.serverId) ?? []; @@ -91,6 +95,7 @@ class MemoryStorage implements ReplicaRowStore { } async deleteHost(serverId: string): Promise { + await this.beforeDeleteHost?.(); for (const [key, row] of this.rows) if (row.serverId === serverId) this.rows.delete(key); } @@ -436,40 +441,49 @@ describe("ReplicaCache", () => { await expectLegacyWorkspaceRefresh(cached); }); - it("rejects pre-baseline-fix checkpoints without discarding cached rows or timelines", async () => { - const storage = new MemoryStorage(); - const writer = createCache(storage); - const baseline = directory(); - writer.replaceDirectoryBaseline(SERVER_ID, baseline); - writer.commitTimeline(SERVER_ID, "agent-1", timeline()); - await writer.flush(); - await storage.apply({ - deletes: [], - upserts: [ - { - serverId: SERVER_ID, - kind: "checkpoint", - id: "singleton", - payload: JSON.stringify(baseline.checkpoint), - }, - ], - }); + it.each([ + ["before the baseline fix", (checkpoint: DirectoryCheckpoint): unknown => checkpoint], + [ + "before the eviction fix", + (checkpoint: DirectoryCheckpoint): unknown => ({ version: 1, cursors: checkpoint }), + ], + ])( + "rejects checkpoints written %s without discarding cached rows or timelines", + async (_era, legacyPayload) => { + const storage = new MemoryStorage(); + const writer = createCache(storage); + const baseline = directory(); + writer.replaceDirectoryBaseline(SERVER_ID, baseline); + writer.commitTimeline(SERVER_ID, "agent-1", timeline()); + await writer.flush(); + await storage.apply({ + deletes: [], + upserts: [ + { + serverId: SERVER_ID, + kind: "checkpoint", + id: "singleton", + payload: JSON.stringify(legacyPayload(baseline.checkpoint)), + }, + ], + }); - const reader = createCache(storage); - const restored = await reader.readDirectory(SERVER_ID); - expect(restored.checkpoint).toBeUndefined(); - expect([...restored.workspaces.keys()]).toEqual([...baseline.workspaces.keys()]); - expect([...restored.projects.keys()]).toEqual([...baseline.projects.keys()]); - expect([...restored.agents.keys()]).toEqual([...baseline.agents.keys()]); - expect(await reader.readTimeline(SERVER_ID, "agent-1")).toEqual(timeline()); - expect(await storage.read(SERVER_ID, ["checkpoint"])).toEqual([]); + const reader = createCache(storage); + const restored = await reader.readDirectory(SERVER_ID); + expect(restored.checkpoint).toBeUndefined(); + expect([...restored.workspaces.keys()]).toEqual([...baseline.workspaces.keys()]); + expect([...restored.projects.keys()]).toEqual([...baseline.projects.keys()]); + expect([...restored.agents.keys()]).toEqual([...baseline.agents.keys()]); + expect(await reader.readTimeline(SERVER_ID, "agent-1")).toEqual(timeline()); + expect(await storage.read(SERVER_ID, ["checkpoint"])).toEqual([]); - reader.replaceDirectoryBaseline(SERVER_ID, baseline); - await reader.flush(); - expect((await createCache(storage).readDirectory(SERVER_ID)).checkpoint).toEqual( - baseline.checkpoint, - ); - }); + reader.replaceDirectoryBaseline(SERVER_ID, baseline); + await reader.flush(); + expect((await createCache(storage).readDirectory(SERVER_ID)).checkpoint).toEqual( + baseline.checkpoint, + ); + }, + ); it("does nothing until an owner explicitly commits data", async () => { const storage = new MemoryStorage(); @@ -1102,6 +1116,113 @@ describe("ReplicaCache", () => { ).toEqual(["agent", "checkpoint", "project", "workspace"]); }); + it.each(["beforeReadAll", "beforeDeleteHost"] as const)( + "stores no checkpoint for a host evicted after its update was queued (%s)", + async (pausePoint) => { + const storage = new MemoryStorage(); + const previousSession = new ReplicaCache(storage, noLegacyCleanup); + previousSession.setHosts([SERVER_ID, "other-host"]); + previousSession.replaceDirectoryBaseline(SERVER_ID, directory()); + previousSession.replaceDirectoryBaseline("other-host", directory()); + await previousSession.flush(); + + const cache = new ReplicaCache(storage, { ...noLegacyCleanup, maxBytes: 8_192 }); + cache.setHosts([SERVER_ID, "other-host"]); + const paused = deferred(); + const resume = deferred(); + storage[pausePoint] = async () => { + storage[pausePoint] = null; + paused.resolve(); + await resume.promise; + }; + // Larger than the whole budget, so this persist evicts both hosts. + cache.commitTimeline(SERVER_ID, "agent-1", timeline("x".repeat(16_384))); + const eviction = cache.flush(); + await paused.promise; + cache.commitDirectoryMutations(SERVER_ID, [], { agents: { generation: "g", afterSeq: 13 } }); + resume.resolve(); + await eviction; + await cache.flush(); + + const restored = await createCache(storage).readDirectory(SERVER_ID); + expect(restored.agents.size).toBe(0); + expect(restored.checkpoint).toBeUndefined(); + }, + ); + + it("stores no checkpoint for a host evicted while a malformed row is repaired", async () => { + const { storage } = await cacheWithoutBackgroundVisibility(); + const reader = createCache(storage, 8_192); + storage.onRead = () => { + if (!storage.reads.at(-1)?.kinds.includes("checkpoint")) return; + storage.onRead = null; + // Larger than the whole budget, so this persist evicts the host before the repair writes. + reader.commitTimeline(SERVER_ID, "agent-1", timeline("x".repeat(16_384))); + void reader.flush(); + }; + + expect(await reader.readWorkspace(SERVER_ID, "workspace-1")).toBeUndefined(); + await reader.flush(); + + const restored = await createCache(storage).readDirectory(SERVER_ID); + expect(restored.projects.size).toBe(0); + expect(restored.checkpoint).toBeUndefined(); + }); + + it("repairs a malformed row's cursor when the overlapping eviction fails", async () => { + const { storage } = await cacheWithoutBackgroundVisibility(); + const reader = createCache(storage, 8_192); + storage.onRead = () => { + if (!storage.reads.at(-1)?.kinds.includes("checkpoint")) return; + storage.onRead = null; + storage.beforeDeleteHost = async () => { + storage.beforeDeleteHost = null; + throw new Error("Eviction aborted"); + }; + reader.commitTimeline(SERVER_ID, "agent-1", timeline("x".repeat(16_384))); + void reader.flush(); + }; + + expect(await reader.readWorkspace(SERVER_ID, "workspace-1")).toBeUndefined(); + // A newer timeline fits the budget, so no later eviction clears the checkpoint. + reader.commitTimeline(SERVER_ID, "agent-1", timeline()); + await reader.flush(); + + const restored = await createCache(storage).readDirectory(SERVER_ID); + expect(restored.workspaces.size).toBe(0); + expect(restored.checkpoint).toEqual({ + agents: { generation: "g", afterSeq: 12 }, + projects: { generation: "g", afterSeq: 4 }, + }); + }); + + it("keeps an update queued behind the baseline that rebuilds an evicted host", async () => { + const storage = new MemoryStorage(); + const cache = new ReplicaCache(storage, { ...noLegacyCleanup, maxBytes: 2_500 }); + cache.setHosts([SERVER_ID, "other-host"]); + const baseline = directory(); + cache.replaceDirectoryBaseline(SERVER_ID, baseline); + await cache.flush(); + cache.replaceDirectoryBaseline("other-host", baseline); + await cache.flush(); + expect((await cache.readDirectory(SERVER_ID)).agents.size).toBe(0); + + const renamed = { ...agent(), title: "Renamed agent" }; + cache.replaceDirectoryBaseline(SERVER_ID, baseline); + cache.commitDirectoryMutations( + SERVER_ID, + [{ kind: "agent", type: "upsert", id: renamed.id, value: renamed }], + { agents: { generation: "g", afterSeq: 13 } }, + ); + await cache.flush(); + cache.commitDirectoryMutations(SERVER_ID, [], { agents: { generation: "g", afterSeq: 14 } }); + await cache.flush(); + + const restored = await createCache(storage).readDirectory(SERVER_ID); + expect(restored.checkpoint).toEqual({ agents: { generation: "g", afterSeq: 14 } }); + expect(restored.agents.get(renamed.id)?.title).toBe("Renamed agent"); + }); + it("runs legacy cleanup once when storage is first used", async () => { const storage = new MemoryStorage(); const cache = new ReplicaCache(storage, { diff --git a/packages/app/src/runtime/replica-cache/index.ts b/packages/app/src/runtime/replica-cache/index.ts index c8f1cb6fd1..ee91f3145e 100644 --- a/packages/app/src/runtime/replica-cache/index.ts +++ b/packages/app/src/runtime/replica-cache/index.ts @@ -397,15 +397,19 @@ const DirectoryCheckpointSchema = z.strictObject({ agents: DirectoryCursorSchema.optional(), }); -// Old checkpoints could describe a baseline that an overlapping refresh had discarded. -// Revalidate directory contents once without evicting cached rows or timelines. +// A checkpoint is trusted only when this version wrote it. Earlier writers could store one for a +// directory the cache no longer held: unversioned ones after an overlapping refresh, version 1 +// after an eviction. Rejecting them revalidates directory contents once without evicting cached +// rows or timelines. +const DIRECTORY_CHECKPOINT_VERSION = 2; + const StoredDirectoryCheckpointSchema = z.strictObject({ - version: z.literal(1), + version: z.literal(DIRECTORY_CHECKPOINT_VERSION), cursors: DirectoryCheckpointSchema, }); function serializeDirectoryCheckpoint(cursors: DirectoryCheckpoint) { - return { version: 1 as const, cursors }; + return { version: DIRECTORY_CHECKPOINT_VERSION, cursors }; } function deserializeDirectoryCheckpoint(payload: string): DirectoryCheckpoint { @@ -968,6 +972,7 @@ export class ReplicaCache { private pendingUpserts = new Map(); private pendingDeletes = new Map(); private pendingBaselines = new Set(); + // Hosts whose stored directory was evicted. Only a baseline stores their directory again. private readonly invalidatedHosts = new Set(); private readonly maxBytes: number; private totalBytes = 0; @@ -1109,11 +1114,7 @@ export class ReplicaCache { private async deleteInvalidRow(row: ReplicaRow): Promise { const invalidEntity = directoryEntityForRow(row); if (!invalidEntity) { - const changes = { upserts: [], deletes: [row] } satisfies ReplicaRowChanges; - await this.queueOperation(async () => { - await this.rowStore.apply(changes); - this.applyStoredChanges(changes); - }); + await this.applyRepair(row.serverId, { upserts: [], deletes: [row] }); return; } @@ -1144,10 +1145,7 @@ export class ReplicaCache { ] : [], }; - await this.queueOperation(async () => { - await this.rowStore.apply(changes); - this.applyStoredChanges(changes); - }); + await this.applyRepair(row.serverId, changes); } private async repairInvalidDirectoryRows( @@ -1170,9 +1168,16 @@ export class ReplicaCache { ] : [], }; - await this.queueOperation(async () => { - await this.rowStore.apply(changes); - this.applyStoredChanges(changes); + await this.applyRepair(serverId, changes); + } + + // A repair is queued behind writes that can evict its host. If one did, the host has no + // directory left for the repaired checkpoint to describe, so only the deletes still apply. + private applyRepair(serverId: string, changes: ReplicaRowChanges): Promise { + return this.queueOperation(async () => { + const applied = this.invalidatedHosts.has(serverId) ? { ...changes, upserts: [] } : changes; + await this.rowStore.apply(applied); + this.applyStoredChanges(applied); }); } @@ -1182,7 +1187,6 @@ export class ReplicaCache { checkpoint?: DirectoryCheckpoint, ): void { if (!this.activeServerIds.has(serverId)) return; - if (this.invalidatedHosts.has(serverId)) return; for (const mutation of mutations) { if (mutation.type === "delete") { this.queueEntityDelete(serverId, mutation.kind, mutation.id); @@ -1412,9 +1416,17 @@ export class ReplicaCache { } private materializePendingChanges(pending: PendingReplicaChanges): ReplicaRowChanges { - const deletes = new Map(pending.deletes.map((row) => [pendingRowKey(row), row])); + // Decided here, inside the serialized write, because eviction happens there too. A mutation + // queued before its host was evicted would otherwise store a checkpoint without its directory. + const isStorable = (row: ReplicaRowKey): boolean => + row.kind === "timeline" || + !this.invalidatedHosts.has(row.serverId) || + pending.baselines.has(row.serverId); + const deletes = new Map( + pending.deletes.filter(isStorable).map((row) => [pendingRowKey(row), row]), + ); const upserts: ReplicaRow[] = []; - for (const upsert of pending.upserts) { + for (const upsert of pending.upserts.filter(isStorable)) { const row = this.materializeUpsert(upsert); if (row) upserts.push(row); else deletes.set(pendingRowKey(upsert), upsert); @@ -1563,8 +1575,10 @@ export class ReplicaCache { projectedTotal -= bytes; projectedBytes.delete(serverId); evicted.add(serverId); - this.invalidatedHosts.add(serverId); if (this.storedRows.has(serverId)) await this.rowStore.deleteHost(serverId); + // Only after the delete commits: a failed eviction leaves the rows stored, and writers that + // check this set must keep their checkpoint in step with those rows. + this.invalidatedHosts.add(serverId); this.removeStoredHost(serverId); }