mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-20 23:35:27 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
02e2c5f3ec | ||
|
|
cf0c4243a3 | ||
|
|
1f28cde1a9 | ||
|
|
a138bea3ee | ||
|
|
d328f0fbbf |
+7
-2
@@ -582,12 +582,17 @@ host, kinds, and ids. Opening a cached workspace uses exact workspace and projec
|
||||
directory scan. One invalid row is deleted and returned as a miss without affecting other rows.
|
||||
An invalid directory row and its affected checkpoint cursor are repaired in one transaction, so a
|
||||
later launch cannot accept a checkpoint for a partial baseline. Directory changes and their
|
||||
checkpoint are also applied in one transaction. Reads wait for accepted changes to reach storage,
|
||||
checkpoint are also applied in one transaction. A stored checkpoint carries a version. When a writer
|
||||
bug could have stored a checkpoint without its directory, bump the version: older checkpoints are
|
||||
rejected on read, cached rows still paint, and each host re-syncs its directory once. Reads wait for
|
||||
accepted changes to reach storage,
|
||||
then retry only when a requested row changes during the read. Unrelated agent streams do not
|
||||
invalidate the read. Directory baseline replacements and host removal or renaming still invalidate
|
||||
the affected reads.
|
||||
|
||||
The cache is capped at 32 MiB and evicts whole hosts in least-recently-written order. Budget
|
||||
The cache is capped at 32 MiB and evicts whole hosts in least-recently-written order. After an
|
||||
eviction, only a baseline replacement stores that host's directory again. Mutations queued around
|
||||
the eviction are dropped when they are written, so a checkpoint never outlives its rows. Budget
|
||||
bookkeeping may scan opaque row sizes during a deferred write, never during host registry startup or
|
||||
before a requested cache row can paint. The row store is not encrypted. A cached timeline can contain
|
||||
source code, prompts, and tool output; encrypted-at-rest storage is a separate security decision.
|
||||
|
||||
@@ -206,13 +206,14 @@ async function simulateDamagedLegacyDirectoryCache(
|
||||
return;
|
||||
}
|
||||
const stored = JSON.parse(checkpoint.payload);
|
||||
// Accept either writer so this fixture also reproduces on the unfixed client.
|
||||
const cursors = stored.version === 1 ? stored.cursors : stored;
|
||||
if (!cursors.workspaces?.generation) {
|
||||
// Read the cursors from whichever version wrote them, then store them as version 1:
|
||||
// the format whose writer could keep a checkpoint after evicting the rows it describes.
|
||||
const cursors = typeof stored.version === "number" ? stored.cursors : stored;
|
||||
if (!cursors?.workspaces?.generation) {
|
||||
transaction.abort();
|
||||
return;
|
||||
}
|
||||
store.put({ ...checkpoint, payload: JSON.stringify(cursors) });
|
||||
store.put({ ...checkpoint, payload: JSON.stringify({ version: 1, cursors }) });
|
||||
for (const row of rows) {
|
||||
if (
|
||||
row.serverId === serverId &&
|
||||
|
||||
@@ -10,7 +10,10 @@ import type {
|
||||
DaemonClient,
|
||||
WorkspaceLabelListPayload,
|
||||
} from "@getpaseo/client/internal/daemon-client";
|
||||
import type { SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import type {
|
||||
SessionOutboundMessage,
|
||||
WorkspaceDescriptorPayload,
|
||||
} from "@getpaseo/protocol/messages";
|
||||
import {
|
||||
normalizeProjectDescriptor,
|
||||
normalizeWorkspaceDescriptor,
|
||||
@@ -1667,6 +1670,187 @@ it("fills every cached workspace beneath live updates received during the SQLite
|
||||
database.close();
|
||||
});
|
||||
|
||||
/**
|
||||
* A host whose cache was damaged before the eviction fix: workspaces survived, project rows did not,
|
||||
* and a version 1 checkpoint still claims the project directory is current. The daemon holds
|
||||
* projects A and B; renaming A is its update at `renameSeq`.
|
||||
*/
|
||||
async function createDamagedLegacyProjectDirectory(serverId: string, renameSeq: number) {
|
||||
serverIds.add(serverId);
|
||||
const { cache, database, storage, reopen, holdRead } = createSqliteCache();
|
||||
cache.setHosts([serverId]);
|
||||
const workspacePayload = (id: string, projectId: string): WorkspaceDescriptorPayload => ({
|
||||
id,
|
||||
projectId,
|
||||
projectDisplayName: projectId,
|
||||
projectRootPath: `/repo/${projectId}`,
|
||||
workspaceDirectory: `/repo/${projectId}`,
|
||||
projectKind: "git",
|
||||
workspaceKind: "local_checkout",
|
||||
name: id,
|
||||
status: "done",
|
||||
statusEnteredAt: null,
|
||||
activityAt: null,
|
||||
archivingAt: null,
|
||||
diffStat: null,
|
||||
scripts: [],
|
||||
});
|
||||
const project = (projectId: string, projectDisplayName: string) => ({
|
||||
projectId,
|
||||
projectDisplayName,
|
||||
projectRootPath: `/repo/${projectId}`,
|
||||
projectKind: "git" as const,
|
||||
});
|
||||
const workspaceEntries = [workspacePayload("wa", "A"), workspacePayload("wb", "B")];
|
||||
cache.replaceDirectoryBaseline(serverId, {
|
||||
agents: new Map(),
|
||||
workspaces: new Map(
|
||||
workspaceEntries.map((entry) => [entry.id, normalizeWorkspaceDescriptor(entry)]),
|
||||
),
|
||||
projects: new Map(),
|
||||
});
|
||||
await cache.flush();
|
||||
await storage.apply({
|
||||
deletes: [],
|
||||
upserts: [
|
||||
{
|
||||
serverId,
|
||||
kind: "checkpoint",
|
||||
id: "singleton",
|
||||
payload: JSON.stringify({
|
||||
version: 1,
|
||||
cursors: { projects: { generation: "g", afterSeq: renameSeq - 1 } },
|
||||
}),
|
||||
},
|
||||
],
|
||||
});
|
||||
const reopened = reopen();
|
||||
reopened.setHosts([serverId]);
|
||||
const client = new FakeDirectoryClient();
|
||||
let failNextProjectRequest = false;
|
||||
client.listProjects = async (options?: unknown): Promise<ProjectListResult> => {
|
||||
client.lastProjectOptions = options;
|
||||
if (failNextProjectRequest) {
|
||||
failNextProjectRequest = false;
|
||||
throw new Error("Request timed out");
|
||||
}
|
||||
const resumesAfterRename =
|
||||
(options as { sync?: { afterSeq?: number } } | undefined)?.sync?.afterSeq === renameSeq;
|
||||
return resumesAfterRename
|
||||
? {
|
||||
requestId: "projects",
|
||||
projects: [],
|
||||
sync: { generation: "g", headSeq: renameSeq, mode: "changes", removals: [] },
|
||||
}
|
||||
: {
|
||||
requestId: "projects",
|
||||
projects: [project("A", "Renamed"), project("B", "B")],
|
||||
sync: { generation: "g", headSeq: renameSeq, mode: "snapshot", removals: [] },
|
||||
};
|
||||
};
|
||||
client.fetchWorkspaces = async (): Promise<WorkspaceFetchResult> => ({
|
||||
requestId: "workspaces",
|
||||
entries: workspaceEntries,
|
||||
emptyProjects: [],
|
||||
pageInfo: { hasMore: false, nextCursor: null, prevCursor: null },
|
||||
sync: { generation: "g", headSeq: 2, mode: "snapshot", removals: [] },
|
||||
});
|
||||
const directory = new DirectorySync(
|
||||
serverId,
|
||||
{
|
||||
onAgentStoppedRunning: () => {},
|
||||
markAgentLoading: () => {},
|
||||
markAgentReady: () => {},
|
||||
markAgentError: () => {},
|
||||
},
|
||||
reopened,
|
||||
);
|
||||
const store = useSessionStore.getState();
|
||||
store.initializeSession(serverId, client as unknown as DaemonClient, 1);
|
||||
store.updateSessionServerInfo(serverId, {
|
||||
serverId,
|
||||
hostname: null,
|
||||
version: "test",
|
||||
features: { workspaceMultiplicity: true, directorySync: true, projectList: true },
|
||||
});
|
||||
directory.connectionChanged({
|
||||
client: client as unknown as DaemonClient,
|
||||
status: "online",
|
||||
source: { clientGeneration: 1, connectionEpoch: 1 },
|
||||
});
|
||||
return {
|
||||
client,
|
||||
directory,
|
||||
holdRead,
|
||||
failNextProjectRequest: () => {
|
||||
failNextProjectRequest = true;
|
||||
},
|
||||
renameProjectA: () =>
|
||||
client.emit({
|
||||
type: "project.update",
|
||||
payload: {
|
||||
kind: "upsert",
|
||||
project: project("A", "Renamed"),
|
||||
generation: "g",
|
||||
seq: renameSeq,
|
||||
},
|
||||
}),
|
||||
sessionProjectIds: () =>
|
||||
[...useSessionStore.getState().sessions[serverId].projects.keys()].sort(),
|
||||
persistedProjectIds: async () => {
|
||||
await reopened.flush();
|
||||
const restarted = reopen();
|
||||
restarted.setHosts([serverId]);
|
||||
return [...(await restarted.readDirectory(serverId)).projects.keys()].sort();
|
||||
},
|
||||
close: async () => {
|
||||
directory.dispose();
|
||||
await reopened.flush();
|
||||
database.close();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("re-syncs every project when a project update lands while an untrusted cache loads", async () => {
|
||||
const damaged = await createDamagedLegacyProjectDirectory("untrusted-checkpoint-race", 6);
|
||||
try {
|
||||
let renamed = false;
|
||||
damaged.holdRead(async () => {
|
||||
if (renamed) return;
|
||||
renamed = true;
|
||||
damaged.renameProjectA();
|
||||
});
|
||||
|
||||
damaged.directory.setDemand({}, true);
|
||||
await damaged.directory.refreshDemand();
|
||||
|
||||
expect(renamed).toBe(true);
|
||||
expect(damaged.client.lastProjectOptions).toEqual({ sync: {} });
|
||||
expect(damaged.sessionProjectIds()).toEqual(["A", "B"]);
|
||||
expect(await damaged.persistedProjectIds()).toEqual(["A", "B"]);
|
||||
} finally {
|
||||
await damaged.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("re-syncs every project when a project update lands after the first sync fails", async () => {
|
||||
const damaged = await createDamagedLegacyProjectDirectory("failed-first-sync", 3);
|
||||
try {
|
||||
damaged.failNextProjectRequest();
|
||||
damaged.directory.setDemand({}, true);
|
||||
await expect(damaged.directory.refreshDemand()).rejects.toThrow("Request timed out");
|
||||
|
||||
damaged.renameProjectA();
|
||||
await damaged.directory.refreshDemand();
|
||||
|
||||
expect(damaged.client.lastProjectOptions).toEqual({ sync: {} });
|
||||
expect(damaged.sessionProjectIds()).toEqual(["A", "B"]);
|
||||
expect(await damaged.persistedProjectIds()).toEqual(["A", "B"]);
|
||||
} finally {
|
||||
await damaged.close();
|
||||
}
|
||||
});
|
||||
|
||||
function createWorkspaceLabelDirectory(serverId: string) {
|
||||
const { client, directory } = createDirectory(serverId, { workspaceLabels: true });
|
||||
const store = useSessionStore.getState();
|
||||
|
||||
@@ -288,7 +288,7 @@ export class DirectorySync {
|
||||
this.revision += 1;
|
||||
if (this.agentTransactions.record(source, delta)) return;
|
||||
this.agents.applyDelta(delta);
|
||||
this.noteLiveCursor("agents", delta);
|
||||
if (this.extendsRefreshedBaseline(source)) this.noteLiveCursor("agents", delta);
|
||||
this.persistCheckpoint();
|
||||
}
|
||||
|
||||
@@ -300,7 +300,7 @@ export class DirectorySync {
|
||||
this.workspaceRevision += 1;
|
||||
if (this.workspaceTransactions.record(source, delta)) return;
|
||||
this.applyWorkspaceDelta(delta);
|
||||
if (delta.kind !== "script_status")
|
||||
if (delta.kind !== "script_status" && this.extendsRefreshedBaseline(source))
|
||||
this.noteLiveCursor(
|
||||
"projectId" in delta || "project" in delta ? "projects" : "workspaces",
|
||||
delta,
|
||||
@@ -308,6 +308,20 @@ export class DirectorySync {
|
||||
this.persistCheckpoint();
|
||||
}
|
||||
|
||||
/**
|
||||
* A live event's sequence covers everything before it only when a refresh completed on the
|
||||
* subscriptions that delivered it. Before that (first sync still pending or failed) or after
|
||||
* they lapse, the client may have missed earlier changes, so the event updates rows but leaves
|
||||
* the cursor where the next refresh can still fetch what was missed.
|
||||
*/
|
||||
private extendsRefreshedBaseline(source: DirectorySourceToken): boolean {
|
||||
const refreshed = this.satisfiedDemandSource;
|
||||
return (
|
||||
refreshed?.clientGeneration === source.clientGeneration &&
|
||||
refreshed.connectionEpoch === source.connectionEpoch
|
||||
);
|
||||
}
|
||||
|
||||
private observeDirectoryEvents(): void {
|
||||
if (this.eventSubscription) return;
|
||||
const { client, source } = this.requireOnline();
|
||||
@@ -566,24 +580,16 @@ export class DirectorySync {
|
||||
input: { subscribe?: boolean } | undefined,
|
||||
loadDirectoryCache: boolean,
|
||||
): Promise<void> {
|
||||
if (loadDirectoryCache) await this.loadCachedDirectory();
|
||||
if (loadDirectoryCache) {
|
||||
this.primeWorkspaceTransactionForCacheLoad();
|
||||
await this.loadCachedDirectory();
|
||||
}
|
||||
const onlineConnection = this.getOnlineConnection();
|
||||
if (!onlineConnection) return;
|
||||
const { client, source } = onlineConnection;
|
||||
const transaction = this.workspaceTransactions.begin(source, () => {
|
||||
const baseline = this.workspaces.snapshot();
|
||||
return {
|
||||
workspaces: new Map(baseline.workspaces),
|
||||
projects: new Map(baseline.projects),
|
||||
// Cache hydration can advance the live replica while these requests await responses.
|
||||
// Every page must use the cursors belonging to the maps captured here.
|
||||
requestCursors: { ...this.cursors },
|
||||
syncCursors: {},
|
||||
syncModes: {},
|
||||
touchedWorkspaceIds: new Set(),
|
||||
touchedProjectIds: new Set(),
|
||||
};
|
||||
});
|
||||
const transaction = this.workspaceTransactions.begin(source, () =>
|
||||
this.createWorkspaceRefreshSnapshot(),
|
||||
);
|
||||
try {
|
||||
await this.waitForSessionMetadata(client, source);
|
||||
const serverInfo = useSessionStore.getState().sessions[this.serverId]?.serverInfo;
|
||||
@@ -807,6 +813,31 @@ export class DirectorySync {
|
||||
}));
|
||||
}
|
||||
|
||||
// Updates that arrive before a refresh completes have no trusted baseline to advance from.
|
||||
// Holding them keeps their cursor out of the cache until the refresh they are applied over.
|
||||
private primeWorkspaceTransactionForCacheLoad(): void {
|
||||
const connection = this.getOnlineConnection();
|
||||
if (!connection) return;
|
||||
this.workspaceTransactions.begin(connection.source, () =>
|
||||
this.createWorkspaceRefreshSnapshot(),
|
||||
);
|
||||
}
|
||||
|
||||
private createWorkspaceRefreshSnapshot(): WorkspaceRefreshSnapshot {
|
||||
const baseline = this.workspaces.snapshot();
|
||||
return {
|
||||
workspaces: new Map(baseline.workspaces),
|
||||
projects: new Map(baseline.projects),
|
||||
// Cache hydration can advance the live replica while these requests await responses.
|
||||
// Every page must use the cursors belonging to the maps captured here.
|
||||
requestCursors: { ...this.cursors },
|
||||
syncCursors: {},
|
||||
syncModes: {},
|
||||
touchedWorkspaceIds: new Set(),
|
||||
touchedProjectIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
private supportsDirectorySync(): boolean {
|
||||
return (
|
||||
useSessionStore.getState().sessions[this.serverId]?.serverInfo?.features?.directorySync ===
|
||||
|
||||
@@ -37,6 +37,9 @@ class MemoryStorage implements ReplicaRowStore {
|
||||
onRead: (() => void) | null = null;
|
||||
/** Throws once a read goes past this count, so a non-terminating read loop fails the test. */
|
||||
readLimit: number | null = null;
|
||||
/** Awaited before the store lists every row or deletes a host, so a test can interleave there. */
|
||||
beforeReadAll: (() => Promise<void>) | null = null;
|
||||
beforeDeleteHost: (() => Promise<void>) | null = null;
|
||||
|
||||
private key(row: Pick<ReplicaRow, "serverId" | "kind" | "id">): string {
|
||||
return `${row.serverId}:${row.kind}:${row.id}`;
|
||||
@@ -68,6 +71,7 @@ class MemoryStorage implements ReplicaRowStore {
|
||||
}
|
||||
|
||||
async readAll(): Promise<ReplicaHostRows[]> {
|
||||
await this.beforeReadAll?.();
|
||||
const hosts = new Map<string, ReplicaRow[]>();
|
||||
for (const row of this.rows.values()) {
|
||||
const rows = hosts.get(row.serverId) ?? [];
|
||||
@@ -91,6 +95,7 @@ class MemoryStorage implements ReplicaRowStore {
|
||||
}
|
||||
|
||||
async deleteHost(serverId: string): Promise<void> {
|
||||
await this.beforeDeleteHost?.();
|
||||
for (const [key, row] of this.rows) if (row.serverId === serverId) this.rows.delete(key);
|
||||
}
|
||||
|
||||
@@ -436,40 +441,49 @@ describe("ReplicaCache", () => {
|
||||
await expectLegacyWorkspaceRefresh(cached);
|
||||
});
|
||||
|
||||
it("rejects pre-baseline-fix checkpoints without discarding cached rows or timelines", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const writer = createCache(storage);
|
||||
const baseline = directory();
|
||||
writer.replaceDirectoryBaseline(SERVER_ID, baseline);
|
||||
writer.commitTimeline(SERVER_ID, "agent-1", timeline());
|
||||
await writer.flush();
|
||||
await storage.apply({
|
||||
deletes: [],
|
||||
upserts: [
|
||||
{
|
||||
serverId: SERVER_ID,
|
||||
kind: "checkpoint",
|
||||
id: "singleton",
|
||||
payload: JSON.stringify(baseline.checkpoint),
|
||||
},
|
||||
],
|
||||
});
|
||||
it.each([
|
||||
["before the baseline fix", (checkpoint: DirectoryCheckpoint): unknown => checkpoint],
|
||||
[
|
||||
"before the eviction fix",
|
||||
(checkpoint: DirectoryCheckpoint): unknown => ({ version: 1, cursors: checkpoint }),
|
||||
],
|
||||
])(
|
||||
"rejects checkpoints written %s without discarding cached rows or timelines",
|
||||
async (_era, legacyPayload) => {
|
||||
const storage = new MemoryStorage();
|
||||
const writer = createCache(storage);
|
||||
const baseline = directory();
|
||||
writer.replaceDirectoryBaseline(SERVER_ID, baseline);
|
||||
writer.commitTimeline(SERVER_ID, "agent-1", timeline());
|
||||
await writer.flush();
|
||||
await storage.apply({
|
||||
deletes: [],
|
||||
upserts: [
|
||||
{
|
||||
serverId: SERVER_ID,
|
||||
kind: "checkpoint",
|
||||
id: "singleton",
|
||||
payload: JSON.stringify(legacyPayload(baseline.checkpoint)),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const reader = createCache(storage);
|
||||
const restored = await reader.readDirectory(SERVER_ID);
|
||||
expect(restored.checkpoint).toBeUndefined();
|
||||
expect([...restored.workspaces.keys()]).toEqual([...baseline.workspaces.keys()]);
|
||||
expect([...restored.projects.keys()]).toEqual([...baseline.projects.keys()]);
|
||||
expect([...restored.agents.keys()]).toEqual([...baseline.agents.keys()]);
|
||||
expect(await reader.readTimeline(SERVER_ID, "agent-1")).toEqual(timeline());
|
||||
expect(await storage.read(SERVER_ID, ["checkpoint"])).toEqual([]);
|
||||
const reader = createCache(storage);
|
||||
const restored = await reader.readDirectory(SERVER_ID);
|
||||
expect(restored.checkpoint).toBeUndefined();
|
||||
expect([...restored.workspaces.keys()]).toEqual([...baseline.workspaces.keys()]);
|
||||
expect([...restored.projects.keys()]).toEqual([...baseline.projects.keys()]);
|
||||
expect([...restored.agents.keys()]).toEqual([...baseline.agents.keys()]);
|
||||
expect(await reader.readTimeline(SERVER_ID, "agent-1")).toEqual(timeline());
|
||||
expect(await storage.read(SERVER_ID, ["checkpoint"])).toEqual([]);
|
||||
|
||||
reader.replaceDirectoryBaseline(SERVER_ID, baseline);
|
||||
await reader.flush();
|
||||
expect((await createCache(storage).readDirectory(SERVER_ID)).checkpoint).toEqual(
|
||||
baseline.checkpoint,
|
||||
);
|
||||
});
|
||||
reader.replaceDirectoryBaseline(SERVER_ID, baseline);
|
||||
await reader.flush();
|
||||
expect((await createCache(storage).readDirectory(SERVER_ID)).checkpoint).toEqual(
|
||||
baseline.checkpoint,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("does nothing until an owner explicitly commits data", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
@@ -1102,6 +1116,113 @@ describe("ReplicaCache", () => {
|
||||
).toEqual(["agent", "checkpoint", "project", "workspace"]);
|
||||
});
|
||||
|
||||
it.each(["beforeReadAll", "beforeDeleteHost"] as const)(
|
||||
"stores no checkpoint for a host evicted after its update was queued (%s)",
|
||||
async (pausePoint) => {
|
||||
const storage = new MemoryStorage();
|
||||
const previousSession = new ReplicaCache(storage, noLegacyCleanup);
|
||||
previousSession.setHosts([SERVER_ID, "other-host"]);
|
||||
previousSession.replaceDirectoryBaseline(SERVER_ID, directory());
|
||||
previousSession.replaceDirectoryBaseline("other-host", directory());
|
||||
await previousSession.flush();
|
||||
|
||||
const cache = new ReplicaCache(storage, { ...noLegacyCleanup, maxBytes: 8_192 });
|
||||
cache.setHosts([SERVER_ID, "other-host"]);
|
||||
const paused = deferred();
|
||||
const resume = deferred();
|
||||
storage[pausePoint] = async () => {
|
||||
storage[pausePoint] = null;
|
||||
paused.resolve();
|
||||
await resume.promise;
|
||||
};
|
||||
// Larger than the whole budget, so this persist evicts both hosts.
|
||||
cache.commitTimeline(SERVER_ID, "agent-1", timeline("x".repeat(16_384)));
|
||||
const eviction = cache.flush();
|
||||
await paused.promise;
|
||||
cache.commitDirectoryMutations(SERVER_ID, [], { agents: { generation: "g", afterSeq: 13 } });
|
||||
resume.resolve();
|
||||
await eviction;
|
||||
await cache.flush();
|
||||
|
||||
const restored = await createCache(storage).readDirectory(SERVER_ID);
|
||||
expect(restored.agents.size).toBe(0);
|
||||
expect(restored.checkpoint).toBeUndefined();
|
||||
},
|
||||
);
|
||||
|
||||
it("stores no checkpoint for a host evicted while a malformed row is repaired", async () => {
|
||||
const { storage } = await cacheWithoutBackgroundVisibility();
|
||||
const reader = createCache(storage, 8_192);
|
||||
storage.onRead = () => {
|
||||
if (!storage.reads.at(-1)?.kinds.includes("checkpoint")) return;
|
||||
storage.onRead = null;
|
||||
// Larger than the whole budget, so this persist evicts the host before the repair writes.
|
||||
reader.commitTimeline(SERVER_ID, "agent-1", timeline("x".repeat(16_384)));
|
||||
void reader.flush();
|
||||
};
|
||||
|
||||
expect(await reader.readWorkspace(SERVER_ID, "workspace-1")).toBeUndefined();
|
||||
await reader.flush();
|
||||
|
||||
const restored = await createCache(storage).readDirectory(SERVER_ID);
|
||||
expect(restored.projects.size).toBe(0);
|
||||
expect(restored.checkpoint).toBeUndefined();
|
||||
});
|
||||
|
||||
it("repairs a malformed row's cursor when the overlapping eviction fails", async () => {
|
||||
const { storage } = await cacheWithoutBackgroundVisibility();
|
||||
const reader = createCache(storage, 8_192);
|
||||
storage.onRead = () => {
|
||||
if (!storage.reads.at(-1)?.kinds.includes("checkpoint")) return;
|
||||
storage.onRead = null;
|
||||
storage.beforeDeleteHost = async () => {
|
||||
storage.beforeDeleteHost = null;
|
||||
throw new Error("Eviction aborted");
|
||||
};
|
||||
reader.commitTimeline(SERVER_ID, "agent-1", timeline("x".repeat(16_384)));
|
||||
void reader.flush();
|
||||
};
|
||||
|
||||
expect(await reader.readWorkspace(SERVER_ID, "workspace-1")).toBeUndefined();
|
||||
// A newer timeline fits the budget, so no later eviction clears the checkpoint.
|
||||
reader.commitTimeline(SERVER_ID, "agent-1", timeline());
|
||||
await reader.flush();
|
||||
|
||||
const restored = await createCache(storage).readDirectory(SERVER_ID);
|
||||
expect(restored.workspaces.size).toBe(0);
|
||||
expect(restored.checkpoint).toEqual({
|
||||
agents: { generation: "g", afterSeq: 12 },
|
||||
projects: { generation: "g", afterSeq: 4 },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps an update queued behind the baseline that rebuilds an evicted host", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const cache = new ReplicaCache(storage, { ...noLegacyCleanup, maxBytes: 2_500 });
|
||||
cache.setHosts([SERVER_ID, "other-host"]);
|
||||
const baseline = directory();
|
||||
cache.replaceDirectoryBaseline(SERVER_ID, baseline);
|
||||
await cache.flush();
|
||||
cache.replaceDirectoryBaseline("other-host", baseline);
|
||||
await cache.flush();
|
||||
expect((await cache.readDirectory(SERVER_ID)).agents.size).toBe(0);
|
||||
|
||||
const renamed = { ...agent(), title: "Renamed agent" };
|
||||
cache.replaceDirectoryBaseline(SERVER_ID, baseline);
|
||||
cache.commitDirectoryMutations(
|
||||
SERVER_ID,
|
||||
[{ kind: "agent", type: "upsert", id: renamed.id, value: renamed }],
|
||||
{ agents: { generation: "g", afterSeq: 13 } },
|
||||
);
|
||||
await cache.flush();
|
||||
cache.commitDirectoryMutations(SERVER_ID, [], { agents: { generation: "g", afterSeq: 14 } });
|
||||
await cache.flush();
|
||||
|
||||
const restored = await createCache(storage).readDirectory(SERVER_ID);
|
||||
expect(restored.checkpoint).toEqual({ agents: { generation: "g", afterSeq: 14 } });
|
||||
expect(restored.agents.get(renamed.id)?.title).toBe("Renamed agent");
|
||||
});
|
||||
|
||||
it("runs legacy cleanup once when storage is first used", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const cache = new ReplicaCache(storage, {
|
||||
|
||||
@@ -397,15 +397,19 @@ const DirectoryCheckpointSchema = z.strictObject({
|
||||
agents: DirectoryCursorSchema.optional(),
|
||||
});
|
||||
|
||||
// Old checkpoints could describe a baseline that an overlapping refresh had discarded.
|
||||
// Revalidate directory contents once without evicting cached rows or timelines.
|
||||
// A checkpoint is trusted only when this version wrote it. Earlier writers could store one for a
|
||||
// directory the cache no longer held: unversioned ones after an overlapping refresh, version 1
|
||||
// after an eviction. Rejecting them revalidates directory contents once without evicting cached
|
||||
// rows or timelines.
|
||||
const DIRECTORY_CHECKPOINT_VERSION = 2;
|
||||
|
||||
const StoredDirectoryCheckpointSchema = z.strictObject({
|
||||
version: z.literal(1),
|
||||
version: z.literal(DIRECTORY_CHECKPOINT_VERSION),
|
||||
cursors: DirectoryCheckpointSchema,
|
||||
});
|
||||
|
||||
function serializeDirectoryCheckpoint(cursors: DirectoryCheckpoint) {
|
||||
return { version: 1 as const, cursors };
|
||||
return { version: DIRECTORY_CHECKPOINT_VERSION, cursors };
|
||||
}
|
||||
|
||||
function deserializeDirectoryCheckpoint(payload: string): DirectoryCheckpoint {
|
||||
@@ -968,6 +972,7 @@ export class ReplicaCache {
|
||||
private pendingUpserts = new Map<string, StructuredReplicaUpsert>();
|
||||
private pendingDeletes = new Map<string, ReplicaRowKey>();
|
||||
private pendingBaselines = new Set<string>();
|
||||
// Hosts whose stored directory was evicted. Only a baseline stores their directory again.
|
||||
private readonly invalidatedHosts = new Set<string>();
|
||||
private readonly maxBytes: number;
|
||||
private totalBytes = 0;
|
||||
@@ -1109,11 +1114,7 @@ export class ReplicaCache {
|
||||
private async deleteInvalidRow(row: ReplicaRow): Promise<void> {
|
||||
const invalidEntity = directoryEntityForRow(row);
|
||||
if (!invalidEntity) {
|
||||
const changes = { upserts: [], deletes: [row] } satisfies ReplicaRowChanges;
|
||||
await this.queueOperation(async () => {
|
||||
await this.rowStore.apply(changes);
|
||||
this.applyStoredChanges(changes);
|
||||
});
|
||||
await this.applyRepair(row.serverId, { upserts: [], deletes: [row] });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1144,10 +1145,7 @@ export class ReplicaCache {
|
||||
]
|
||||
: [],
|
||||
};
|
||||
await this.queueOperation(async () => {
|
||||
await this.rowStore.apply(changes);
|
||||
this.applyStoredChanges(changes);
|
||||
});
|
||||
await this.applyRepair(row.serverId, changes);
|
||||
}
|
||||
|
||||
private async repairInvalidDirectoryRows(
|
||||
@@ -1170,9 +1168,16 @@ export class ReplicaCache {
|
||||
]
|
||||
: [],
|
||||
};
|
||||
await this.queueOperation(async () => {
|
||||
await this.rowStore.apply(changes);
|
||||
this.applyStoredChanges(changes);
|
||||
await this.applyRepair(serverId, changes);
|
||||
}
|
||||
|
||||
// A repair is queued behind writes that can evict its host. If one did, the host has no
|
||||
// directory left for the repaired checkpoint to describe, so only the deletes still apply.
|
||||
private applyRepair(serverId: string, changes: ReplicaRowChanges): Promise<void> {
|
||||
return this.queueOperation(async () => {
|
||||
const applied = this.invalidatedHosts.has(serverId) ? { ...changes, upserts: [] } : changes;
|
||||
await this.rowStore.apply(applied);
|
||||
this.applyStoredChanges(applied);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1182,7 +1187,6 @@ export class ReplicaCache {
|
||||
checkpoint?: DirectoryCheckpoint,
|
||||
): void {
|
||||
if (!this.activeServerIds.has(serverId)) return;
|
||||
if (this.invalidatedHosts.has(serverId)) return;
|
||||
for (const mutation of mutations) {
|
||||
if (mutation.type === "delete") {
|
||||
this.queueEntityDelete(serverId, mutation.kind, mutation.id);
|
||||
@@ -1412,9 +1416,17 @@ export class ReplicaCache {
|
||||
}
|
||||
|
||||
private materializePendingChanges(pending: PendingReplicaChanges): ReplicaRowChanges {
|
||||
const deletes = new Map(pending.deletes.map((row) => [pendingRowKey(row), row]));
|
||||
// Decided here, inside the serialized write, because eviction happens there too. A mutation
|
||||
// queued before its host was evicted would otherwise store a checkpoint without its directory.
|
||||
const isStorable = (row: ReplicaRowKey): boolean =>
|
||||
row.kind === "timeline" ||
|
||||
!this.invalidatedHosts.has(row.serverId) ||
|
||||
pending.baselines.has(row.serverId);
|
||||
const deletes = new Map(
|
||||
pending.deletes.filter(isStorable).map((row) => [pendingRowKey(row), row]),
|
||||
);
|
||||
const upserts: ReplicaRow[] = [];
|
||||
for (const upsert of pending.upserts) {
|
||||
for (const upsert of pending.upserts.filter(isStorable)) {
|
||||
const row = this.materializeUpsert(upsert);
|
||||
if (row) upserts.push(row);
|
||||
else deletes.set(pendingRowKey(upsert), upsert);
|
||||
@@ -1563,8 +1575,10 @@ export class ReplicaCache {
|
||||
projectedTotal -= bytes;
|
||||
projectedBytes.delete(serverId);
|
||||
evicted.add(serverId);
|
||||
this.invalidatedHosts.add(serverId);
|
||||
if (this.storedRows.has(serverId)) await this.rowStore.deleteHost(serverId);
|
||||
// Only after the delete commits: a failed eviction leaves the rows stored, and writers that
|
||||
// check this set must keep their checkpoint in step with those rows.
|
||||
this.invalidatedHosts.add(serverId);
|
||||
this.removeStoredHost(serverId);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user