From bb18cf618a3468c49a53ab28584bfb8be4e76b8f Mon Sep 17 00:00:00 2001 From: Mohamed Boudra Date: Wed, 3 Jun 2026 17:25:07 +0700 Subject: [PATCH] Write persisted files atomically --- docs/data-model.md | 4 +-- .../server/src/server/agent/agent-storage.ts | 12 ++------- packages/server/src/server/atomic-file.ts | 25 +++++++++++++++++++ .../server/src/server/chat/chat-service.ts | 6 ++--- packages/server/src/server/daemon-keypair.ts | 4 +-- packages/server/src/server/loop-service.ts | 4 +-- .../server/src/server/persisted-config.ts | 9 ++++--- packages/server/src/server/private-files.ts | 9 ------- packages/server/src/server/schedule/store.ts | 5 ++-- packages/server/src/server/server-id.ts | 6 ++--- .../server/src/server/workspace-registry.ts | 8 ++---- packages/server/src/tasks/task-document.ts | 5 ++-- 12 files changed, 52 insertions(+), 45 deletions(-) create mode 100644 packages/server/src/server/atomic-file.ts diff --git a/docs/data-model.md b/docs/data-model.md index 324502625..306e9552b 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -30,7 +30,7 @@ $PASEO_HOME/ └── push-tokens.json # Expo push notification tokens ``` -The `agents/{sanitized-cwd}/` directory name is derived from the agent's `cwd` by stripping the filesystem root and replacing path separators with `-` (Windows drive letters become a `C-` style prefix). Atomic writes (temp file + rename): agent records, chat, project/workspace registries, push tokens. Non-atomic (plain `writeFile`): `config.json`, `schedules/*.json`, `loops/loops.json`, `server-id`, `daemon-keypair.json`. +The `agents/{sanitized-cwd}/` directory name is derived from the agent's `cwd` by stripping the filesystem root and replacing path separators with `-` (Windows drive letters become a `C-` style prefix). Persistent server stores write atomically by writing a temp file in the target directory and then renaming it into place. --- @@ -187,7 +187,7 @@ Local speech model ids are intentionally narrow: STT uses `parakeet-tdt-0.6b-v2- **Path:** `$PASEO_HOME/schedules/{id}.json` -One file per schedule. ID is 8 hex characters. Writes are direct (not atomic). +One file per schedule. ID is 8 hex characters. | Field | Type | Description | | ----------- | ------------------------------------- | -------------------------------- | diff --git a/packages/server/src/server/agent/agent-storage.ts b/packages/server/src/server/agent/agent-storage.ts index 8bb673890..03501fbec 100644 --- a/packages/server/src/server/agent/agent-storage.ts +++ b/packages/server/src/server/agent/agent-storage.ts @@ -1,9 +1,9 @@ -import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; import path from "node:path"; import { z } from "zod"; import type { Logger } from "pino"; +import { writeJsonFileAtomic } from "../atomic-file.js"; import { AgentFeatureSchema, AgentStatusSchema } from "../messages.js"; import { toStoredAgentRecord } from "./agent-projections.js"; import type { ManagedAgent } from "./agent-manager.js"; @@ -143,8 +143,7 @@ export class AgentStorage { const nextPath = this.buildRecordPath(record); const previousPath = this.pathById.get(agentId); - await fs.mkdir(path.dirname(nextPath), { recursive: true }); - await writeFileAtomically(nextPath, JSON.stringify(record, null, 2)); + await writeJsonFileAtomic(nextPath, record); this.addIndexedPath(agentId, nextPath); if (previousPath && previousPath !== nextPath) { @@ -387,10 +386,3 @@ function projectDirNameFromCwd(cwd: string): string { } return prefix + withoutRoot.replace(/[\\/]+/g, "-"); } - -async function writeFileAtomically(targetPath: string, payload: string) { - const directory = path.dirname(targetPath); - const tempPath = path.join(directory, `.agent.tmp-${process.pid}-${Date.now()}-${randomUUID()}`); - await fs.writeFile(tempPath, payload, "utf8"); - await fs.rename(tempPath, targetPath); -} diff --git a/packages/server/src/server/atomic-file.ts b/packages/server/src/server/atomic-file.ts new file mode 100644 index 000000000..cedc8d51e --- /dev/null +++ b/packages/server/src/server/atomic-file.ts @@ -0,0 +1,25 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import path from "node:path"; + +export async function writeFileAtomic( + filePath: string, + data: string | NodeJS.ArrayBufferView, +): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + const tempPath = path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`, + ); + try { + await fs.writeFile(tempPath, data, "utf8"); + await fs.rename(tempPath, filePath); + } catch (error) { + await fs.rm(tempPath, { force: true }); + throw error; + } +} + +export async function writeJsonFileAtomic(filePath: string, value: unknown): Promise { + await writeFileAtomic(filePath, JSON.stringify(value, null, 2)); +} diff --git a/packages/server/src/server/chat/chat-service.ts b/packages/server/src/server/chat/chat-service.ts index 2eb5eb0f0..08706d5ee 100644 --- a/packages/server/src/server/chat/chat-service.ts +++ b/packages/server/src/server/chat/chat-service.ts @@ -3,6 +3,7 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import type pino from "pino"; import { z } from "zod"; +import { writeJsonFileAtomic } from "../atomic-file.js"; import { ChatMessageSchema, ChatRoomDetailSchema, @@ -364,10 +365,7 @@ export class FileBackedChatService { .flat() .sort((left, right) => left.createdAt.localeCompare(right.createdAt)), }; - await fs.mkdir(path.dirname(this.filePath), { recursive: true }); - const tempPath = `${this.filePath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`; - await fs.writeFile(tempPath, JSON.stringify(payload, null, 2), "utf8"); - await fs.rename(tempPath, this.filePath); + await writeJsonFileAtomic(this.filePath, payload); } private findRoomByName(name: string): ChatRoom | null { diff --git a/packages/server/src/server/daemon-keypair.ts b/packages/server/src/server/daemon-keypair.ts index 6916996ac..27ae0bbf6 100644 --- a/packages/server/src/server/daemon-keypair.ts +++ b/packages/server/src/server/daemon-keypair.ts @@ -11,7 +11,7 @@ import { importSecretKey, type KeyPair, } from "@getpaseo/relay/e2ee"; -import { ensurePrivateFile, writePrivateFileSync } from "./private-files.js"; +import { ensurePrivateFile, writePrivateFileAtomicSync } from "./private-files.js"; const KeyPairSchema = z.object({ v: z.literal(2), @@ -62,7 +62,7 @@ export async function loadOrCreateDaemonKeyPair( secretKeyB64, }; - writePrivateFileSync(filePath, JSON.stringify(payload, null, 2) + "\n"); + writePrivateFileAtomicSync(filePath, JSON.stringify(payload, null, 2) + "\n"); log?.info({ filePath }, "Saved daemon keypair"); return { keyPair, publicKeyB64 }; diff --git a/packages/server/src/server/loop-service.ts b/packages/server/src/server/loop-service.ts index f53140fcb..784f197f7 100644 --- a/packages/server/src/server/loop-service.ts +++ b/packages/server/src/server/loop-service.ts @@ -3,6 +3,7 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import { z } from "zod"; import type { Logger } from "pino"; +import { writeJsonFileAtomic } from "./atomic-file.js"; import { curateAgentActivity } from "./agent/activity-curator.js"; import type { AgentManager } from "./agent/agent-manager.js"; import { getStructuredAgentResponse } from "./agent/agent-response-loop.js"; @@ -891,11 +892,10 @@ export class LoopService { private async persist(): Promise { const nextPersist = this.persistQueue.then(async () => { - await fs.mkdir(path.dirname(this.storePath), { recursive: true }); const records = Array.from(this.loops.values()).sort((left, right) => left.createdAt.localeCompare(right.createdAt), ); - await fs.writeFile(this.storePath, JSON.stringify(records, null, 2), "utf8"); + await writeJsonFileAtomic(this.storePath, records); return; }); this.persistQueue = nextPersist.catch(() => {}); diff --git a/packages/server/src/server/persisted-config.ts b/packages/server/src/server/persisted-config.ts index de6d27be9..a5ba18970 100644 --- a/packages/server/src/server/persisted-config.ts +++ b/packages/server/src/server/persisted-config.ts @@ -8,7 +8,7 @@ import { ProviderOverridesSchema, } from "./agent/provider-launch-config.js"; import type { AgentProviderRuntimeSettingsMap } from "./agent/provider-launch-config.js"; -import { ensurePrivateFile, writePrivateFileSync } from "./private-files.js"; +import { ensurePrivateFile, writePrivateFileAtomicSync } from "./private-files.js"; export const LogLevelSchema = z.enum(["trace", "debug", "info", "warn", "error", "fatal"]); export const LogFormatSchema = z.enum(["pretty", "json"]); @@ -357,7 +357,10 @@ export function loadPersistedConfig(paseoHome: string, logger?: LoggerLike): Per if (!existsSync(configPath)) { try { - writePrivateFileSync(configPath, JSON.stringify(DEFAULT_PERSISTED_CONFIG, null, 2) + "\n"); + writePrivateFileAtomicSync( + configPath, + JSON.stringify(DEFAULT_PERSISTED_CONFIG, null, 2) + "\n", + ); log?.info(`Initialized config file at ${configPath}`); } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -416,7 +419,7 @@ export function savePersistedConfig( } try { - writePrivateFileSync(configPath, JSON.stringify(result.data, null, 2) + "\n"); + writePrivateFileAtomicSync(configPath, JSON.stringify(result.data, null, 2) + "\n"); log?.info(`Saved to ${configPath}`); } catch (err) { const message = err instanceof Error ? err.message : String(err); diff --git a/packages/server/src/server/private-files.ts b/packages/server/src/server/private-files.ts index 049e92d29..5ac035a21 100644 --- a/packages/server/src/server/private-files.ts +++ b/packages/server/src/server/private-files.ts @@ -26,15 +26,6 @@ export function ensurePrivateFile(filePath: string): void { chmodBestEffort(filePath, PRIVATE_FILE_MODE); } -export function writePrivateFileSync( - filePath: string, - data: string | NodeJS.ArrayBufferView, -): void { - ensurePrivateDirectory(path.dirname(filePath)); - writeFileSync(filePath, data, { mode: PRIVATE_FILE_MODE }); - ensurePrivateFile(filePath); -} - export function writePrivateFileAtomicSync( filePath: string, data: string | NodeJS.ArrayBufferView, diff --git a/packages/server/src/server/schedule/store.ts b/packages/server/src/server/schedule/store.ts index 7038da897..37e9a402e 100644 --- a/packages/server/src/server/schedule/store.ts +++ b/packages/server/src/server/schedule/store.ts @@ -1,7 +1,8 @@ import { randomBytes } from "node:crypto"; -import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, rm } from "node:fs/promises"; import { join } from "node:path"; import { StoredScheduleSchema, type StoredSchedule } from "@getpaseo/protocol/schedule/types"; +import { writeJsonFileAtomic } from "../atomic-file.js"; function generateScheduleId(): string { return randomBytes(4).toString("hex"); @@ -53,7 +54,7 @@ export class ScheduleStore { async put(schedule: StoredSchedule): Promise { await this.ensureDir(); - await writeFile(this.filePath(schedule.id), JSON.stringify(schedule, null, 2), "utf-8"); + await writeJsonFileAtomic(this.filePath(schedule.id), schedule); } async delete(id: string): Promise { diff --git a/packages/server/src/server/server-id.ts b/packages/server/src/server/server-id.ts index 98153f53b..59dd52a9e 100644 --- a/packages/server/src/server/server-id.ts +++ b/packages/server/src/server/server-id.ts @@ -2,7 +2,7 @@ import path from "node:path"; import { existsSync, readFileSync } from "node:fs"; import { randomBytes } from "node:crypto"; -import { ensurePrivateFile, writePrivateFileSync } from "./private-files.js"; +import { ensurePrivateFile, writePrivateFileAtomicSync } from "./private-files.js"; interface LoggerLike { child(bindings: Record): LoggerLike; @@ -49,7 +49,7 @@ export function getOrCreateServerId( // Persist the override for consistent identity across restarts. if (!existsSync(serverIdPath)) { try { - writePrivateFileSync(serverIdPath, `${envOverride}\n`); + writePrivateFileAtomicSync(serverIdPath, `${envOverride}\n`); log?.info({ serverId: envOverride }, "Persisted PASEO_SERVER_ID override"); } catch (error) { log?.warn({ error }, "Failed to persist PASEO_SERVER_ID override"); @@ -75,7 +75,7 @@ export function getOrCreateServerId( const created = generateServerId(); try { - writePrivateFileSync(serverIdPath, `${created}\n`); + writePrivateFileAtomicSync(serverIdPath, `${created}\n`); } catch (error) { log?.warn({ error }, "Failed to persist serverId (continuing with in-memory id)"); } diff --git a/packages/server/src/server/workspace-registry.ts b/packages/server/src/server/workspace-registry.ts index 401afa610..0b2139061 100644 --- a/packages/server/src/server/workspace-registry.ts +++ b/packages/server/src/server/workspace-registry.ts @@ -1,10 +1,9 @@ -import { randomUUID } from "node:crypto"; import { promises as fs } from "node:fs"; -import path from "node:path"; import type { Logger } from "pino"; import { z } from "zod"; +import { writeJsonFileAtomic } from "./atomic-file.js"; import type { PersistedProjectKind, PersistedWorkspaceKind } from "./workspace-registry-model.js"; const PersistedProjectRecordSchema = z.object({ @@ -161,10 +160,7 @@ class FileBackedRegistry { private async persist(): Promise { const records = Array.from(this.cache.values()); - await fs.mkdir(path.dirname(this.filePath), { recursive: true }); - const tempPath = `${this.filePath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`; - await fs.writeFile(tempPath, JSON.stringify(records, null, 2), "utf8"); - await fs.rename(tempPath, this.filePath); + await writeJsonFileAtomic(this.filePath, records); } private async enqueuePersist(): Promise { diff --git a/packages/server/src/tasks/task-document.ts b/packages/server/src/tasks/task-document.ts index 50e324e3c..9d75a85e4 100644 --- a/packages/server/src/tasks/task-document.ts +++ b/packages/server/src/tasks/task-document.ts @@ -1,4 +1,5 @@ -import { readFile, writeFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; +import { writeFileAtomic } from "../server/atomic-file.js"; import type { Task, TaskStatus } from "./types.js"; function serializeTask(task: Task): string { @@ -130,5 +131,5 @@ export async function readTaskDocument(filePath: string): Promise { } export async function writeTaskDocument(filePath: string, task: Task): Promise { - await writeFile(filePath, serializeTask(task), "utf-8"); + await writeFileAtomic(filePath, serializeTask(task)); }