From a3438f96f893d6a8d61c9e4bf7f585b7e38eba9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BB=B4=E5=A5=B9=E5=91=BD=40?= <908280968@qq.com> Date: Fri, 24 Jul 2026 01:43:09 +0800 Subject: [PATCH] fix(app): preserve file line endings and UTF-8 BOM (#2277) * fix(app): preserve file line endings and UTF-8 BOM * refactor(app): simplify line ending preservation Let the editor parse newline variants and serialize with the file's first separator. Mixed-ending files become uniform on edit without a separate normalization layer. --------- Co-authored-by: Mohamed Boudra --- packages/app/e2e/file-editing.spec.ts | 23 ++++ .../app/src/file-explorer/read-result.test.ts | 34 +++++ packages/app/src/file-explorer/read-result.ts | 5 + .../app/src/file-pane/editor/model.test.ts | 119 +++++++++++++++++- packages/app/src/file-pane/editor/model.ts | 20 ++- .../app/src/file-pane/editor/view.web.tsx | 11 +- packages/app/src/file-pane/pane.tsx | 9 +- packages/app/src/stores/session-store.ts | 2 + 8 files changed, 211 insertions(+), 12 deletions(-) create mode 100644 packages/app/src/file-explorer/read-result.test.ts diff --git a/packages/app/e2e/file-editing.spec.ts b/packages/app/e2e/file-editing.spec.ts index 5dd0cd602..fe8a3d0f2 100644 --- a/packages/app/e2e/file-editing.spec.ts +++ b/packages/app/e2e/file-editing.spec.ts @@ -239,6 +239,29 @@ test.describe("CodeMirror workspace file editing", () => { await expect(editor(page)).toContainText("const afterReconnect = 9;"); }); + test("preserves a UTF-8 BOM and uses the first line separator after saving", async ({ + page, + withWorkspace, + }) => { + const workspace = await withWorkspace({ prefix: "file-editing-encoding-" }); + const sourcePath = path.join(workspace.repoPath, "windows.ts"); + await writeFile( + sourcePath, + Buffer.from("\uFEFFconst initial = true;\r\nconst mixed = true;\n", "utf8"), + ); + await workspace.navigateTo(); + await openWorkspaceFile(page, "windows.ts"); + + await replaceEditorText(page, "const saved = true;\nconst normalized = true;\n"); + await editor(page).press("Control+s"); + + const expected = Buffer.from( + "\uFEFFconst saved = true;\r\nconst normalized = true;\r\n", + "utf8", + ).toString("hex"); + await expect.poll(async () => (await readFile(sourcePath)).toString("hex")).toBe(expected); + }); + test("warns before closing a panel with an unsaved draft", async ({ page, withWorkspace }) => { const workspace = await withWorkspace({ prefix: "file-editing-draft-" }); const sourcePath = path.join(workspace.repoPath, "draft.ts"); diff --git a/packages/app/src/file-explorer/read-result.test.ts b/packages/app/src/file-explorer/read-result.test.ts new file mode 100644 index 000000000..fa5d3bb88 --- /dev/null +++ b/packages/app/src/file-explorer/read-result.test.ts @@ -0,0 +1,34 @@ +import type { FileReadResult } from "@getpaseo/client/internal/daemon-client"; +import { describe, expect, it } from "vitest"; +import { explorerFileFromReadResult } from "./read-result"; + +function textRead(bytes: Uint8Array): FileReadResult { + return { + bytes, + mime: "text/plain", + size: bytes.byteLength, + path: "notes.txt", + kind: "text", + modifiedAt: "2026-07-21T00:00:00.000Z", + }; +} + +describe("explorerFileFromReadResult", () => { + it("records and hides a leading UTF-8 BOM", () => { + const file = explorerFileFromReadResult( + textRead(new Uint8Array([0xef, 0xbb, 0xbf, 0x68, 0x69])), + ); + + expect(file).toMatchObject({ content: "hi", hasBom: true }); + }); + + it("does not mark BOM-free text or non-leading U+FEFF as BOM files", () => { + const plain = explorerFileFromReadResult(textRead(new TextEncoder().encode("hi"))); + const embedded = explorerFileFromReadResult( + textRead(new Uint8Array([0x68, 0x69, 0xef, 0xbb, 0xbf])), + ); + + expect(plain.hasBom).toBe(false); + expect(embedded.hasBom).toBe(false); + }); +}); diff --git a/packages/app/src/file-explorer/read-result.ts b/packages/app/src/file-explorer/read-result.ts index 693e0a4e1..51b8ee10a 100644 --- a/packages/app/src/file-explorer/read-result.ts +++ b/packages/app/src/file-explorer/read-result.ts @@ -8,8 +8,13 @@ export function explorerFileFromReadResult(file: FileReadResult): ExplorerFile { kind: file.kind, encoding: isText ? "utf-8" : "none", content: isText ? new TextDecoder().decode(file.bytes) : undefined, + hasBom: isText && hasUtf8Bom(file.bytes), mimeType: file.mime, size: file.size, modifiedAt: file.modifiedAt, }; } + +function hasUtf8Bom(bytes: Uint8Array): boolean { + return bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf; +} diff --git a/packages/app/src/file-pane/editor/model.test.ts b/packages/app/src/file-pane/editor/model.test.ts index 9f1655277..026e8bc44 100644 --- a/packages/app/src/file-pane/editor/model.test.ts +++ b/packages/app/src/file-pane/editor/model.test.ts @@ -77,8 +77,17 @@ function ready( return { status: "ready", cwd: "/workspace", path: "file.ts", size, modifiedAt }; } -function makeModel() { - const file = { content: "one", version: ready() as Extract }; +interface MakeModelInput { + content?: string; + hasBom?: boolean; +} + +function makeModel(input: MakeModelInput = {}) { + const file = { + content: input.content ?? "one", + hasBom: input.hasBom ?? false, + version: ready() as Extract, + }; const session = new FileSession(file); const clock = new TestClock(); return { model: new FileEditorModel({ file, session, clock }), session, clock }; @@ -142,10 +151,55 @@ describe("FileEditorModel", () => { expect(model.getSnapshot().status).toBe("clean"); }); + test("keeps CRLF content in file form", async () => { + const { model, session } = makeModel({ content: "one\r\ntwo\r\n" }); + + expect(model.getSnapshot()).toMatchObject({ + content: "one\r\ntwo\r\n", + lineSeparator: "\r\n", + }); + model.edit("one\r\ntwo\r\nthree\r\n"); + await model.save(); + + expect(session.writes).toEqual([ + { + content: "one\r\ntwo\r\nthree\r\n", + expectedModifiedAt: "2026-07-18T00:00:00.000Z", + }, + ]); + }); + + test("restores a UTF-8 BOM before writing a CRLF file", async () => { + const { model, session } = makeModel({ content: "one\r\n", hasBom: true }); + + model.edit("saved\r\n"); + await model.save(); + model.edit("saved again\r\n"); + await model.save(); + + expect(session.writes).toEqual([ + { + content: "\uFEFFsaved\r\n", + expectedModifiedAt: "2026-07-18T00:00:00.000Z", + }, + { + content: "\uFEFFsaved again\r\n", + expectedModifiedAt: "2026-07-18T00:00:01.000Z", + }, + ]); + }); + + test("uses the first line separator when a file mixes styles", () => { + const { model } = makeModel({ content: "one\r\ntwo\nthree\r" }); + + expect(model.getSnapshot().lineSeparator).toBe("\r\n"); + }); + test("reloads a clean editor when the disk version changes", async () => { const { model, session } = makeModel(); session.file = { content: "external", + hasBom: false, version: ready("2026-07-18T00:00:02.000Z", 8) as Extract, }; @@ -155,6 +209,26 @@ describe("FileEditorModel", () => { expect(model.getSnapshot()).toMatchObject({ status: "clean", content: "external" }); }); + test("adopts the format from a clean remote refresh", async () => { + const { model, session } = makeModel({ content: "local\r\n", hasBom: true }); + session.file = { + content: "remote\n", + hasBom: false, + version: ready("2026-07-18T00:00:02.000Z", 7) as Extract, + }; + + model.receiveFileVersion(session.file.version); + await Promise.resolve(); + expect(model.getSnapshot().lineSeparator).toBe("\n"); + model.edit("saved\n"); + await model.save(); + + expect(session.writes.at(-1)).toEqual({ + content: "saved\n", + expectedModifiedAt: "2026-07-18T00:00:02.000Z", + }); + }); + test("coalesces consecutive clean disk updates onto the latest reload", async () => { const { model, session } = makeModel(); const reads: Array<(file: FileEditorFile) => void> = []; @@ -164,9 +238,9 @@ describe("FileEditorModel", () => { model.receiveFileVersion(firstVersion); model.receiveFileVersion(latestVersion); - reads[0]?.({ content: "first", version: firstVersion }); + reads[0]?.({ content: "first", hasBom: false, version: firstVersion }); await Promise.resolve(); - reads[1]?.({ content: "latest", version: latestVersion }); + reads[1]?.({ content: "latest", hasBom: false, version: latestVersion }); await Promise.resolve(); expect(model.getSnapshot()).toMatchObject({ status: "clean", content: "latest" }); @@ -186,6 +260,21 @@ describe("FileEditorModel", () => { expect(model.getSnapshot().status).toBe("clean"); }); + test("keeps the local CRLF and BOM when overwriting a conflict", async () => { + const { model, session } = makeModel({ content: "one\r\n", hasBom: true }); + model.edit("local\r\n"); + model.receiveFileVersion(ready("2026-07-18T00:00:02.000Z", 4)); + + await model.overwrite(); + + expect(session.writes).toEqual([ + { + content: "\uFEFFlocal\r\n", + expectedModifiedAt: "2026-07-18T00:00:02.000Z", + }, + ]); + }); + test("reload discards a conflicted local buffer for the disk contents", async () => { const { model, session } = makeModel(); model.edit("local"); @@ -193,7 +282,7 @@ describe("FileEditorModel", () => { FileVersion, { status: "ready" } >; - session.file = { content: "disk", version: diskVersion }; + session.file = { content: "disk", hasBom: false, version: diskVersion }; model.receiveFileVersion(diskVersion); await model.reload(); @@ -201,6 +290,26 @@ describe("FileEditorModel", () => { expect(model.getSnapshot()).toMatchObject({ status: "clean", content: "disk" }); }); + test("adopts the remote format when reloading a conflict", async () => { + const { model, session } = makeModel({ content: "one\r\n", hasBom: true }); + model.edit("local\r\n"); + const diskVersion = ready("2026-07-18T00:00:02.000Z", 5) as Extract< + FileVersion, + { status: "ready" } + >; + session.file = { content: "disk\n", hasBom: false, version: diskVersion }; + model.receiveFileVersion(diskVersion); + + await model.reload(); + model.edit("saved\n"); + await model.save(); + + expect(session.writes.at(-1)).toEqual({ + content: "saved\n", + expectedModifiedAt: "2026-07-18T00:00:02.000Z", + }); + }); + test("reports failed saves without losing the local buffer", async () => { const { model, session } = makeModel(); session.nextWrite = new Error("disk full"); diff --git a/packages/app/src/file-pane/editor/model.ts b/packages/app/src/file-pane/editor/model.ts index a2b081d7b..48cc77e95 100644 --- a/packages/app/src/file-pane/editor/model.ts +++ b/packages/app/src/file-pane/editor/model.ts @@ -1,10 +1,12 @@ import type { FileVersion, FileWriteResult } from "@getpaseo/protocol/messages"; export type FileEditorStatus = "loading" | "clean" | "dirty" | "saving" | "conflict" | "error"; +export type FileLineSeparator = "\n" | "\r\n" | "\r"; export interface FileEditorSnapshot { status: FileEditorStatus; content: string; + lineSeparator: FileLineSeparator; modified: boolean; version: FileVersion; observedVersion: FileVersion; @@ -13,6 +15,7 @@ export interface FileEditorSnapshot { export interface FileEditorFile { content: string; + hasBom: boolean; version: Extract; } @@ -49,6 +52,7 @@ export class FileEditorModel { private disposed = false; private observedWhileSaving: FileVersion | null = null; private persistedContent: string; + private hasBom: boolean; constructor(input: { file: FileEditorFile; @@ -58,9 +62,11 @@ export class FileEditorModel { this.session = input.session; this.clock = input.clock ?? systemClock; this.persistedContent = input.file.content; + this.hasBom = input.file.hasBom; this.snapshot = { status: "clean", content: input.file.content, + lineSeparator: detectLineSeparator(input.file.content), modified: false, version: input.file.version, observedVersion: input.file.version, @@ -167,10 +173,11 @@ export class FileEditorModel { const content = this.snapshot.content; this.observedWhileSaving = null; this.setSnapshot({ ...this.snapshot, status: "saving", error: null }); + const serializedContent = this.hasBom ? `\uFEFF${content}` : content; let result: FileWriteResult; try { result = await this.session.write({ - content, + content: serializedContent, expectedModifiedAt: expectedVersion.modifiedAt, expectedRevision: expectedVersion.revision, }); @@ -241,9 +248,11 @@ export class FileEditorModel { return; } this.persistedContent = file.content; + this.hasBom = file.hasBom; this.setSnapshot({ status: "clean", content: file.content, + lineSeparator: detectLineSeparator(file.content), modified: false, version: file.version, observedVersion: file.version, @@ -290,6 +299,15 @@ export class FileEditorModel { } } +function detectLineSeparator(content: string): FileLineSeparator { + for (let index = 0; index < content.length; index += 1) { + const character = content.charCodeAt(index); + if (character === 10) return "\n"; + if (character === 13) return content.charCodeAt(index + 1) === 10 ? "\r\n" : "\r"; + } + return "\n"; +} + function sameVersion(left: FileVersion, right: FileVersion): boolean { if (left.status !== right.status || left.cwd !== right.cwd || left.path !== right.path) return false; diff --git a/packages/app/src/file-pane/editor/view.web.tsx b/packages/app/src/file-pane/editor/view.web.tsx index cc211c7a5..abf79f4ed 100644 --- a/packages/app/src/file-pane/editor/view.web.tsx +++ b/packages/app/src/file-pane/editor/view.web.tsx @@ -56,7 +56,8 @@ export function FileEditorView({ update.docChanged && !update.transactions.some((tr) => tr.annotation(remoteUpdate)) ) { - values.model.edit(update.state.doc.toString()); + const { lineSeparator } = values.model.getSnapshot(); + values.model.edit(update.state.doc.sliceString(0, undefined, lineSeparator)); } if (update.selectionSet || update.docChanged) { const head = update.state.selection.main.head; @@ -77,10 +78,12 @@ export function FileEditorView({ useEffect(() => { const view = viewRef.current; - if (!view || view.state.doc.toString() === snapshot.content) return; - const head = Math.min(view.state.selection.main.head, snapshot.content.length); + if (!view) return; + const document = view.state.toText(snapshot.content); + if (view.state.doc.eq(document)) return; + const head = Math.min(view.state.selection.main.head, document.length); view.dispatch({ - changes: { from: 0, to: view.state.doc.length, insert: snapshot.content }, + changes: { from: 0, to: view.state.doc.length, insert: document }, selection: { anchor: head }, annotations: [remoteUpdate.of(true), Transaction.addToHistory.of(false)], }); diff --git a/packages/app/src/file-pane/pane.tsx b/packages/app/src/file-pane/pane.tsx index ef8a74987..db7a4a5ae 100644 --- a/packages/app/src/file-pane/pane.tsx +++ b/packages/app/src/file-pane/pane.tsx @@ -637,9 +637,13 @@ function EditableFilePane({ () => ({ async read(): Promise { const file = await client.readFile(cwd, path); - if (file.kind !== "text") throw new Error("File is no longer text."); + const decodedFile = explorerFileFromReadResult(file); + if (decodedFile.kind !== "text" || decodedFile.content === undefined) { + throw new Error("File is no longer text."); + } return { - content: new TextDecoder().decode(file.bytes), + content: decodedFile.content, + hasBom: decodedFile.hasBom, version: { status: "ready", cwd, @@ -661,6 +665,7 @@ function EditableFilePane({ new FileEditorModel({ file: { content: preview.content ?? "", + hasBom: preview.hasBom, version: { status: "ready", cwd, diff --git a/packages/app/src/stores/session-store.ts b/packages/app/src/stores/session-store.ts index d6e37a171..1d61867f2 100644 --- a/packages/app/src/stores/session-store.ts +++ b/packages/app/src/stores/session-store.ts @@ -278,6 +278,8 @@ export interface ExplorerFile { kind: ExplorerFileKind; encoding: ExplorerEncoding; content?: string; + // TextDecoder removes a leading UTF-8 BOM; retain this bit so file writes can restore it. + hasBom: boolean; mimeType?: string; size: number; modifiedAt: string;