feat(protocol): define streamed diff revisions and resolve tag bases

This commit is contained in:
Mohamed Boudra
2026-11-06 04:01:48 +01:00
parent 3ed57e9dd7
commit 8bc533279a
12 changed files with 749 additions and 8 deletions
@@ -1,3 +1,4 @@
import { decodeDiffFrame, DiffOpcode, type DiffFrame } from "./diff.js";
import { HOST_ROUTE_OPCODE, decodeHostRouteFrame, type HostRouteFrame } from "./host-route.js";
import {
decodeFileTransferFrame,
@@ -12,12 +13,20 @@ import {
} from "./terminal.js";
export type BinaryFrame =
| { kind: "diff"; frame: DiffFrame }
| { kind: "host_route"; frame: HostRouteFrame }
| { kind: "terminal"; frame: TerminalStreamFrame }
| { kind: "file_transfer"; frame: FileTransferFrame }
| { kind: "plugin"; frame: PluginFrame };
export function decodeBinaryFrame(bytes: Uint8Array): BinaryFrame | null {
if (Object.values(DiffOpcode).some((opcode) => opcode === bytes[0])) {
return { kind: "diff", frame: decodeDiffFrame(bytes) };
}
return decodeExistingBinaryFrame(bytes);
}
function decodeExistingBinaryFrame(bytes: Uint8Array): BinaryFrame | null {
switch (bytes[0]) {
case HOST_ROUTE_OPCODE: {
const frame = decodeHostRouteFrame(bytes);
@@ -0,0 +1,240 @@
import { expect, test } from "vitest";
import { decodeDiffFrame, encodeDiffFrame } from "./diff.js";
import { decodeBinaryFrame } from "./demux.js";
import { decodePluginFrame } from "./plugin.js";
test("routes diff revisions independently of plugin frames", () => {
const frame = {
opcode: 0x40 as const,
slot: 0,
revision: 1,
baseRevision: 0,
baseSha: "",
headSha: "",
mergeBaseSha: "",
entries: [],
};
const bytes = encodeDiffFrame(frame);
expect([...bytes]).toEqual([64, 0, 1, 1, 0, 0, 0, 0, 0]);
expect(decodeDiffFrame(bytes)).toEqual(frame);
expect(decodeBinaryFrame(bytes)).toEqual({ kind: "diff", frame });
expect(decodePluginFrame(bytes)).toBeNull();
});
test("roundtrips Unicode runs and materializes the existing file shape", async () => {
const { materializeDiffFile } = await import("./diff.js");
const frame = {
opcode: 0x41 as const,
slot: 255,
revision: 128,
files: [
{
path: "é.ts",
hunks: [
{
oldStart: 1,
oldCount: 1,
newStart: 1,
newCount: 1,
lines: [
{
type: "add" as const,
content: "😀é",
runs: [
{ style: "string", length: 2 },
{ style: null, length: 2 },
],
},
],
},
],
},
],
};
expect(decodeDiffFrame(encodeDiffFrame(frame))).toEqual(frame);
const file = materializeDiffFile(
{
path: "é.ts",
oldPath: null,
change: "modified",
content: "text",
additions: 1,
deletions: 1,
body: true,
},
frame.files[0],
false,
);
expect(file.hunks[0].lines[0].tokens).toEqual([
{ text: "😀", style: "string" },
{ text: "é", style: null },
]);
});
test("preserves binary additions, modifications, deletions and mode-only text", async () => {
const { materializeDiffFile } = await import("./diff.js");
for (const change of ["added", "modified", "deleted"] as const) {
const entry = {
path: "image.png",
oldPath: null,
change,
content: "binary" as const,
additions: 0,
deletions: 0,
body: false,
};
const begin = {
opcode: 0x40 as const,
slot: 1,
revision: 1,
baseRevision: 0,
baseSha: "",
headSha: "",
mergeBaseSha: "",
entries: [entry],
};
expect(decodeDiffFrame(encodeDiffFrame(begin))).toEqual(begin);
expect(materializeDiffFile(entry, null, false)).toEqual({
path: "image.png",
isNew: change === "added",
isDeleted: change === "deleted",
additions: 0,
deletions: 0,
status: "binary",
hunks: [],
});
}
expect(
materializeDiffFile(
{
path: "script",
oldPath: null,
change: "modified",
content: "text",
additions: 0,
deletions: 0,
body: true,
},
{ path: "script", hunks: [] },
false,
),
).toEqual({
path: "script",
isNew: false,
isDeleted: false,
additions: 0,
deletions: 0,
status: "ok",
hunks: [],
});
});
test("rejects every truncated prefix, unknown version, trailing data, and invalid coverage", () => {
const end = {
opcode: 0x42 as const,
slot: 1,
revision: 9007199254740991,
diffTooLarge: true,
omitted: ["a.ts"],
error: { code: "REF_NOT_FOUND" as const, message: "missing" },
};
const bytes = encodeDiffFrame(end);
expect(decodeDiffFrame(bytes)).toEqual(end);
for (let length = 0; length < bytes.length; length++)
expect(() => decodeDiffFrame(bytes.slice(0, length))).toThrow();
const version = bytes.slice();
version[2] = 2;
expect(() => decodeDiffFrame(version)).toThrow("version");
expect(() => decodeDiffFrame(Uint8Array.from([...bytes, 0]))).toThrow("Trailing");
expect(() =>
encodeDiffFrame({
opcode: 0x41,
slot: 1,
revision: 1,
files: [
{
path: "x",
hunks: [
{
oldStart: 0,
oldCount: 0,
newStart: 1,
newCount: 1,
lines: [{ type: "add", content: "😀", runs: [{ style: null, length: 1 }] }],
},
],
},
],
}),
).toThrow("cover");
});
test("uses fixed UTF-8 byte vectors while runs count UTF-16 units", () => {
const bytes = Uint8Array.from([
65, 0, 1, 1, 1, 1, 120, 1, 0, 0, 1, 1, 1, 129, 7, 240, 159, 152, 128, 101, 204, 129, 1, 3, 4,
]);
const frame = {
opcode: 0x41 as const,
slot: 0,
revision: 1,
files: [
{
path: "x",
hunks: [
{
oldStart: 0,
oldCount: 0,
newStart: 1,
newCount: 1,
lines: [
{ type: "add" as const, content: "😀é", runs: [{ style: "string", length: 4 }] },
],
},
],
},
],
};
expect(encodeDiffFrame(frame)).toEqual(bytes);
expect(decodeDiffFrame(bytes)).toEqual(frame);
const unknownStyle = bytes.slice();
unknownStyle[23] = 255;
expect(decodeDiffFrame(unknownStyle)).toEqual({
...frame,
files: [
{
...frame.files[0],
hunks: [
{
...frame.files[0].hunks[0],
lines: [{ type: "add", content: "😀é", runs: [{ style: null, length: 4 }] }],
},
],
},
],
});
});
test("roundtrips generated entry lists across varint boundaries", () => {
for (const revision of [0, 1, 127, 128, 16383, 16384, 4294967296, Number.MAX_SAFE_INTEGER]) {
const entries = Array.from({ length: 130 }, (_, index) => ({
path: `src/${index}-😀.ts`,
oldPath: `old/${index}.ts`,
change: "renamed" as const,
content: "text" as const,
additions: index * 128,
deletions: index,
body: true,
}));
const begin = {
opcode: 0x40 as const,
slot: 255,
revision,
baseRevision: 0,
baseSha: "base",
headSha: "head",
mergeBaseSha: "merge",
entries,
};
expect(decodeDiffFrame(encodeDiffFrame(begin))).toEqual(begin);
}
});
+359
View File
@@ -0,0 +1,359 @@
import type { ParsedDiffFile } from "../messages.js";
// Opcodes 0x40–0x4f are reserved for diff revisions. Plugins own 0x20–0x2f.
export const DiffOpcode = { Begin: 0x40, Bodies: 0x41, End: 0x42 } as const;
export const DIFF_HIGHLIGHT_STYLES = [
null,
"keyword",
"comment",
"string",
"number",
"literal",
"function",
"definition",
"class",
"type",
"tag",
"attribute",
"property",
"variable",
"operator",
"punctuation",
"regexp",
"escape",
"meta",
"heading",
"link",
] as const;
const CHANGES = ["added", "modified", "deleted", "renamed"] as const;
const CONTENTS = ["text", "binary"] as const;
const KINDS = ["context", "add", "remove", "header"] as const;
const ERRORS = [
null,
"NOT_GIT_REPO",
"REF_NOT_FOUND",
"INVALID_COMPARE",
"TOO_MANY_SUBSCRIPTIONS",
"UNKNOWN",
] as const;
export interface DiffEntry {
path: string;
oldPath: string | null;
change: (typeof CHANGES)[number];
content: (typeof CONTENTS)[number];
additions: number;
deletions: number;
body: boolean;
}
export interface DiffRun {
style: string | null;
length: number;
}
export interface DiffStreamLine {
type: (typeof KINDS)[number];
content: string;
runs?: DiffRun[];
}
export interface DiffStreamHunk {
oldStart: number;
oldCount: number;
newStart: number;
newCount: number;
lines: DiffStreamLine[];
}
export interface DiffBody {
path: string;
hunks: DiffStreamHunk[];
}
export interface DiffBegin {
opcode: typeof DiffOpcode.Begin;
slot: number;
revision: number;
baseRevision: number;
baseSha: string;
headSha: string;
mergeBaseSha: string;
entries: DiffEntry[];
}
export interface DiffBodies {
opcode: typeof DiffOpcode.Bodies;
slot: number;
revision: number;
files: DiffBody[];
}
export interface DiffEnd {
opcode: typeof DiffOpcode.End;
slot: number;
revision: number;
diffTooLarge: boolean;
omitted: string[];
error: { code: NonNullable<(typeof ERRORS)[number]>; message: string } | null;
}
export type DiffFrame = DiffBegin | DiffBodies | DiffEnd;
export class InvalidDiffFrameError extends Error {}
class Writer {
private bytes: number[] = [];
byte(value: number): void {
if (!Number.isInteger(value) || value < 0 || value > 255)
throw new InvalidDiffFrameError("Invalid byte");
this.bytes.push(value);
}
uint(value: number): void {
if (!Number.isSafeInteger(value) || value < 0)
throw new InvalidDiffFrameError("Invalid unsigned integer");
do {
const next = value % 128;
value = Math.floor(value / 128);
this.byte(next + (value ? 128 : 0));
} while (value);
}
str(value: string): void {
const bytes = new TextEncoder().encode(value);
this.uint(bytes.length);
for (const byte of bytes) this.byte(byte);
}
finish(): Uint8Array {
return Uint8Array.from(this.bytes);
}
}
class Reader {
private offset = 0;
constructor(private bytes: Uint8Array) {}
byte(): number {
if (this.offset >= this.bytes.length) throw new InvalidDiffFrameError("Truncated diff frame");
return this.bytes[this.offset++];
}
uint(): number {
let value = 0;
let factor = 1;
for (let index = 0; index < 8; index++) {
const byte = this.byte();
value += (byte & 127) * factor;
if (!Number.isSafeInteger(value)) throw new InvalidDiffFrameError("Integer overflow");
if (byte < 128) return value;
factor *= 128;
}
throw new InvalidDiffFrameError("Invalid varint");
}
count(): number {
const value = this.uint();
if (value > this.bytes.length - this.offset)
throw new InvalidDiffFrameError("Count exceeds frame");
return value;
}
str(): string {
const size = this.count();
const start = this.offset;
this.offset += size;
try {
return new TextDecoder("utf-8", { fatal: true }).decode(
this.bytes.subarray(start, this.offset),
);
} catch {
throw new InvalidDiffFrameError("Invalid UTF-8");
}
}
done(): void {
if (this.offset !== this.bytes.length) throw new InvalidDiffFrameError("Trailing bytes");
}
}
function enumAt<T>(values: readonly T[], index: number): T {
if (index >= values.length) throw new InvalidDiffFrameError("Unknown enum value");
return values[index];
}
function validateRuns(line: DiffStreamLine): void {
if (!line.runs) return;
let length = 0;
for (const run of line.runs) {
if (!Number.isSafeInteger(run.length) || run.length <= 0)
throw new InvalidDiffFrameError("Invalid run length");
length += run.length;
}
if (length !== line.content.length) throw new InvalidDiffFrameError("Runs do not cover line");
}
function writeLine(writer: Writer, line: DiffStreamLine): void {
validateRuns(line);
writer.byte(KINDS.indexOf(line.type) | (line.runs ? 128 : 0));
writer.str(line.content);
writer.uint(line.runs ? line.runs.length : 0);
for (const run of line.runs ?? []) {
const style = DIFF_HIGHLIGHT_STYLES.findIndex((value) => value === run.style);
writer.byte(Math.max(0, style));
writer.uint(run.length);
}
}
function readLine(reader: Reader): DiffStreamLine {
const flags = reader.byte();
const type = enumAt(KINDS, flags & 127);
const content = reader.str();
const count = reader.count();
const runs: DiffRun[] = [];
for (let index = 0; index < count; index++) {
const styleIndex = reader.byte();
const style = DIFF_HIGHLIGHT_STYLES[styleIndex] ?? null;
runs.push({ style, length: reader.uint() });
}
if (!(flags & 128)) {
if (count) throw new InvalidDiffFrameError("Runs without flag");
return { type, content };
}
const line = { type, content, runs };
validateRuns(line);
return line;
}
export function encodeDiffFrame(frame: DiffFrame): Uint8Array {
const writer = new Writer();
writer.byte(frame.opcode);
writer.byte(frame.slot);
writer.byte(1);
writer.uint(frame.revision);
if (frame.opcode === DiffOpcode.Begin) {
writer.uint(frame.baseRevision);
writer.str(frame.baseSha);
writer.str(frame.headSha);
writer.str(frame.mergeBaseSha);
writer.uint(frame.entries.length);
for (const entry of frame.entries) {
if (entry.content === "binary" && entry.body)
throw new InvalidDiffFrameError("Binary entry has a body");
writer.str(entry.path);
writer.str(entry.oldPath ?? "");
writer.byte(CHANGES.indexOf(entry.change));
writer.byte(CONTENTS.indexOf(entry.content));
writer.uint(entry.additions);
writer.uint(entry.deletions);
writer.byte(Number(entry.body));
}
} else if (frame.opcode === DiffOpcode.Bodies) {
writer.uint(frame.files.length);
for (const file of frame.files) {
writer.str(file.path);
writer.uint(file.hunks.length);
for (const hunk of file.hunks) {
writer.uint(hunk.oldStart);
writer.uint(hunk.oldCount);
writer.uint(hunk.newStart);
writer.uint(hunk.newCount);
writer.uint(hunk.lines.length);
for (const line of hunk.lines) writeLine(writer, line);
}
}
} else {
writer.byte(Number(frame.diffTooLarge));
writer.uint(frame.omitted.length);
for (const path of frame.omitted) writer.str(path);
writer.byte(frame.error ? ERRORS.indexOf(frame.error.code) : 0);
writer.str(frame.error ? frame.error.message : "");
}
return writer.finish();
}
export function decodeDiffFrame(bytes: Uint8Array): DiffFrame {
const reader = new Reader(bytes);
const opcode = reader.byte();
const slot = reader.byte();
if (reader.byte() !== 1) throw new InvalidDiffFrameError("Unknown diff version");
const revision = reader.uint();
let frame: DiffFrame;
if (opcode === DiffOpcode.Begin) {
const baseRevision = reader.uint();
const baseSha = reader.str();
const headSha = reader.str();
const mergeBaseSha = reader.str();
const count = reader.count();
const entries: DiffEntry[] = [];
for (let index = 0; index < count; index++) {
const path = reader.str();
const oldPath = reader.str() || null;
const change = enumAt(CHANGES, reader.byte());
const content = enumAt(CONTENTS, reader.byte());
const additions = reader.uint();
const deletions = reader.uint();
const bodyByte = reader.byte();
if (bodyByte > 1 || (content === "binary" && bodyByte))
throw new InvalidDiffFrameError("Invalid entry body");
entries.push({ path, oldPath, change, content, additions, deletions, body: bodyByte === 1 });
}
frame = { opcode, slot, revision, baseRevision, baseSha, headSha, mergeBaseSha, entries };
} else if (opcode === DiffOpcode.Bodies) {
const count = reader.count();
const files: DiffBody[] = [];
for (let index = 0; index < count; index++) {
const path = reader.str();
const hunkCount = reader.count();
const hunks: DiffStreamHunk[] = [];
for (let h = 0; h < hunkCount; h++) {
const oldStart = reader.uint();
const oldCount = reader.uint();
const newStart = reader.uint();
const newCount = reader.uint();
const lineCount = reader.count();
const lines: DiffStreamLine[] = [];
for (let l = 0; l < lineCount; l++) lines.push(readLine(reader));
hunks.push({ oldStart, oldCount, newStart, newCount, lines });
}
files.push({ path, hunks });
}
frame = { opcode, slot, revision, files };
} else if (opcode === DiffOpcode.End) {
const flags = reader.byte();
if (flags > 1) throw new InvalidDiffFrameError("Unknown flags");
const count = reader.count();
const omitted: string[] = [];
for (let index = 0; index < count; index++) omitted.push(reader.str());
const code = enumAt(ERRORS, reader.byte());
const message = reader.str();
frame = {
opcode,
slot,
revision,
diffTooLarge: flags === 1,
omitted,
error: code ? { code, message } : null,
};
} else {
throw new InvalidDiffFrameError("Unknown diff opcode");
}
reader.done();
return frame;
}
export function runsToTokens(
line: DiffStreamLine,
): NonNullable<ParsedDiffFile["hunks"][number]["lines"][number]["tokens"]> {
let offset = 0;
return (line.runs ?? []).map((run) => {
const text = line.content.slice(offset, offset + run.length);
offset += run.length;
return { text, style: run.style };
});
}
export function materializeDiffFile(
entry: DiffEntry,
body: DiffBody | null,
omitted: boolean,
): ParsedDiffFile {
let status: "ok" | "binary" | "too_large" = "ok";
if (entry.content === "binary") status = "binary";
else if (omitted) status = "too_large";
const hunks = body
? body.hunks.map((hunk) => ({
...hunk,
lines: hunk.lines.map((line) => {
if (line.runs)
return { type: line.type, content: line.content, tokens: runsToTokens(line) };
return { type: line.type, content: line.content };
}),
}))
: [];
const file: ParsedDiffFile = {
path: entry.path,
isNew: entry.change === "added",
isDeleted: entry.change === "deleted",
additions: entry.additions,
deletions: entry.deletions,
hunks,
status,
};
if (entry.oldPath !== null) file.oldPath = entry.oldPath;
return file;
}
@@ -3,3 +3,4 @@ export * from "./file-transfer.js";
export * from "./terminal.js";
export * from "./host-route.js";
export * from "./plugin.js";
export * from "./diff.js";
@@ -0,0 +1,24 @@
import { expect, test } from "vitest";
import { DiffSubscribeRequestSchema, DiffSubscribeResponseSchema } from "./messages.js";
test("accepts explicit endpoints without changing legacy schemas", () => {
const request = {
type: "diff.subscribe.request",
requestId: "r",
cwd: "/repo",
base: { kind: "rev", rev: "v1" },
head: { kind: "worktree" },
mergeBase: true,
highlight: false,
};
expect(DiffSubscribeRequestSchema.parse(request)).toEqual(request);
expect(
DiffSubscribeRequestSchema.safeParse({ ...request, base: { kind: "unknown" } }).success,
).toBe(false);
expect(
DiffSubscribeResponseSchema.safeParse({
type: "diff.subscribe.response",
payload: { requestId: "r", subscriptionId: "s", slot: 256, error: null },
}).success,
).toBe(false);
});
+50
View File
@@ -2288,6 +2288,51 @@ const CheckoutDiffCompareSchema = z.object({
ignoreWhitespace: z.boolean().optional(),
});
export const DiffEndpointSchema = z.discriminatedUnion("kind", [
z.object({ kind: z.literal("rev"), rev: z.string() }),
z.object({ kind: z.literal("index") }),
z.object({ kind: z.literal("worktree") }),
z.object({ kind: z.literal("empty") }),
]);
export const DiffErrorSchema = z.object({
code: z.enum([
"NOT_GIT_REPO",
"REF_NOT_FOUND",
"INVALID_COMPARE",
"TOO_MANY_SUBSCRIPTIONS",
"UNKNOWN",
]),
message: z.string(),
});
export const DiffSubscribeRequestSchema = z.object({
type: z.literal("diff.subscribe.request"),
requestId: z.string(),
cwd: z.string(),
base: DiffEndpointSchema,
head: DiffEndpointSchema,
mergeBase: z.boolean().optional(),
ignoreWhitespace: z.boolean().optional(),
highlight: z.boolean().optional(),
});
// Unsubscribe is fire-and-forget; release has no correlated response.
export const DiffUnsubscribeRequestSchema = z.object({
type: z.literal("diff.unsubscribe.request"),
subscriptionId: z.string(),
});
export const DiffSubscribeResponseSchema = z.object({
type: z.literal("diff.subscribe.response"),
payload: z.object({
requestId: z.string(),
subscriptionId: z.string(),
slot: z.number().int().min(0).max(255),
error: DiffErrorSchema.nullable(),
}),
});
export type DiffEndpoint = z.infer<typeof DiffEndpointSchema>;
export type DiffError = z.infer<typeof DiffErrorSchema>;
export type DiffSubscribeRequest = z.infer<typeof DiffSubscribeRequestSchema>;
export type DiffSubscribeResponse = z.infer<typeof DiffSubscribeResponseSchema>;
export const CheckoutStatusRequestSchema = z.object({
type: z.literal("checkout_status_request"),
cwd: z.string(),
@@ -3428,6 +3473,8 @@ export const SessionInboundMessageSchema = z.discriminatedUnion("type", [
AgentDetachRequestMessageSchema,
AgentRewindRequestMessageSchema,
AgentPermissionResponseMessageSchema,
DiffSubscribeRequestSchema,
DiffUnsubscribeRequestSchema,
CheckoutStatusRequestSchema,
CheckoutDiffGetRequestSchema,
SubscribeCheckoutDiffRequestSchema,
@@ -3887,6 +3934,7 @@ export const ServerInfoStatusPayloadSchema = z
selectiveAgentTimeline: z.boolean().optional(),
explicitEventSubscriptions: z.boolean().optional(),
ownedSubscriptions: z.boolean().optional(),
diffSubscription: z.boolean().optional(),
binaryChunkingV1: z.boolean().optional(),
workspaceSetupObservation: z.boolean().optional(),
workspaceSortTimes: z.boolean().optional(),
@@ -5428,6 +5476,7 @@ const CheckoutStatusCommonSchema = z.object({
error: CheckoutErrorSchema.nullable(),
requestId: z.string(),
baseRefExact: z.string().nullable().optional(),
comparisonBaseRef: z.string().nullable().optional(),
// The full ref currentBranch tracks, as git resolves `<branch>@{upstream}`:
// "refs/remotes/origin/main", "refs/remotes/upstream/main" on a fork, or a
// "refs/heads/..." ref for a branch tracking a local branch. Null when there is no
@@ -7235,6 +7284,7 @@ export const SessionOutboundMessageSchema = z.discriminatedUnion("type", [
AgentDeletedMessageSchema,
AgentArchivedMessageSchema,
CloseItemsResponseSchema,
DiffSubscribeResponseSchema,
CheckoutStatusResponseSchema,
CheckoutStatusUpdateSchema,
CheckoutDiffGetResponseSchema,