diff --git a/docs/protocol-compatibility.md b/docs/protocol-compatibility.md index f7244e67ba..74650255a5 100644 --- a/docs/protocol-compatibility.md +++ b/docs/protocol-compatibility.md @@ -52,6 +52,33 @@ On capable daemons, connecting creates no timeline or event demand. Client subsc release it on unsubscribe, and restore it after reconnect. Raw message observers inspect traffic without requesting streams. Application caches and which agents are visible remain caller-owned. +## Binary chunking + +`binary_chunking_v1` in hello and optional `server_info.features.binaryChunkingV1` +negotiate connection-scoped framing. Application serializers and routed envelopes use +the same boundary. Small messages and handshakes keep their existing representation. +Older peers retain existing frame limits. + +The reserved binary opcode `0x31` is followed by version `1`, original kind +(`0` text, `1` binary), and START/END flag bits `0`/`1`. Remaining bytes are an +unchanged slice. There is no total length or chunk count; a future receiver can +consume slices incrementally. Today the receiver buffers up to the existing +100 MiB logical limit and decodes UTF-8 only after END. Transfers cannot interleave. +Disconnect clears partial state and negotiation. + +The lower transport supplies its plaintext frame budget, including encryption +overhead. The sender waits for drain between chunks while ordinary messages keep +the existing delivery path. Slow-client protections count messages queued behind +an active transfer as well as bytes already in the socket. +The framing module never selects an encryption construction. + +Native uploads retain the platform's existing queue limits. React Native's +WebSocket does not report its queued bytes, so the client cannot wait for native +drain. Android's installed OkHttp transport closes when a send would put more +than 16 MiB in its queue. The native transport disables outbound chunking, including +on routed channels, and retains its existing single-frame admission errors. +Inbound chunking remains enabled; daemon-to-native responses use the daemon's paced writes. + ## Owned observations `owned_subscriptions` and `server_info.features.ownedSubscriptions` negotiate the source-owned diff --git a/packages/app/e2e/browser/composer-diff-stat.spec.ts b/packages/app/e2e/browser/composer-diff-stat.spec.ts index 9b80222432..959567f58b 100644 --- a/packages/app/e2e/browser/composer-diff-stat.spec.ts +++ b/packages/app/e2e/browser/composer-diff-stat.spec.ts @@ -18,7 +18,7 @@ function visibleMainPane(page: Page) { } function composerChangesPill(page: Page) { - return page.getByTestId("composer-diff-stat-pill"); + return page.getByRole("button", { name: "Open Changes tab", exact: true }); } async function revealComposerChangesInExplorer(page: Page) { @@ -498,6 +498,7 @@ test.describe("send review feedback", () => { test("only open tabs in this workspace appear in compact and wide recipient menus", async ({ page, }) => { + test.setTimeout(120000); const workspace = await seedChangedAgent("review-feedback-multiple-"); const other = await seedChangedAgent("review-feedback-other-"); try { diff --git a/packages/app/e2e/browser/new-workspace-composer-draft.spec.ts b/packages/app/e2e/browser/new-workspace-composer-draft.spec.ts index 906efcfb42..01fd4638eb 100644 --- a/packages/app/e2e/browser/new-workspace-composer-draft.spec.ts +++ b/packages/app/e2e/browser/new-workspace-composer-draft.spec.ts @@ -1,3 +1,4 @@ +import { chooseModel, startWithoutRememberedModel } from "../support/helpers/model-memory"; import { composerLocator, enterComposerFullscreen, @@ -97,6 +98,7 @@ test.describe("New workspace composer draft", () => { const project = await seedWorkspace({ repoPrefix: "new-workspace-submitted-draft-" }); try { + await startWithoutRememberedModel(page); await gotoAppShell(page); await waitForSidebarHydration(page); await openNewWorkspaceComposer(page, { @@ -104,6 +106,7 @@ test.describe("New workspace composer draft", () => { projectDisplayName: project.projectDisplayName, }); + await chooseModel(page, "mock", "Ten second stream"); const composer = page.getByRole("textbox", { name: "Message agent..." }); const createButton = page.getByTestId("workspace-create-submit"); await expect(composer).toBeEditable({ timeout: 30_000 }); diff --git a/packages/app/src/utils/test-daemon-connection.test.ts b/packages/app/src/utils/test-daemon-connection.test.ts index f528065437..29382b5c89 100644 --- a/packages/app/src/utils/test-daemon-connection.test.ts +++ b/packages/app/src/utils/test-daemon-connection.test.ts @@ -76,6 +76,25 @@ describe("test-daemon-connection connectToDaemon", () => { probe = new FakeDaemonProbe(); }); + it.each([ + { id: "direct:lan:6767", type: "directTcp" as const, endpoint: "lan:6767" }, + { + id: "relay:test", + type: "relay" as const, + relayEndpoint: "relay.paseo.sh:443", + daemonPublicKeyB64: "pubkey", + }, + ])("adoptable $type probe clients use the app WebSocket factory", async (connection) => { + const { connectToDaemon } = await import("./test-daemon-connection"); + const { createAppWebSocketFactory } = await import("@/runtime/websocket-factory"); + const result = await connectToDaemon(connection, { serverId: "srv_probe_test" }, probe.deps); + try { + expect(result.client.config.webSocketFactory).toBe(createAppWebSocketFactory()); + } finally { + await result.client.close(); + } + }); + it("reuses the app clientId for direct connections", async () => { const { connectToDaemon } = await import("./test-daemon-connection"); const first = await connectToDaemon( diff --git a/packages/app/src/utils/test-daemon-connection.ts b/packages/app/src/utils/test-daemon-connection.ts index d292c67001..1d1605af62 100644 --- a/packages/app/src/utils/test-daemon-connection.ts +++ b/packages/app/src/utils/test-daemon-connection.ts @@ -6,6 +6,7 @@ import { } from "@getpaseo/client/internal/daemon-client"; import type { DaemonClientConfig } from "@getpaseo/client/internal/daemon-client"; import type { HostConnection } from "@/types/host-connection"; +import { createAppWebSocketFactory } from "@/runtime/websocket-factory"; import { getOrCreateClientId } from "./client-id"; import { resolveAppVersion } from "./app-version"; import { @@ -156,6 +157,7 @@ export async function buildClientConfig( clientType: "mobile" as const, appVersion: deps.resolveAppVersion() ?? undefined, suppressSendErrors: true, + webSocketFactory: createAppWebSocketFactory(), reconnect: { enabled: false }, ...resolveConnectionCredentials(connection, options), ...(options.capabilities ? { capabilities: options.capabilities } : {}), diff --git a/packages/client/src/connection/index.ts b/packages/client/src/connection/index.ts index eda2fec26b..1ad4b34dee 100644 --- a/packages/client/src/connection/index.ts +++ b/packages/client/src/connection/index.ts @@ -125,6 +125,7 @@ export class ConnectionSubscriptions extends OwnedSubscriptions { // Protocol support belongs to the installed client. Only browser hosting needs // a resource supplied by the caller. Keep this exhaustive as the protocol evolves. export const DEFAULT_CLIENT_CAPABILITIES = { + [CLIENT_CAPS.binaryChunkingV1]: true, [CLIENT_CAPS.helloRejection]: true, [CLIENT_CAPS.ownedSubscriptions]: true, [CLIENT_CAPS.allProviders]: true, diff --git a/packages/client/src/daemon-client-relay-e2ee-transport.ts b/packages/client/src/daemon-client-relay-e2ee-transport.ts index 258c0aa9b5..0faecdcb5e 100644 --- a/packages/client/src/daemon-client-relay-e2ee-transport.ts +++ b/packages/client/src/daemon-client-relay-e2ee-transport.ts @@ -1,3 +1,4 @@ +import { maxSecuredPlaintextByteLength } from "@getpaseo/relay"; import { MAX_RELAY_WEBSOCKET_MESSAGE_BYTES } from "@getpaseo/protocol/websocket-message-limit"; import { openClientSecureChannel, @@ -155,18 +156,15 @@ export function createEncryptedTransport( const relayTransport: RelayTransport = { send: (data) => { if (typeof data === "string") { - base.send(data); - return; + return base.send(data); } if (ArrayBuffer.isView(data)) { - base.send(normalizeTransportPayload(data)); - return; + return base.send(normalizeTransportPayload(data)); } if (data instanceof ArrayBuffer) { - base.send(data); - return; + return base.send(data); } - base.send(String(data)); + return base.send(String(data)); }, // The channel decides the close code. Report it now: a socket close event // arrives asynchronously, and the consumer must not act on an earlier @@ -226,6 +224,8 @@ export function createEncryptedTransport( }); return { + supportsDrain: base.supportsDrain, + maxFrameBytes: maxSecuredPlaintextByteLength(MAX_RELAY_WEBSOCKET_MESSAGE_BYTES), send: (data) => { if (!channel) { throw new Error("Encrypted channel not ready"); @@ -239,8 +239,9 @@ export function createEncryptedTransport( // Encryption and its transport handoff are synchronous up to send's first await. // Reject capacity failures here so a routed sender loses only its own channel. assertOutboundCapacity(wireBytes); - void channel.send(payload).catch((error) => { + return channel.send(payload).catch((error) => { emitError(error); + throw error; }); }, close: (code?: number, reason?: string) => { diff --git a/packages/client/src/daemon-client-transport-types.ts b/packages/client/src/daemon-client-transport-types.ts index ddcb57246b..64f84cf8c1 100644 --- a/packages/client/src/daemon-client-transport-types.ts +++ b/packages/client/src/daemon-client-transport-types.ts @@ -1,5 +1,8 @@ export interface DaemonTransport { - send: (data: string | Uint8Array | ArrayBuffer) => void; + /** False when the underlying socket cannot report write drain. */ + supportsDrain?: boolean; + maxFrameBytes?: number; + send: (data: string | Uint8Array | ArrayBuffer) => void | Promise; close: (code?: number, reason?: string) => void; onMessage: (handler: (data: unknown, isBinary: boolean) => void) => () => void; onOpen: (handler: () => void) => () => void; @@ -13,12 +16,16 @@ export type DaemonTransportFactory = (options: { protocols?: string[]; }) => DaemonTransport; -export type WebSocketFactory = ( - url: string, - options?: { headers?: Record; protocols?: string[] }, -) => WebSocketLike; +export interface WebSocketFactory { + ( + url: string, + options?: { headers?: Record; protocols?: string[] }, + ): WebSocketLike; + supportsDrain?: boolean; +} export interface WebSocketLike { + bufferedAmount?: number; readyState: number; send: (data: string | Uint8Array | ArrayBuffer) => void; close: (code?: number, reason?: string) => void; diff --git a/packages/client/src/daemon-client-transport.test.ts b/packages/client/src/daemon-client-transport.test.ts index 963a8b541c..c95c534129 100644 --- a/packages/client/src/daemon-client-transport.test.ts +++ b/packages/client/src/daemon-client-transport.test.ts @@ -175,3 +175,18 @@ describe("daemon-client transport helpers", () => { expect(decodeMessageData(bytes)).toBe("hello"); }); }); + +test("native drain capability survives WebSocket and encryption wrappers", async () => { + const { nativeWebSocketFactory } = await import("./daemon-client-websocket-transport.js"); + expect(nativeWebSocketFactory.supportsDrain).toBe(false); + const factory = Object.assign(() => ({ readyState: 1, send() {}, close() {} }), { + supportsDrain: false, + }); + const base = createWebSocketTransportFactory(factory)({ url: "ws://test" }); + expect(base.supportsDrain).toBe(false); + const encrypted = createEncryptedTransport(base, { + daemonPublicKeyB64: "unused-key", + logger: { warn() {} }, + }); + expect(encrypted.supportsDrain).toBe(false); +}); diff --git a/packages/client/src/daemon-client-websocket-transport.ts b/packages/client/src/daemon-client-websocket-transport.ts index f5734b4b51..c7ec87555d 100644 --- a/packages/client/src/daemon-client-websocket-transport.ts +++ b/packages/client/src/daemon-client-websocket-transport.ts @@ -39,6 +39,9 @@ export function nativeWebSocketFactory( return new (getGlobalWebSocket())(url, options?.protocols, { headers: options?.headers }); } +// React Native leaves bufferedAmount at zero, including while native output is queued. +nativeWebSocketFactory.supportsDrain = false; + export function createWebSocketTransportFactory(factory: WebSocketFactory): DaemonTransportFactory { return ({ url, headers, protocols }) => { const ws = factory(url, { headers, protocols }); @@ -50,11 +53,28 @@ export function createWebSocketTransportFactory(factory: WebSocketFactory): Daem } } return { + supportsDrain: factory.supportsDrain !== false, send: (data) => { if (typeof ws.readyState === "number" && ws.readyState !== 1) { throw new Error(`WebSocket not open (readyState=${ws.readyState})`); } ws.send(data); + if ((ws.bufferedAmount ?? 0) > 1024 * 1024) { + return new Promise((resolve, reject) => { + const drain = () => { + if (ws.readyState !== 1) { + reject(new Error("WebSocket closed while draining")); + return; + } + if ((ws.bufferedAmount ?? 0) <= 1024 * 1024) { + resolve(); + return; + } + setTimeout(drain, 10); + }; + drain(); + }); + } }, close: (code?: number, reason?: string) => { // Node's `ws` may emit an `error` when a connecting socket is closed before the diff --git a/packages/client/src/daemon-client.test.ts b/packages/client/src/daemon-client.test.ts index 112113caf6..6ef72a0e1a 100644 --- a/packages/client/src/daemon-client.test.ts +++ b/packages/client/src/daemon-client.test.ts @@ -1094,6 +1094,7 @@ test("advertises client capabilities in hello", async () => { provider_snapshot_references: true, explicit_event_subscriptions: true, owned_subscriptions: true, + binary_chunking_v1: true, compact_provider_snapshots: true, custom_mode_icons: true, project_updates: true, @@ -6817,91 +6818,102 @@ test("drops legacy agent_stream tool_call payloads and logs validation warning", expect(logger.warn).toHaveBeenCalled(); }); -test("parses canonical fetch_agent_timeline_response payloads without crashing", async () => { - const logger = createMockLogger(); - const mock = createMockTransport(); +test.each([false, true])( + "parses canonical timeline responses with chunking=%s", + async (chunked) => { + const logger = createMockLogger(); + const mock = createMockTransport(); - const client = new DaemonClient({ - url: "ws://test", - clientId: "clsk_unit_test", - logger, - reconnect: { enabled: false }, - transportFactory: () => mock.transport, - }); - clients.push(client); + const client = new DaemonClient({ + url: "ws://test", + clientId: "clsk_unit_test", + logger, + reconnect: { enabled: false }, + transportFactory: () => mock.transport, + }); + clients.push(client); - const connectPromise = client.connect(); - mock.triggerOpen(); - await connectPromise; + const connectPromise = client.connect(); + mock.triggerOpen({ features: { ownedSubscriptions: true, binaryChunkingV1: chunked } }); + await connectPromise; - const received: unknown[] = []; - const unsubscribe = client.on("fetch_agent_timeline_response", (msg) => { - received.push(msg); - }); + const received: unknown[] = []; + const unsubscribe = client.on("fetch_agent_timeline_response", (msg) => { + received.push(msg); + }); - mock.triggerMessage( - wrapSessionMessage({ - type: "fetch_agent_timeline_response", + const command = chunked ? "x".repeat(50 * 1024 * 1024) : "pwd"; + const { ChunkedTransport } = await import("@getpaseo/protocol/chunked-transport"); + const sender = new ChunkedTransport({ + frameBytes: 24 * 1024 * 1024, + send: (frame) => mock.triggerMessage(frame), + deliver: () => {}, + }); + sender.enabled = chunked; + await sender.send( + wrapSessionMessage({ + type: "fetch_agent_timeline_response", + payload: { + requestId: "req-1", + agentId: "agent_cli", + agent: null, + direction: "tail", + projection: "projected", + epoch: "epoch-1", + reset: false, + staleCursor: false, + gap: false, + window: { minSeq: 1, maxSeq: 1, nextSeq: 2 }, + startCursor: { epoch: "epoch-1", seq: 1 }, + endCursor: { epoch: "epoch-1", seq: 1 }, + hasOlder: false, + hasNewer: false, + entries: [ + { + timestamp: "2026-02-08T20:20:00.000Z", + provider: "codex", + seqStart: 1, + seqEnd: 1, + sourceSeqRanges: [{ startSeq: 1, endSeq: 1 }], + collapsed: [], + item: { + type: "tool_call", + callId: "call_cli_snapshot", + name: "shell", + status: "running", + detail: { + type: "shell", + command, + }, + error: null, + }, + }, + ], + error: null, + }, + }), + ); + + unsubscribe(); + + expect(received).toHaveLength(1); + expect(received[0]).toMatchObject({ payload: { - requestId: "req-1", - agentId: "agent_cli", - agent: null, - direction: "tail", - projection: "projected", - epoch: "epoch-1", - reset: false, - staleCursor: false, - gap: false, - window: { minSeq: 1, maxSeq: 1, nextSeq: 2 }, - startCursor: { epoch: "epoch-1", seq: 1 }, - endCursor: { epoch: "epoch-1", seq: 1 }, - hasOlder: false, - hasNewer: false, entries: [ { - timestamp: "2026-02-08T20:20:00.000Z", - provider: "codex", - seqStart: 1, - seqEnd: 1, - sourceSeqRanges: [{ startSeq: 1, endSeq: 1 }], - collapsed: [], item: { type: "tool_call", - callId: "call_cli_snapshot", - name: "shell", status: "running", - detail: { - type: "shell", - command: "pwd", - }, error: null, + detail: { type: "shell", command }, }, }, ], - error: null, }, - }), - ); - - unsubscribe(); - - expect(received).toHaveLength(1); - expect(received[0]).toMatchObject({ - payload: { - entries: [ - { - item: { - type: "tool_call", - status: "running", - error: null, - detail: { type: "shell" }, - }, - }, - ], - }, - }); - expect(logger.warn).not.toHaveBeenCalled(); -}); + }); + expect(logger.warn).not.toHaveBeenCalled(); + }, +); test("rejects and logs a correlated response that violates the protocol schema", async () => { const mock = createMockTransport(); @@ -8471,3 +8483,302 @@ test("closes before admission when a protected-capable daemon is reached in lega expect(daemonReceived).toHaveLength(1); expect(JSON.parse(daemonReceived[0] as string)).toMatchObject({ type: "hello" }); }); + +test("uses the frame budget selected by an asynchronously opened transport", async () => { + const mock = createMockTransport(); + mock.transport.maxFrameBytes = 1024; + const client = new DaemonClient({ + url: "ws://test", + clientId: "dynamic-budget", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.transport.maxFrameBytes = 16; + mock.triggerOpen({ features: { binaryChunkingV1: true } }); + await connecting; + client.sendHostFrame("route", new Uint8Array(100)); + await vi.waitFor(() => expect(mock.sent.length).toBeGreaterThan(1)); + expect(mock.sent.every((frame) => typeof frame !== "string" && frame.byteLength <= 16)).toBe( + true, + ); +}); + +test.each([false, true])("routed sends wait for origin drain (binary=%s)", async (binary) => { + const mock = createMockTransport(); + let drain!: () => void; + let blocked = false; + const send = mock.transport.send; + mock.transport.send = (frame) => { + send(frame); + if (blocked) + return new Promise((resolve) => { + drain = resolve; + }); + }; + const client = new DaemonClient({ + url: "ws://test", + clientId: "routed-drain", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen({ features: { binaryChunkingV1: true } }); + await connecting; + blocked = true; + let finished = false; + const write = Promise.resolve( + client.sendHostFrame("route", binary ? new Uint8Array([1]) : "hello"), + ).then(() => { + finished = true; + return undefined; + }); + await Promise.resolve(); + expect(finished).toBe(false); + drain(); + await write; + expect(finished).toBe(true); +}); + +test("a queued admission failure rejects its RPC without closing the client", async () => { + const mock = createMockTransport(); + const send = mock.transport.send; + let block = false; + let drain!: () => void; + mock.transport.send = (frame) => { + if (!block) return send(frame); + if (typeof frame !== "string") + return new Promise((resolve) => { + drain = resolve; + }); + throw new Error("capacity"); + }; + const client = new DaemonClient({ + url: "ws://test", + clientId: "queued-admission", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen(); + await connecting; + block = true; + const first = client.sendHostFrame("route", new Uint8Array([1])); + const ping = client.ping({ timeoutMs: 1000 }); + void ping.catch(() => {}); + drain(); + await first; + await expect(ping).rejects.toThrow("capacity"); + expect(client.isConnected).toBe(true); +}); + +test("RPC deadlines remain active while a preceding frame waits for drain", async () => { + const mock = createMockTransport(); + const send = mock.transport.send; + let blocked = false; + let drain!: () => void; + mock.transport.send = (frame) => { + if (!blocked) return send(frame); + if (typeof frame !== "string") + return new Promise((resolve) => { + drain = resolve; + }); + }; + const client = new DaemonClient({ + url: "ws://test", + clientId: "drain-deadline", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen(); + await connecting; + blocked = true; + const first = client.sendHostFrame("route", new Uint8Array([1])); + try { + await expect(client.ping({ timeoutMs: 20 })).rejects.toThrow("Timeout waiting for message"); + expect(client.isConnected).toBe(true); + } finally { + drain(); + await first; + } +}); + +test.each([false, true])( + "event listener errors do not close the transport (chunked=%s)", + async (chunked) => { + const mock = createMockTransport(); + const closed = vi.fn(); + mock.transport.close = closed; + const client = new DaemonClient({ + url: "ws://test", + clientId: "listener-error", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen({ features: { binaryChunkingV1: chunked } }); + await connecting; + const unsubscribe = client.subscribe(() => { + throw new Error("listener failed"); + }); + const text = wrapSessionMessage({ + type: "agent_deleted", + payload: { agentId: "agent-test", requestId: "listener-test" }, + }); + const payload = new TextEncoder().encode(text); + const frame = chunked ? new Uint8Array([0x31, 1, 0, 3, ...payload]) : text; + expect(() => mock.triggerMessage(frame)).toThrow("listener failed"); + expect(closed).not.toHaveBeenCalled(); + unsubscribe(); + const events: string[] = []; + client.subscribe((event) => { + events.push(event.type); + }); + mock.triggerMessage(frame); + expect(events).toEqual(["agent_deleted"]); + await client.sendHostFrame("route", "still connected"); + expect(client.isConnected).toBe(true); + }, +); + +test("malformed chunk metadata still closes the client transport", async () => { + const mock = createMockTransport(); + const closed = vi.fn(); + mock.transport.close = closed; + const client = new DaemonClient({ + url: "ws://test", + clientId: "malformed-chunk", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen({ features: { binaryChunkingV1: true } }); + await connecting; + mock.triggerMessage(new Uint8Array([0x31, 2, 0, 3, 65])); + expect(closed).toHaveBeenCalledWith(4003, "Invalid chunked transport"); +}); + +test.each([FileTransferOpcode.FileBegin, FileTransferOpcode.FileChunk, FileTransferOpcode.FileEnd])( + "upload stops at a queued pre-write rejection (opcode=%s)", + async (opcode) => { + const mock = createMockTransport(); + const send = mock.transport.send; + let drain!: () => void; + let blocked = false; + const attempts: number[] = []; + mock.transport.send = (frame) => { + if (typeof frame !== "string") { + const decoded = decodeFileTransferFrame(assertUint8Array(frame)); + if (decoded) { + attempts.push(decoded.opcode); + if (decoded.opcode === opcode) throw new Error("upload capacity"); + } else if (blocked) + return new Promise((resolve) => { + drain = resolve; + }); + } + return send(frame); + }; + const client = new DaemonClient({ + url: "ws://test", + clientId: "upload-admission", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen(); + await connecting; + blocked = true; + const first = client.sendHostFrame("route", new Uint8Array([1])); + const upload = client.uploadFile({ + fileName: "file.bin", + mimeType: "application/octet-stream", + bytes: new Uint8Array([1, 2]), + chunkSize: 1, + }); + let rejected: unknown; + void upload.catch((error) => { + rejected = error; + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + drain(); + await first; + await vi.waitFor(() => expect(rejected).toMatchObject({ message: "upload capacity" }), { + timeout: 500, + }); + expect(attempts.at(-1)).toBe(opcode); + expect(client.isConnected).toBe(true); + }, +); + +test.each(["latency", "dictation start", "dictation finish"])( + "queued control send rejection reaches its caller (%s)", + async (operation) => { + const mock = createMockTransport(); + const send = mock.transport.send; + let blocked = false; + let drain!: () => void; + mock.transport.send = (frame) => { + if (!blocked) return send(frame); + if (typeof frame !== "string") + return new Promise((resolve) => { + drain = resolve; + }); + throw new Error("control capacity"); + }; + const client = new DaemonClient({ + url: "ws://test", + clientId: "control-admission", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen(); + await connecting; + blocked = true; + const first = client.sendHostFrame("route", new Uint8Array([1])); + let result: Promise; + if (operation === "latency") result = client.measureLatency({ timeoutMs: 1000 }); + else if (operation === "dictation start") + result = client.startDictationStream("dictation", "pcm"); + else result = client.finishDictationStream("dictation", 0); + const rejected = expect(result).rejects.toThrow("control capacity"); + drain(); + await first; + await rejected; + expect(client.isConnected).toBe(true); + }, +); + +test("routed channels inherit an origin without drain support", async () => { + const { createHostChannelTransport } = await import("./hosts/index.js"); + const mock = createMockTransport(); + mock.transport.supportsDrain = false; + const client = new DaemonClient({ + url: "ws://test", + clientId: "unpaced-route", + transportFactory: () => mock.transport, + reconnect: { enabled: false }, + }); + clients.push(client); + const connecting = client.connect(); + mock.triggerOpen(); + await connecting; + const route = createHostChannelTransport( + client, + "srv_destination", + "scope", + "route", + )({ url: "ws://route" }); + expect(route.supportsDrain).toBe(false); + route.close(); +}); diff --git a/packages/client/src/daemon-client.ts b/packages/client/src/daemon-client.ts index d4b0f28a39..9152159f1c 100644 --- a/packages/client/src/daemon-client.ts +++ b/packages/client/src/daemon-client.ts @@ -1,3 +1,7 @@ +import { + ChunkedTransport, + ChunkedTransportProtocolError, +} from "@getpaseo/protocol/chunked-transport"; import { HOST_ROUTE_OPCODE, encodeHostRouteFrame, @@ -1266,7 +1270,7 @@ function toReasonCode(reason: string | null | undefined): string | null { } interface PendingSend { - send: () => void; + send: () => void | Promise; resolve: () => void; reject: (error: Error) => void; timeoutHandle: ReturnType; @@ -1366,6 +1370,7 @@ export class DaemonClient { failed: (error) => this.logger.error({ err: error }, "Subscription failed"), }); private transport: DaemonTransport | null = null; + private chunkedTransport: ChunkedTransport | null = null; private helloAuth: HelloAuth; private transportCleanup: Array<() => void> = []; private rawMessageListeners: Set<(message: SessionOutboundMessage) => void> = new Set(); @@ -1560,6 +1565,17 @@ export class DaemonClient { }); this.traceConnectionAttempt += 1; this.transport = transport; + this.chunkedTransport = new ChunkedTransport({ + get supportsDrain() { + return transport.supportsDrain; + }, + get frameBytes() { + return transport.maxFrameBytes ?? 32 * 1024 * 1024; + }, + send: (frame) => transport.send(frame), + onFailure: () => transport.close(4003, "Transport send failed"), + deliver: (frame) => this.handleTransportMessage(frame), + }); this.lastServerInfoMessage = null; this.authFailureReasonValue = null; @@ -1666,7 +1682,16 @@ export class DaemonClient { }); }), transport.onMessage((data) => { - if (this.transport === transport) this.handleTransportMessage(data); + if (this.transport !== transport) return; + const bytes = asUint8Array(data); + try { + if (typeof data === "string" || bytes) + this.chunkedTransport!.receive(typeof data === "string" ? data : bytes!); + else this.handleTransportMessage(data); + } catch (error) { + if (!(error instanceof ChunkedTransportProtocolError)) throw error; + transport.close(4003, "Invalid chunked transport"); + } }), ]; } catch (error) { @@ -1867,9 +1892,9 @@ export class DaemonClient { }) ).routeId; } - sendHostFrame(routeId: string, data: string | Uint8Array): void { + sendHostFrame(routeId: string, data: string | Uint8Array): Promise { if (typeof data === "string") { - this.sendSessionMessageStrict({ + return this.sendSessionMessageStrict({ type: "hosts.route.frame.request", routeId, data, @@ -1877,7 +1902,7 @@ export class DaemonClient { }); } else { if (!this.isConnected || !this.transport) throw this.unavailableError(); - this.sendTransportFrame(encodeHostRouteFrame(routeId, data)); + return this.sendTransportFrame(encodeHostRouteFrame(routeId, data)); } } private readonly hostFrameListeners = new Set<(frame: HostRouteFrame) => void>(); @@ -1935,6 +1960,11 @@ export class DaemonClient { // Message Subscription // ============================================================================ + /** Drain capability inherited by transports routed through this connection. */ + get supportsDrain(): boolean { + return this.transport?.supportsDrain !== false; + } + subscribe(handler: DaemonEventHandler): () => void { this.eventListeners.add(handler); return () => { @@ -2043,7 +2073,7 @@ export class DaemonClient { messageType: string, message: unknown, submitted?: () => void, - ): void { + ): Promise { const isOpen = this.config.trace?.isEnabled() === true && this.beginTraceSection("paseo.ws.message.outbound", { @@ -2052,25 +2082,30 @@ export class DaemonClient { ...describeRequestIdentity(message), }); try { - this.sendTransportFrame(JSON.stringify(message), submitted); + return this.sendTransportFrame(JSON.stringify(message), submitted); } finally { this.endTraceSection(isOpen); } } - private sendSessionPayload(payload: SessionInboundMessage): void { - this.sendJsonMessage("session", payload.type, { type: "session", message: payload }, () => { - if (!("requestId" in payload) || typeof payload.requestId !== "string") return; - for (const waiter of this.waiters) { - if (waiter.requestId === payload.requestId) waiter.submitted = true; - } - }); + private sendSessionPayload(payload: SessionInboundMessage): Promise { + return this.sendJsonMessage( + "session", + payload.type, + { type: "session", message: payload }, + () => { + if (!("requestId" in payload) || typeof payload.requestId !== "string") return; + for (const waiter of this.waiters) { + if (waiter.requestId === payload.requestId) waiter.submitted = true; + } + }, + ); } private sendTransportFrame( frame: string | Uint8Array | ArrayBuffer, submitted?: () => void, - ): void { + ): Promise { if (!this.transport) { throw this.unavailableError(); } @@ -2083,7 +2118,8 @@ export class DaemonClient { }); try { submitted?.(); - this.transport.send(frame); + const payload = typeof frame === "string" ? frame : asUint8Array(frame)!; + return this.chunkedTransport!.send(payload); } finally { this.endTraceSection(isOpen); } @@ -2112,10 +2148,10 @@ export class DaemonClient { } } - private sendBinaryFrame(frame: Uint8Array): void { + private sendBinaryFrame(frame: Uint8Array): Promise { if (!this.transport || this.connectionState.status !== "connected") { if (this.config.suppressSendErrors) { - return; + return Promise.resolve(); } throw new Error(`Transport not connected (status: ${this.connectionState.status})`); } @@ -2124,10 +2160,10 @@ export class DaemonClient { envelopeType: "binary", messageType: "binary", }); - this.sendTransportFrame(frame); + return this.sendTransportFrame(frame); } catch (error) { if (this.config.suppressSendErrors) { - return; + return Promise.resolve(); } throw error instanceof Error ? error : new Error(String(error)); } @@ -2142,7 +2178,7 @@ export class DaemonClient { private sendSessionMessageOrThrow(message: SessionInboundMessage): Promise { return this.sendWhenConnected(() => { const payload = SessionInboundMessageSchema.parse(message); - this.sendSessionPayload(payload); + return this.sendSessionPayload(payload); }); } @@ -2151,13 +2187,12 @@ export class DaemonClient { return this.sendWhenConnected(() => undefined); } - private sendWhenConnected(send: () => void): Promise { + private sendWhenConnected(send: () => void | Promise): Promise { const status = this.connectionState.status; // If connected, send immediately if (this.transport && status === "connected") { - send(); - return Promise.resolve(); + return Promise.resolve(send()); } // If connecting, queue the message to be sent once connected @@ -2196,8 +2231,7 @@ export class DaemonClient { clearTimeout(pending.timeoutHandle); try { if (this.transport && this.connectionState.status === "connected") { - pending.send(); - pending.resolve(); + void Promise.resolve(pending.send()).then(pending.resolve, pending.reject); } else { pending.reject(new DaemonConnectionError("Connection lost before message could be sent")); } @@ -2253,19 +2287,17 @@ export class DaemonClient { { ...params.options, requestId: params.requestId }, ); + // The response deadline still applies while the transport waits for drain. + void promise.catch(() => undefined); try { - await this.sendSessionMessageOrThrow(wire.message); + const sent = this.sendSessionMessageOrThrow(wire.message); + const result = await Promise.race([promise, sent.then(() => promise)]); + if (result.kind === "error") throw result.error; + return result.value; } catch (error) { const err = error instanceof Error ? error : new Error(String(error)); cancel(err); - void promise.catch(() => undefined); throw err; - } - - try { - const result = await promise; - if (result.kind === "error") throw result.error; - return result.value; } finally { await wire.finish(); } @@ -2355,13 +2387,13 @@ export class DaemonClient { }); } - private sendSessionMessageStrict(message: SessionInboundMessage): void { + private sendSessionMessageStrict(message: SessionInboundMessage): Promise { if (!this.transport || this.connectionState.status !== "connected") { throw this.unavailableError(); } const payload = SessionInboundMessageSchema.parse(message); try { - this.sendSessionPayload(payload); + return this.sendSessionPayload(payload); } catch (error) { throw error instanceof Error ? error : new Error(String(error)); } @@ -2559,15 +2591,16 @@ export class DaemonClient { }; this.pingProbe = probe; - try { - this.sendJsonMessage("ping", "ping", { type: "ping" }); - } catch (error) { - this.clearPingProbe(); + const rejectSend = (error: unknown) => { + if (this.pingProbe !== probe) return; const sendError = error instanceof Error ? error : new Error(String(error)); - if (probe.drivesLivenessFailure) { - this.recordLivenessFailure(sendError); - } - return Promise.reject(sendError); + this.rejectPingProbe(sendError); + if (probe.drivesLivenessFailure) this.recordLivenessFailure(sendError); + }; + try { + void this.sendJsonMessage("ping", "ping", { type: "ping" }).catch(rejectSend); + } catch (error) { + rejectSend(error); } return promise; @@ -4222,7 +4255,7 @@ export class DaemonClient { } async sendVoiceAudioChunk(audio: string, format: string, isLast = false): Promise { - this.sendSessionMessage({ type: "voice_audio_chunk", audio, format, isLast }); + return this.sendSessionMessageStrict({ type: "voice_audio_chunk", audio, format, isLast }); } async startDictationStream(dictationId: string, format: string): Promise { @@ -4263,8 +4296,13 @@ export class DaemonClient { const cleanupError = new Error("Cancelled dictation start waiter"); try { - this.sendSessionMessageStrict({ type: "dictation_stream_start", dictationId, format }); - await Promise.race([ackPromise, errorPromise]); + const sent = this.sendSessionMessageStrict({ + type: "dictation_stream_start", + dictationId, + format, + }); + const response = Promise.race([ackPromise, errorPromise]); + await Promise.race([response, sent.then(() => response)]); } finally { ack.cancel(cleanupError); streamError.cancel(cleanupError); @@ -4399,12 +4437,17 @@ export class DaemonClient { const cleanupError = new Error("Cancelled dictation finish waiter"); try { - this.sendSessionMessageStrict({ type: "dictation_stream_finish", dictationId, finalSeq }); - const firstOutcome = await Promise.race([ + const sent = this.sendSessionMessageStrict({ + type: "dictation_stream_finish", + dictationId, + finalSeq, + }); + const response = Promise.race([ finalOutcomePromise, errorOutcomePromise, finishAcceptedOutcomePromise, ]); + const firstOutcome = await Promise.race([response, sent.then(() => response)]); if (firstOutcome.kind === "final") { return firstOutcome.payload; @@ -5320,8 +5363,10 @@ export class DaemonClient { return undefined; }, ); + const sendFrame = (frame: Uint8Array) => + Promise.race([this.sendBinaryFrame(frame), responsePromise]); try { - this.sendBinaryFrame( + await sendFrame( encodeFileTransferFrame({ opcode: FileTransferOpcode.FileBegin, requestId: resolvedRequestId, @@ -5335,6 +5380,7 @@ export class DaemonClient { }), ); + if (settled) return await responsePromise; const chunkSize = input.chunkSize ?? 128 * 1024; for (let offset = 0; offset < bytes.byteLength; offset += chunkSize) { // Native WebSocket.send encodes binary synchronously. Let rendering and @@ -5344,7 +5390,7 @@ export class DaemonClient { if (this.transport !== uploadTransport || this.connectionState.status !== "connected") { throw new DaemonConnectionError("Connection changed during file upload"); } - this.sendBinaryFrame( + await sendFrame( encodeFileTransferFrame({ opcode: FileTransferOpcode.FileChunk, requestId: resolvedRequestId, @@ -5353,7 +5399,7 @@ export class DaemonClient { ); } - this.sendBinaryFrame( + await sendFrame( encodeFileTransferFrame({ opcode: FileTransferOpcode.FileEnd, requestId: resolvedRequestId, @@ -6763,6 +6809,8 @@ export class DaemonClient { } private disposeTransport(code = 1001, reason = "Reconnecting"): void { + this.chunkedTransport?.close(); + this.chunkedTransport = null; this.owned.disconnected(); this.providerSnapshotUpdates.clear(); this.stopLivenessHeartbeat(); @@ -7232,15 +7280,6 @@ export class DaemonClient { probe.resolve(perfNow() - probe.startedAt); } - private clearPingProbe(): void { - const probe = this.pingProbe; - if (!probe) { - return; - } - this.pingProbe = null; - clearTimeout(probe.timeoutHandle); - } - private rejectPingProbe(error: Error): void { const probe = this.pingProbe; if (!probe) { @@ -7289,6 +7328,10 @@ export class DaemonClient { if (consumerMessage.type === "status") { const serverInfo = parseServerInfoStatusPayload(consumerMessage.payload); if (serverInfo) { + if (this.chunkedTransport) + this.chunkedTransport.enabled = + serverInfo.features?.binaryChunkingV1 === true && + this.config.capabilities?.[CLIENT_CAPS.binaryChunkingV1] !== false; this.lastServerInfoMessage = serverInfo; this.destinationServerId = serverInfo.serverId; this.connectionFailure = undefined; diff --git a/packages/client/src/hosts/index.ts b/packages/client/src/hosts/index.ts index 1baa9ebe43..aa5c910158 100644 --- a/packages/client/src/hosts/index.ts +++ b/packages/client/src/hosts/index.ts @@ -1,3 +1,4 @@ +import { MAX_HOST_CHANNEL_FRAME_BYTES } from "@getpaseo/protocol/binary-frames/index"; import { safeRandomId } from "../daemon-client-transport-utils.js"; import { DaemonClient } from "../daemon-client.js"; import type { DaemonTransportFactory } from "../daemon-client-transport-types.js"; @@ -84,13 +85,17 @@ export function createHostChannelTransport( }) .catch(close); return { + get supportsDrain() { + return origin.supportsDrain; + }, + maxFrameBytes: MAX_HOST_CHANNEL_FRAME_BYTES, send(data) { if (closed || !routeId) throw Object.assign(new Error(`Host ${serverId} is offline`), { code: "host_offline", serverId, }); - origin.sendHostFrame( + return origin.sendHostFrame( routeId, typeof data === "string" || data instanceof Uint8Array ? data : new Uint8Array(data), ); diff --git a/packages/client/src/node/connector.test.ts b/packages/client/src/node/connector.test.ts index 6e916d4bec..882a2f5706 100644 --- a/packages/client/src/node/connector.test.ts +++ b/packages/client/src/node/connector.test.ts @@ -44,12 +44,14 @@ test("a slow consumer cannot exceed the shared byte budget and close releases re socket.pause(); await ready; const frame = new Uint8Array(1024 * 1024); - for (let index = 0; index < 4; index++) transport.send(frame); + const writes = Array.from({ length: 4 }, () => transport.send(frame)); + const drained = Promise.allSettled(writes); expect(budget.bufferedBytes).toBe(4 * 1024 * 1024); expect(() => transport.send(frame)).toThrow("Connected host buffer limit reached"); expect(budget.bufferedBytes).toBeLessThanOrEqual(budget.limit); transport.close(); await expect.poll(() => budget.bufferedBytes).toBe(0); + await drained; }); test("direct connector preserves 25 MiB binary frames in both directions", async () => { const server = await endpoint(); @@ -65,12 +67,13 @@ test("direct connector preserves 25 MiB binary frames in both directions", async await ready; const bytes = new Uint8Array(25 * 1024 * 1024).fill(173); const echoed = once(socket, "message"); - transport.send(bytes); + const sending = transport.send(bytes); const [sent, binary] = await echoed; expect(binary).toBe(true); expect(Buffer.from(sent as Uint8Array).equals(bytes)).toBe(true); socket.send(bytes); expect(Buffer.from(await received).equals(bytes)).toBe(true); + await sending; }); import { @@ -124,24 +127,29 @@ test("encrypted connector rejects overload synchronously and preserves the physi }); await new Promise((resolve) => transport.onOpen(resolve)); await ready; + expect(transport.maxFrameBytes).toBeLessThan(32 * 1024 * 1024); peer.pause(); const frame = new Uint8Array(1024 * 1024).fill(173); let rejected: unknown; + const writes: (void | Promise)[] = []; for (let index = 0; index < 8; index++) { try { - transport.send(frame); + writes.push(transport.send(frame)); } catch (error) { rejected = error; break; } } + const drained = Promise.allSettled(writes); expect(rejected).toMatchObject({ code: "resource_limit" }); expect(budget.bufferedBytes).toBeLessThanOrEqual(budget.limit); await new Promise((resolve) => setImmediate(resolve)); expect(errors).toEqual([]); expect(closed).toBe(false); - transport.send("sibling control"); + const sibling = transport.send("sibling control"); peer.resume(); + await drained; + await sibling; await expect.poll(() => received.includes("sibling control")).toBe(true); expect(received.filter((data) => data instanceof ArrayBuffer).length).toBeGreaterThan(0); expect( diff --git a/packages/client/src/node/index.ts b/packages/client/src/node/index.ts index 46b5cff17a..30d2b8744f 100644 --- a/packages/client/src/node/index.ts +++ b/packages/client/src/node/index.ts @@ -172,9 +172,15 @@ export function createHostTransportFactory( close(); }); return { + get supportsDrain() { + return transport?.supportsDrain; + }, + get maxFrameBytes() { + return transport?.maxFrameBytes; + }, send(data) { if (closed || !transport) throw new Error("Host transport is unavailable"); - transport.send(data); + return transport.send(data); }, close, onOpen(handler) { diff --git a/packages/protocol/src/binary-frames/host-route.ts b/packages/protocol/src/binary-frames/host-route.ts index f45cd714f9..643dcb4e98 100644 --- a/packages/protocol/src/binary-frames/host-route.ts +++ b/packages/protocol/src/binary-frames/host-route.ts @@ -3,6 +3,11 @@ import { MAX_WEBSOCKET_MESSAGE_BYTES } from "../websocket-message-limit.js"; export const HOST_ROUTE_OPCODE = 0x30; export const MAX_HOST_ROUTE_ID_BYTES = 256; export const MAX_HOST_ROUTE_HEADER_BYTES = 3 + MAX_HOST_ROUTE_ID_BYTES; +// A logical host channel crosses the 32 MiB route admission boundary. Text can +// expand sixfold when enclosed in JSON; binary adds the bounded route header. +export const MAX_HOST_CHANNEL_FRAME_BYTES = Math.floor( + (32 * 1024 * 1024 - MAX_HOST_ROUTE_HEADER_BYTES) / 6, +); export const MAX_ROUTED_PAYLOAD_BYTES = MAX_WEBSOCKET_MESSAGE_BYTES - MAX_HOST_ROUTE_HEADER_BYTES; export interface HostRouteFrame { routeId: string; diff --git a/packages/protocol/src/chunked-transport.test.ts b/packages/protocol/src/chunked-transport.test.ts new file mode 100644 index 0000000000..3979b48885 --- /dev/null +++ b/packages/protocol/src/chunked-transport.test.ts @@ -0,0 +1,392 @@ +import { describe, expect, it } from "vitest"; +import { ChunkedTransport } from "./chunked-transport.js"; + +describe("chunked transport", () => { + it("delivers a large UTF-8 logical message once, preserving order", async () => { + const delivered: (string | Uint8Array)[] = []; + const receiver = new ChunkedTransport({ + frameBytes: 17, + send: () => {}, + deliver: (v) => delivered.push(v), + }); + receiver.enabled = true; + const frames: (string | Uint8Array)[] = []; + const sender = new ChunkedTransport({ + frameBytes: 17, + send: (v) => { + frames.push(v); + receiver.receive(v); + }, + deliver: () => {}, + }); + sender.enabled = true; + const text = "🌍".repeat(100); + await sender.send(text); + expect(delivered).toEqual([text]); + expect(frames.length).toBeGreaterThan(1); + expect(frames.every((v) => v instanceof Uint8Array && v.byteLength <= 17)).toBe(true); + }); +}); + +it("preserves binary kind and leaves legacy frames unchanged", async () => { + const delivered: (string | Uint8Array)[] = []; + const receiver = new ChunkedTransport({ + frameBytes: 8, + send: () => {}, + deliver: (v) => delivered.push(v), + }); + const frames: (string | Uint8Array)[] = []; + const sender = new ChunkedTransport({ + frameBytes: 8, + send: (v) => { + frames.push(v); + receiver.receive(v); + }, + deliver: () => {}, + }); + const data = new Uint8Array(40).fill(5); + await sender.send(data); + expect(frames).toEqual([data]); + frames.length = 0; + sender.enabled = receiver.enabled = true; + await sender.send(data); + expect(frames).toHaveLength(10); + expect(delivered).toEqual([data, data]); +}); + +it("rejects malformed sequences and drops incomplete transfers on close", () => { + const delivered: unknown[] = []; + const receiver = new ChunkedTransport({ + frameBytes: 8, + send: () => {}, + deliver: (v) => delivered.push(v), + }); + receiver.enabled = true; + receiver.receive(new Uint8Array([0x31, 1, 0, 1, 65])); + expect(() => receiver.receive("interrupted")).toThrow("Interrupted"); + receiver.receive(new Uint8Array([0x31, 1, 0, 2, 66])); + expect(delivered).toEqual([]); +}); + +it("waits for drain before the next chunk and preserves logical ordering", async () => { + const frames: (string | Uint8Array)[] = []; + let drain!: () => void; + const sender = new ChunkedTransport({ + frameBytes: 8, + send: (v) => { + frames.push(v); + if (frames.length === 1) + return new Promise((resolve) => { + drain = resolve; + }); + }, + deliver: () => {}, + }); + sender.enabled = true; + const first = sender.send("abcdefghij"); + const next = new Uint8Array([1, 2, 3, 4]); + const second = sender.send(next); + await Promise.resolve(); + expect(frames).toHaveLength(1); + expect(sender.queuedByteLength).toBe(4); + drain(); + await Promise.all([first, second]); + expect(frames).toHaveLength(4); + expect(frames[3]).toEqual(next); + expect(sender.queuedByteLength).toBe(0); +}); + +it.each([ + [0x31, 2, 0, 3, 65], + [0x31, 1, 2, 3, 65], + [0x31, 1, 0, 4, 65], + [0x31, 1, 0, 2, 65], + [0x31, 1, 0, 1], +])("rejects malformed chunk metadata %j", (...bytes) => { + const receiver = new ChunkedTransport({ + frameBytes: 8, + send: () => {}, + deliver: () => { + throw new Error("Unexpected delivery"); + }, + }); + receiver.enabled = true; + expect(() => receiver.receive(new Uint8Array(bytes))).toThrow(); +}); + +it("reconnect discards partial state and requires negotiation again", () => { + const delivered: unknown[] = []; + const create = () => + new ChunkedTransport({ frameBytes: 8, send: () => {}, deliver: (v) => delivered.push(v) }); + const old = create(); + old.enabled = true; + old.receive(new Uint8Array([0x31, 1, 0, 1, 65])); + old.close(); + old.receive(new Uint8Array([0x31, 1, 0, 2, 66])); + const fresh = create(); + expect(fresh.enabled).toBe(false); + fresh.enabled = true; + fresh.receive(new Uint8Array([0x31, 1, 0, 3, 67])); + expect(delivered).toEqual(["C"]); +}); + +it("bounds logical messages waiting behind a blocked writer", async () => { + let drain!: () => void; + const transport = new ChunkedTransport({ + frameBytes: 32 * 1024 * 1024, + send: () => + new Promise((resolve) => { + drain = resolve; + }), + deliver: () => {}, + }); + const first = transport.send(new Uint8Array(100 * 1024 * 1024)); + const overflow = transport.send("one more byte"); + try { + await expect(overflow).rejects.toThrow("queue"); + } finally { + drain(); + await first; + } +}); + +it("requests drain for chunks while ordinary frames keep the immediate write path", async () => { + const waits: boolean[] = []; + const transport = new ChunkedTransport({ + frameBytes: 8, + send: (_frame, wait) => { + waits.push(wait); + }, + deliver: () => {}, + }); + transport.enabled = true; + await transport.send("small"); + await transport.send("abcdefghij"); + expect(waits).toEqual([false, true, true, true]); +}); + +it("charges queued ASCII messages by their UTF-8 bytes", async () => { + let drain!: () => void; + let blocked = true; + const transport = new ChunkedTransport({ + frameBytes: 32 * 1024 * 1024, + send: () => { + if (blocked) { + blocked = false; + return new Promise((resolve) => { + drain = resolve; + }); + } + }, + deliver: () => {}, + }); + transport.enabled = true; + const first = transport.send(new Uint8Array([1])); + const text = "x".repeat(10 * 1024 * 1024); + const queued = Array.from({ length: 4 }, () => transport.send(text)); + const settled = Promise.allSettled(queued); + expect(transport.queuedByteLength).toBe(40 * 1024 * 1024); + drain(); + await first; + expect((await settled).map((result) => result.status)).toEqual([ + "fulfilled", + "fulfilled", + "fulfilled", + "fulfilled", + ]); +}); + +it("notifies the socket owner when an awaited send overflows the queue", async () => { + let drain!: () => void; + const failures: unknown[] = []; + const transport = new ChunkedTransport({ + frameBytes: 32 * 1024 * 1024, + send: () => + new Promise((resolve) => { + drain = resolve; + }), + deliver: () => {}, + onFailure: (error) => { + failures.push(error); + }, + }); + const first = transport.send(new Uint8Array(100 * 1024 * 1024)); + const overflow = transport.send("extra"); + await expect(overflow).rejects.toThrow("queue"); + drain(); + await first; + expect(failures).toHaveLength(1); +}); + +it("bounds retained reassembly memory for many one-byte chunks", async () => { + const { spawnSync } = await import("node:child_process"); + // Exercise the built package as a Node consumer; CI builds it before protocol tests. + const entry = new URL("../dist/chunked-transport.js", import.meta.url).href; + const result = spawnSync( + process.execPath, + [ + "--expose-gc", + "--input-type=module", + "-e", + ` + import { ChunkedTransport } from ${JSON.stringify(entry)}; + let delivered = 0; + const receiver = new ChunkedTransport({ frameBytes: 8, send() {}, deliver(frame) { delivered = frame.byteLength; } }); + receiver.enabled = true; + global.gc(); + const before = process.memoryUsage().heapUsed; + receiver.receive(new Uint8Array([0x31, 1, 1, 1, 65])); + for (let index = 0; index < 300000; index++) receiver.receive(new Uint8Array([0x31, 1, 1, 0, 65])); + global.gc(); + const growth = process.memoryUsage().heapUsed - before; + receiver.receive(new Uint8Array([0x31, 1, 1, 2, 66])); + console.log(JSON.stringify({ growth, delivered })); + `, + ], + { encoding: "utf8", timeout: 15000 }, + ); + expect(result.status, result.stderr).toBe(0); + const observed = JSON.parse(result.stdout); + expect(observed.delivered).toBe(300002); + expect(observed.growth).toBeLessThan(8 * 1024 * 1024); +}, 20000); + +it.each(["ASCII", "中", "🌍", "\ud800", "\udc00", "\ud800x"])( + "accounts for queued UTF-8 text %j exactly", + async (text) => { + let drain!: () => void; + let blocked = true; + const transport = new ChunkedTransport({ + frameBytes: 1024, + send: () => { + if (blocked) { + blocked = false; + return new Promise((resolve) => { + drain = resolve; + }); + } + }, + deliver: () => {}, + }); + const first = transport.send(new Uint8Array([1])); + const second = transport.send(text); + expect(transport.queuedByteLength).toBe(new TextEncoder().encode(text).byteLength); + drain(); + await Promise.all([first, second]); + }, +); + +it("notifies the owner once when an accepted writer fails", async () => { + const failures: unknown[] = []; + const error = new Error("write failed"); + const transport = new ChunkedTransport({ + frameBytes: 1024, + deliver: () => {}, + onFailure: (failure) => { + failures.push(failure); + }, + send: () => Promise.reject(error), + }); + await expect(transport.send("hello", { waitForDrain: true })).rejects.toThrow("write failed"); + expect(failures).toEqual([error]); + expect(() => transport.send("again")).toThrow("Transport closed"); + expect(failures).toEqual([error]); +}); + +it.each([false, true])( + "a pre-write rejection preserves the sender (chunked=%s)", + async (chunked) => { + const failures: unknown[] = []; + let reject = true; + const transport = new ChunkedTransport({ + frameBytes: 8, + deliver: () => {}, + onFailure: (error) => { + failures.push(error); + }, + send: () => { + if (reject) throw new Error("capacity"); + }, + }); + transport.enabled = chunked; + expect(() => transport.send("abcdefghij")).toThrow("capacity"); + expect(failures).toEqual([]); + reject = false; + await transport.send("sibling"); + }, +); + +it("a queued pre-write rejection does not poison the next logical send", async () => { + let drain!: () => void; + const failures: unknown[] = []; + const delivered: string[] = []; + const transport = new ChunkedTransport({ + frameBytes: 1024, + deliver: () => {}, + onFailure: (error) => { + failures.push(error); + }, + send: (frame) => { + if (frame === "blocked") + return new Promise((resolve) => { + drain = resolve; + }); + if (frame === "reject") throw new Error("capacity"); + delivered.push(frame as string); + }, + }); + const first = transport.send("blocked"); + const rejected = transport.send("reject"); + const sibling = transport.send("sibling"); + const results = Promise.allSettled([first, rejected, sibling]); + drain(); + expect((await results).map((result) => result.status)).toEqual([ + "fulfilled", + "rejected", + "fulfilled", + ]); + expect(delivered).toEqual(["sibling"]); + expect(failures).toEqual([]); +}); + +it.each([false, true])("delivery errors leave framing usable (chunked=%s)", async (chunked) => { + let fail = true; + const delivered: (string | Uint8Array)[] = []; + const transport = new ChunkedTransport({ + frameBytes: 64, + send: () => {}, + deliver: (frame) => { + if (fail) throw new Error("listener failed"); + delivered.push(frame); + }, + }); + transport.enabled = chunked; + const frame = chunked ? new Uint8Array([0x31, 1, 0, 3, 65]) : "A"; + expect(() => transport.receive(frame)).toThrow("listener failed"); + fail = false; + transport.receive(frame); + expect(delivered).toEqual(["A"]); + await transport.send("still connected"); +}); + +it("unpaced transports retain ordinary outbound admission and accept inbound chunks", async () => { + const frames: unknown[] = []; + const delivered: unknown[] = []; + const transport = new ChunkedTransport({ + frameBytes: 8, + supportsDrain: false, + send: (frame) => { + frames.push(frame); + if (typeof frame === "string" && frame.length > 8) throw new Error("single frame limit"); + }, + deliver: (frame) => { + delivered.push(frame); + }, + }); + transport.enabled = true; + expect(() => transport.send("too large for one frame")).toThrow("single frame limit"); + expect(frames).toEqual(["too large for one frame"]); + transport.receive(new Uint8Array([0x31, 1, 0, 3, 65])); + expect(delivered).toEqual(["A"]); + await transport.send("small"); +}); diff --git a/packages/protocol/src/chunked-transport.ts b/packages/protocol/src/chunked-transport.ts new file mode 100644 index 0000000000..a32d68424f --- /dev/null +++ b/packages/protocol/src/chunked-transport.ts @@ -0,0 +1,214 @@ +import { MAX_WEBSOCKET_MESSAGE_BYTES } from "./websocket-message-limit.js"; + +type Frame = string | Uint8Array; + +/** A malformed wire frame, distinct from a consumer delivery failure. */ +export class ChunkedTransportProtocolError extends Error {} +// Reserved outside the application binary opcode space. Version, kind, flags follow. +const OPCODE = 0x31; +const HEADER = 4; + +/** Connection-scoped ordered framing, below serialization and routing, above encryption. */ +export class ChunkedTransport { + enabled = false; + private assembly: Uint8Array | undefined; + private size = 0; + private kind = 0; + private closed = false; + private pending: Promise | undefined; + private queuedBytes = 0; + private activeBytes = 0; + + constructor( + private readonly io: { + frameBytes: number; + supportsDrain?: boolean; + send: (frame: Frame, waitForDrain: boolean) => void | Promise; + deliver: (frame: Frame) => void; + onFailure?: (error: unknown) => void; + }, + ) { + if (io.frameBytes <= HEADER) throw new Error("Invalid transport frame budget"); + } + + /** Bytes waiting behind the active write; physical buffering owns the active frame. */ + get queuedByteLength(): number { + return this.queuedBytes - this.activeBytes; + } + get isWriting(): boolean { + return this.pending !== undefined; + } + + send(frame: Frame, options: { waitForDrain?: boolean; byteLength?: number } = {}): Promise { + const byteLength = options.byteLength ?? frameByteLength(frame); + if (byteLength > MAX_WEBSOCKET_MESSAGE_BYTES) throw new Error("Logical message too large"); + const chunk = + this.enabled && this.io.supportsDrain !== false && byteLength > this.io.frameBytes; + if (this.closed) throw new Error("Transport closed"); + const write = () => { + if (this.closed) throw new Error("Transport closed"); + this.activeBytes = byteLength; + // A synchronous rejection did not admit a frame. Only accepted writes can + // fail the connection; queued admission failures still belong to their sender. + const result = chunk + ? this.sendChunks( + typeof frame === "string" ? new TextEncoder().encode(frame) : frame, + typeof frame === "string", + ) + : this.io.send(frame, options.waitForDrain === true); + if (!result) return; + return result.catch((error) => { + this.fail(error); + throw error; + }); + }; + if (this.queuedBytes + byteLength > MAX_WEBSOCKET_MESSAGE_BYTES) { + const error = new Error("Logical output queue exceeded its byte limit"); + this.fail(error); + const rejected = Promise.reject(error); + void rejected.catch(() => {}); + return rejected; + } + this.queuedBytes += byteLength; + let result: void | Promise; + try { + result = this.pending ? this.pending.then(write) : write(); + } catch (error) { + this.queuedBytes -= byteLength; + this.activeBytes = 0; + throw error; + } + if (!result) { + this.queuedBytes -= byteLength; + this.activeBytes = 0; + return Promise.resolve(); + } + const completion = Promise.resolve(result).finally(() => { + this.queuedBytes -= byteLength; + this.activeBytes = 0; + }); + const tail = completion.then( + () => undefined, + () => undefined, + ); + this.pending = tail; + void tail.then(() => { + if (this.pending === tail) this.pending = undefined; + return undefined; + }); + return completion; + } + + private sendChunks(bytes: Uint8Array, text: boolean): Promise { + const capacity = this.io.frameBytes - HEADER; + const writeChunk = (offset: number) => { + if (this.closed) throw new Error("Transport closed"); + const end = Math.min(bytes.byteLength, offset + capacity); + const wire = new Uint8Array(HEADER + end - offset); + wire.set([ + OPCODE, + 1, + text ? 0 : 1, + (offset === 0 ? 1 : 0) | (end === bytes.byteLength ? 2 : 0), + ]); + wire.set(bytes.subarray(offset, end), HEADER); + return this.io.send(wire, true); + }; + const first = writeChunk(0); + return (async () => { + await first; + for (let offset = capacity; offset < bytes.byteLength; offset += capacity) + await writeChunk(offset); + })(); + } + + receive(frame: Frame): void { + if (this.closed) return; + let message: Frame | undefined; + try { + message = this.assemble(frame); + } catch (error) { + this.close(); + throw new ChunkedTransportProtocolError( + error instanceof Error ? error.message : String(error), + { cause: error }, + ); + } + if (message !== undefined) this.io.deliver(message); + } + + private assemble(frame: Frame): Frame | undefined { + if (typeof frame === "string" || frame[0] !== OPCODE) { + if (this.assembly) throw new Error("Interrupted chunked message"); + return frame; + } + if ( + !this.enabled || + frame.length <= HEADER || + frame[1] !== 1 || + frame[2]! > 1 || + (frame[3]! & ~3) !== 0 + ) + throw new Error("Invalid chunk header"); + const start = (frame[3]! & 1) !== 0; + const end = (frame[3]! & 2) !== 0; + if (start === !!this.assembly) throw new Error("Invalid chunk sequence"); + if (start) { + this.assembly = new Uint8Array(0); + this.kind = frame[2]!; + this.size = 0; + } + if (frame[2] !== this.kind) throw new Error("Chunk kind changed"); + this.size += frame.length - HEADER; + if (this.size > MAX_WEBSOCKET_MESSAGE_BYTES) throw new Error("Logical message too large"); + const payload = frame.subarray(HEADER); + if (this.assembly!.byteLength < this.size) { + const capacity = Math.min( + MAX_WEBSOCKET_MESSAGE_BYTES, + Math.max(this.size, this.assembly!.byteLength * 2, 16384), + ); + const grown = new Uint8Array(capacity); + grown.set(this.assembly!.subarray(0, this.size - payload.byteLength)); + this.assembly = grown; + } + this.assembly!.set(payload, this.size - payload.byteLength); + if (!end) return; + const data = this.assembly!.subarray(0, this.size); + this.assembly = undefined; + this.size = 0; + return this.kind === 0 ? new TextDecoder("utf-8", { fatal: true }).decode(data) : data.slice(); + } + + private fail(error: unknown): void { + if (this.closed) return; + this.close(); + this.io.onFailure?.(error); + } + + close(): void { + this.closed = true; + this.enabled = false; + this.assembly = undefined; + this.size = 0; + } +} + +function frameByteLength(frame: Frame): number { + if (typeof frame !== "string") return frame.byteLength; + let bytes = 0; + for (let index = 0; index < frame.length; index++) { + const code = frame.charCodeAt(index); + if (code <= 0x7f) bytes++; + else if (code <= 0x7ff) bytes += 2; + else if ( + code >= 0xd800 && + code <= 0xdbff && + frame.charCodeAt(index + 1) >= 0xdc00 && + frame.charCodeAt(index + 1) <= 0xdfff + ) { + bytes += 4; + index++; + } else bytes += 3; + } + return bytes; +} diff --git a/packages/protocol/src/client-capabilities.ts b/packages/protocol/src/client-capabilities.ts index a35852cc06..28ec8456cb 100644 --- a/packages/protocol/src/client-capabilities.ts +++ b/packages/protocol/src/client-capabilities.ts @@ -1,4 +1,5 @@ export const CLIENT_CAPS = { + binaryChunkingV1: "binary_chunking_v1", helloRejection: "hello_rejection", // COMPAT(ownedSubscriptions): added in v0.8.0, remove legacy ownership after 2027-03-09. ownedSubscriptions: "owned_subscriptions", diff --git a/packages/protocol/src/messages.ts b/packages/protocol/src/messages.ts index 8d8f852988..dc63d56b75 100644 --- a/packages/protocol/src/messages.ts +++ b/packages/protocol/src/messages.ts @@ -3834,6 +3834,7 @@ export const ServerInfoStatusPayloadSchema = z selectiveAgentTimeline: z.boolean().optional(), explicitEventSubscriptions: z.boolean().optional(), ownedSubscriptions: z.boolean().optional(), + binaryChunkingV1: z.boolean().optional(), workspaceSetupObservation: z.boolean().optional(), workspaceSortTimes: z.boolean().optional(), // COMPAT(canonicalSubmittedPrompts): added in v0.2.6, remove gate after 2027-01-30. diff --git a/packages/server/src/server/connected-hosts/hosts.e2e.test.ts b/packages/server/src/server/connected-hosts/hosts.e2e.test.ts index b6765051f6..4c6d79aced 100644 --- a/packages/server/src/server/connected-hosts/hosts.e2e.test.ts +++ b/packages/server/src/server/connected-hosts/hosts.e2e.test.ts @@ -2091,3 +2091,80 @@ test.each(["removed", "disabled"] as const)( ).toEqual([]); }, ); + +test( + "a 50 MiB timeline crosses a routed host without dropping content or the route", + { timeout: 30000 }, + async () => { + const b = await daemon(); + const a = await daemon(); + const destination = await api(b); + const origin = await api(a); + const workspace = await destination.paseo.workspaces.create({ + source: { kind: "directory", path: b.staticDir }, + }); + const agent = await workspace.agents.create({ + config: { provider: "claude/claude-test", modeId: "full-access" }, + title: "Large history", + }); + // Keep each row below the established item budget; exercise transport size across rows. + const text = "x".repeat(240 * 1024); + const rowCount = 220; + for (let index = 0; index < rowCount; index++) { + await b.daemon.agentManager.appendTimelineItem(agent.id, { type: "user_message", text }); + } + await origin.paseo.hosts.add({ connection: `127.0.0.1:${b.port}` }); + const serverId = (await online(origin.paseo)).hosts[0]!.serverId!; + const remote = await origin.paseo.hosts.client(serverId); + const response = await remote.agents.ref(agent.id).timeline.refetch({ limit: 0 }); + const messages = response.entries.flatMap((row) => + row.item.type === "user_message" ? [row.item.text] : [], + ); + expect(messages).toHaveLength(rowCount); + expect(messages.every((message) => message === text)).toBe(true); + expect( + messages.reduce((bytes, message) => bytes + Buffer.byteLength(message), 0), + ).toBeGreaterThan(50 * 1024 * 1024); + expect((await remote.workspaces.list()).entries).toHaveLength(1); + await remote.dispose(); + }, +); + +test("a pre-write host budget rejection leaves sibling routes connected", async () => { + const { HostTransportBudget } = await import("@getpaseo/client/node"); + const b = await daemon(); + const origin = createHostClient( + `127.0.0.1:${b.port}`, + { clientId: "budget-origin", reconnect: { enabled: false } }, + new HostTransportBudget(1024 * 1024), + ); + cleanup.push(() => origin.close()); + await origin.connect(); + const serverId = origin.getLastServerInfoMessage()!.serverId; + const routeIds = [ + await origin.openHostLink("budget-first"), + await origin.openHostLink("budget-sibling"), + ]; + const peers = routeIds.map( + (id, index) => + new DaemonClient({ + url: "ws://host-link/ws", + clientId: `budget-child-${index}`, + reconnect: { enabled: false }, + transportFactory: createHostChannelTransport(origin, serverId, `budget-${index}`, id), + }), + ); + for (const peer of peers) { + cleanup.push(() => peer.close()); + await peer.connect(); + } + await expect(async () => + origin.sendHostFrame(routeIds[0]!, new Uint8Array(2 * 1024 * 1024)), + ).rejects.toMatchObject({ code: "resource_limit" }); + await peers[1]!.ping({ timeoutMs: 2000 }); + expect([origin.isConnected, ...peers.map((peer) => peer.isConnected)]).toEqual([ + true, + true, + true, + ]); +}, 30000); diff --git a/packages/server/src/server/connected-hosts/index.ts b/packages/server/src/server/connected-hosts/index.ts index 96d6ac35d2..15b1e16b97 100644 --- a/packages/server/src/server/connected-hosts/index.ts +++ b/packages/server/src/server/connected-hosts/index.ts @@ -510,10 +510,13 @@ export class ConnectedHosts { return; } if (!verified) throw new HostError("host_offline", "Host identity has not been verified"); - try { - transport.send(data); - } catch { + const rejectWrite = () => route.close("resource_limit", "Host buffer limit reached or transport unavailable"); + try { + const write = transport.send(data); + if (write) void write.catch(rejectWrite); + } catch { + rejectWrite(); } }, }; diff --git a/packages/server/src/server/connected-hosts/link.ts b/packages/server/src/server/connected-hosts/link.ts index 85fe06ed4e..a18c89aeb0 100644 --- a/packages/server/src/server/connected-hosts/link.ts +++ b/packages/server/src/server/connected-hosts/link.ts @@ -1,7 +1,11 @@ import type { SessionDelivery } from "../session/owned-subscriptions/index.js"; import { EventEmitter } from "node:events"; import { createHash } from "node:crypto"; -import { encodeHostRouteFrame, type HostRouteFrame } from "@getpaseo/protocol/binary-frames/index"; +import { + MAX_HOST_CHANNEL_FRAME_BYTES, + encodeHostRouteFrame, + type HostRouteFrame, +} from "@getpaseo/protocol/binary-frames/index"; import type { DaemonPermission, SessionInboundMessage, @@ -10,6 +14,7 @@ import type { /** A logical socket has no network resources or authority of its own. */ export class VirtualHostSocket extends EventEmitter { + readonly maxFrameBytes = MAX_HOST_CHANNEL_FRAME_BYTES; readyState = 1; constructor( private readonly output: (data: string | Uint8Array) => Promise, diff --git a/packages/server/src/server/daemon-e2e/relay-transport.e2e.test.ts b/packages/server/src/server/daemon-e2e/relay-transport.e2e.test.ts index 32d3a1f36c..f47accaa55 100644 --- a/packages/server/src/server/daemon-e2e/relay-transport.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/relay-transport.e2e.test.ts @@ -9,7 +9,7 @@ import { } from "@getpaseo/protocol/binary-frames/index"; import { getDaemonAuthFailureReason, - type DaemonClient as HostClient, + DaemonClient as HostClient, } from "@getpaseo/client/internal/daemon-client"; import type { SessionOutboundMessage } from "@getpaseo/protocol/messages"; import { describe, expect, test } from "vitest"; @@ -593,72 +593,112 @@ async function waitForCapturedLog( } }, 90000); - test("oversized attachments fail without disconnecting or resubmitting over the relay", async () => { - await startRelay({ useLocalRelay: true }); - const daemon = await createTestPaseoDaemon({ - relayEnabled: true, - relayEndpoint: `127.0.0.1:${relayPort}`, - relayUseTls: false, - relayPublicUseTls: false, - }); - let client: DaemonClient | null = null; - try { - const offerUrl = await getPairingOfferUrl({ - paseoHome: daemon.paseoHome, + test.each([true, false])( + "large messages cross the relay with unpaced upload admission preserved (supportsDrain=%s)", + async (supportsDrain) => { + await startRelay({ useLocalRelay: true }); + const daemon = await createTestPaseoDaemon({ relayEnabled: true, - relayEndpoint: daemon.config.relayEndpoint, + relayEndpoint: `127.0.0.1:${relayPort}`, + relayUseTls: false, + relayPublicUseTls: false, }); - const { serverId, daemonPublicKeyB64 } = decodeOfferFromFragmentUrl(offerUrl); - client = new DaemonClient({ - url: buildRelayWebSocketUrl({ - endpoint: `127.0.0.1:${relayPort}`, - useTls: false, - serverId, - role: "client", - }), - e2ee: { enabled: true, daemonPublicKeyB64 }, - reconnect: { baseDelayMs: 100, maxDelayMs: 100 }, - }); - await client.connect(); - const statuses: string[] = []; - client.subscribeConnectionStatus((state) => statuses.push(state.status)); - for (const imageMiB of [30, 61]) { - // Attachments are base64 in the JSON, then E2EE base64 expands the frame again. - const images = [ - { data: "A".repeat(4 * Math.ceil((imageMiB * 1024 * 1024) / 3)), mimeType: "image/png" }, - ]; + let client: HostClient | null = null; + try { + const offerUrl = await getPairingOfferUrl({ + paseoHome: daemon.paseoHome, + relayEnabled: true, + relayEndpoint: daemon.config.relayEndpoint, + }); + const { serverId, daemonPublicKeyB64 } = decodeOfferFromFragmentUrl(offerUrl); + client = new HostClient({ + clientId: "paced-relay-regression", + webSocketFactory: Object.assign((url: string) => new WebSocket(url), { supportsDrain }), + url: buildRelayWebSocketUrl({ + endpoint: `127.0.0.1:${relayPort}`, + useTls: false, + serverId, + role: "client", + }), + e2ee: { enabled: true, daemonPublicKeyB64 }, + reconnect: { baseDelayMs: 100, maxDelayMs: 100 }, + }); + await client.connect(); + const statuses: string[] = []; + client.subscribeConnectionStatus((state) => statuses.push(state.status)); + for (const imageMiB of [30, 61]) { + const images = [ + { + data: "A".repeat(4 * Math.ceil((imageMiB * 1024 * 1024) / 3)), + mimeType: "image/png", + }, + ]; + const creation = client.createWorkspace({ + source: { kind: "directory", path: daemon.staticDir }, + agent: { + provider: "claude", + cwd: daemon.staticDir, + initialPrompt: "Describe these screens.", + images, + }, + }); + if (supportsDrain) + await withTimeout(creation, 30000, "Chunked workspace creation did not settle"); + else await expect(creation).rejects.toThrow(/too large.*32\.0 MiB/); + } + expect((await client.fetchAgents()).entries).toHaveLength(supportsDrain ? 2 : 0); await expect( - withTimeout( - client.createWorkspace({ - source: { kind: "directory", path: daemon.staticDir }, - agent: { - provider: "claude", - cwd: daemon.staticDir, - initialPrompt: "Describe these screens.", - images, - }, - }), - 10000, - "Workspace creation did not settle", - ), - ).rejects.toThrow(/too large.*32\.0 MiB/); - expect((await client.fetchAgents()).entries).toEqual([]); - } - const agent = await client.createAgent({ provider: "claude", cwd: daemon.staticDir }); - await expect( - client.sendMessage(agent.id, "Describe these screens.", { + client.createWorkspace({ + source: { kind: "directory", path: daemon.staticDir }, + agent: { + provider: "claude", + cwd: daemon.staticDir, + initialPrompt: "Describe these screens.", + images: [{ data: "A".repeat(101 * 1024 * 1024), mimeType: "image/png" }], + }, + }), + ).rejects.toThrow(/too large.*100\.0 MiB/); + expect((await client.fetchAgents()).entries).toHaveLength(supportsDrain ? 2 : 0); + const agent = await client.createAgent({ provider: "claude", cwd: daemon.staticDir }); + const attachment = client.sendMessage(agent.id, "Describe these screens.", { images: [{ data: "A".repeat(40 * 1024 * 1024), mimeType: "image/png" }], - }), - ).rejects.toThrow(/too large.*32\.0 MiB/); - await client.sendMessage(agent.id, "A small follow-up still works."); - expect((await client.fetchAgents()).entries).toHaveLength(1); - expect(statuses).toEqual(["connected"]); - } finally { - await client?.close(); - await daemon.close(); - await stopRelay(); - } - }, 60000); + }); + if (supportsDrain) await attachment; + else await expect(attachment).rejects.toThrow(/too large.*32\.0 MiB/); + await expect( + client.sendMessage(agent.id, "Describe these screens.", { + images: [{ data: "A".repeat(101 * 1024 * 1024), mimeType: "image/png" }], + }), + ).rejects.toThrow(/too large.*100\.0 MiB/); + await client.sendMessage(agent.id, "A small follow-up still works."); + expect((await client.fetchAgents()).entries).toHaveLength(supportsDrain ? 3 : 1); + // Respect the per-item content budget while exceeding the relay frame limit. + const text = "x".repeat(240 * 1024); + for (let row = 0; row < 220; row++) + await daemon.daemon.agentManager.appendTimelineItem(agent.id, { + type: "user_message", + text, + }); + const timeline = await client.fetchAgentTimeline(agent.id, { + projection: "canonical", + limit: 0, + timeout: 30000, + }); + const delivered = timeline.entries.filter( + (row) => row.item.type === "user_message" && row.item.text === text, + ); + expect(delivered).toHaveLength(220); + expect(delivered.length * Buffer.byteLength(text)).toBeGreaterThan(50 * 1024 * 1024); + await client.ping(); + expect(statuses).toEqual(["connected"]); + } finally { + await client?.close(); + await daemon.close(); + await stopRelay(); + } + }, + 120000, + ); test("daemon connects to relay and client ping/pong works through relay", async () => { process.env.PASEO_PRIMARY_LAN_IP = "192.168.1.12"; diff --git a/packages/server/src/server/daemon-e2e/terminal.e2e.test.ts b/packages/server/src/server/daemon-e2e/terminal.e2e.test.ts index 633bd3bc1c..3c131ecab3 100644 --- a/packages/server/src/server/daemon-e2e/terminal.e2e.test.ts +++ b/packages/server/src/server/daemon-e2e/terminal.e2e.test.ts @@ -1836,3 +1836,58 @@ describe("list terminals across directories", () => { ]); }); }); + +test("a paused terminal consumer catches up through a snapshot instead of retaining every output frame", async () => { + const { existsSync } = await import("node:fs"); + const cwd = tmpCwd(); + tempDirs.push(cwd); + const marker = path.join(cwd, "output-complete"); + const created = await createTerminalInWorkspace(ctx.client, { + cwd, + options: { + command: process.execPath, + args: [ + "-e", + ` + const fs = require('node:fs'); + process.stdin.setRawMode?.(true); + process.stdin.resume(); + process.stdin.once('data', () => { + const chunk = Buffer.alloc(256 * 1024, 120); + let remaining = 128; + const write = () => { + if (remaining-- === 0) { fs.writeFileSync(${JSON.stringify(marker)}, 'done'); return; } + process.stdout.write(chunk, write); + }; + write(); + }); + `, + ], + }, + }); + const ws = await connectRawWebSocket(ctx.daemon.port); + try { + const slot = await subscribeRawTerminal(ws, created.terminal!.id, "slow-terminal"); + await waitForRawBinaryFrame( + ws, + (frame) => frame.slot === slot && frame.opcode === TerminalStreamOpcode.Snapshot, + ); + const capture = startTerminalOutputCapture(ws, slot); + try { + ws.pause(); + sendRawTerminalInput(ws, slot, "go\n"); + await waitForCondition(() => existsSync(marker), 20000); + ws.resume(); + await waitForCondition(() => capture.snapshotCount > 0, 10000); + expect(ws.readyState).toBe(WebSocket.OPEN); + expect(capture.frames.reduce((total, frame) => total + frame.payloadBytes, 0)).toBeLessThan( + 32 * 1024 * 1024, + ); + } finally { + ws.resume(); + capture.stop(); + } + } finally { + await closeWebSocket(ws); + } +}, 40000); diff --git a/packages/server/src/server/websocket-server.file-transfer.e2e.test.ts b/packages/server/src/server/websocket-server.file-transfer.e2e.test.ts index 5131456ee9..abce664ffd 100644 --- a/packages/server/src/server/websocket-server.file-transfer.e2e.test.ts +++ b/packages/server/src/server/websocket-server.file-transfer.e2e.test.ts @@ -90,7 +90,11 @@ test( TEST_TIMEOUT_MS, ); -async function connectSocket(port: number, clientId: string): Promise { +async function connectSocket( + port: number, + clientId: string, + capabilities?: Record, +): Promise { const socket = new WebSocket(`ws://127.0.0.1:${port}/ws`); await new Promise((resolve, reject) => { socket.once("open", resolve); @@ -98,7 +102,7 @@ async function connectSocket(port: number, clientId: string): Promise }); await sendAndWait( socket, - { type: "hello", clientId, clientType: "browser", protocolVersion: 1 }, + { type: "hello", clientId, clientType: "browser", protocolVersion: 1, capabilities }, (message) => message.type === "session" && message.message.type === "status" && @@ -166,3 +170,97 @@ function sendAndWait( socket.send(JSON.stringify(message)); return response; } + +test("an awaited timeline framing overflow closes the actual daemon socket", async () => { + const { createHostClient } = await import("@getpaseo/client/node"); + const { createPaseoApi } = await import("@getpaseo/client"); + const { default: pino } = await import("pino"); + const { Writable } = await import("node:stream"); + const messages: string[] = []; + const logger = pino( + { level: "warn" }, + new Writable({ + write(chunk, _encoding, done) { + messages.push(JSON.parse(chunk.toString()).msg); + done(); + }, + }), + ); + daemon = await createTestPaseoDaemon({ logger }); + const client = createHostClient(`127.0.0.1:${daemon.port}`, { + clientId: "framing-seed", + reconnect: { enabled: false }, + }); + await client.connect(); + const api = createPaseoApi(client); + try { + const workspace = await api.workspaces.create({ + source: { kind: "directory", path: daemon.staticDir }, + }); + const agents = []; + const text = "x".repeat(240 * 1024); + for (const [index, rows] of [350, 90].entries()) { + const agent = await workspace.agents.create({ + config: { provider: "claude/claude-test", modeId: "full-access" }, + title: `Framing overflow ${index}`, + }); + agents.push(agent.id); + for (let row = 0; row < rows; row++) + await daemon.daemon.agentManager.appendTimelineItem(agent.id, { + type: "user_message", + text, + }); + } + const source = await connectSocket(daemon.port, "framing-overflow", { + binary_chunking_v1: true, + }); + sockets.push(source); + const closed = new Promise((resolve) => source.once("close", () => resolve())); + const firstChunk = new Promise((resolve) => { + const listener = (data: RawData, binary: boolean) => { + if (!binary || (data as Buffer)[0] !== 0x31) return; + source.off("message", listener); + source.pause(); + resolve(); + }; + source.on("message", listener); + }); + const fetch = (agentId: string, requestId: string) => + source.send( + JSON.stringify({ + type: "session", + message: { + type: "fetch_agent_timeline_request", + requestId, + agentId, + direction: "tail", + projection: "canonical", + limit: 0, + }, + }), + ); + fetch(agents[0]!, "first"); + await firstChunk; + fetch(agents[1]!, "overflow"); + await expect + .poll(() => messages.includes("Closing physical WebSocket after outbound framing failure"), { + timeout: 15000, + }) + .toBe(true); + expect(messages).not.toContain("Closing physical WebSocket at outbound high-water mark"); + source.resume(); + await closed; + } finally { + await api.dispose(); + await client.close(); + } +}, 60000); + +test("malformed chunk metadata closes the daemon socket with a protocol error", async () => { + daemon = await createTestPaseoDaemon(); + const source = await connectSocket(daemon.port, "invalid-framing", { binary_chunking_v1: true }); + sockets.push(source); + const closed = new Promise((resolve) => source.once("close", (code) => resolve(code))); + source.send(new Uint8Array([0x31, 2, 0, 3, 65])); + expect(await closed).toBe(1002); +}); diff --git a/packages/server/src/server/websocket-server.ts b/packages/server/src/server/websocket-server.ts index 96c81ab49a..7bfc4d5a0f 100644 --- a/packages/server/src/server/websocket-server.ts +++ b/packages/server/src/server/websocket-server.ts @@ -1,3 +1,7 @@ +import { + ChunkedTransport, + ChunkedTransportProtocolError, +} from "@getpaseo/protocol/chunked-transport"; import { type WorkspaceRuntimeStore } from "./workspace-runtime-store/index.js"; import { createAgentMessageDelivery, @@ -124,8 +128,8 @@ import { MAX_PHYSICAL_SOCKET_BUFFERED_BYTES, outboundFrameByteLength, physicalSocketHasCapacity, - sendBoundedPhysicalFrame, sendBoundedPhysicalFrameAndWait, + sendBoundedPhysicalFrame, } from "./websocket/physical-socket.js"; const WS_CLOSE_DAEMON_AUTH_FAILED = 4401; @@ -440,6 +444,7 @@ function bufferFromWsData(data: Buffer | ArrayBuffer | Buffer[] | string): Buffe } export interface WebSocketLike { + maxFrameBytes?: number; readyState: number; bufferedAmount?: number; send: ( @@ -1273,6 +1278,67 @@ export class VoiceAssistantWebSocketServer { return { sockets: writableSockets, payload, bytes: outboundFrameByteLength(payload) }; } + private sendHelloResponse(ws: WebSocketLike, session: Session, hello: WSHelloMessage): void { + // The peer learns our support from this ordinary response before receiving chunks. + this.sendToClient(ws, this.createServerInfoMessage(session)); + this.chunkedSocket(ws).enabled = hello.capabilities?.[CLIENT_CAPS.binaryChunkingV1] === true; + } + + private readonly chunkedSockets = new WeakMap(); + + private chunkedSocket(ws: WebSocketLike): ChunkedTransport { + let transport = this.chunkedSockets.get(ws); + if (!transport) { + transport = new ChunkedTransport({ + frameBytes: ws.maxFrameBytes ?? 32 * 1024 * 1024, + onFailure: (err) => + this.closePhysicalSocket({ + ws, + logMessage: "Closing physical WebSocket after outbound framing failure", + logFields: { err }, + }), + send: (frame, waitForDrain) => { + const params = { + socket: ws, + frame, + onHighWater: () => this.closeAtOutboundHighWater(ws), + }; + if (waitForDrain) + return sendBoundedPhysicalFrameAndWait(params).then((sent) => { + if (!sent) throw new Error("Source is closed"); + return undefined; + }); + if (!sendBoundedPhysicalFrame(params)) throw new Error("Source is closed"); + }, + deliver: (frame) => + this.handleRawMessage(ws, typeof frame === "string" ? frame : Buffer.from(frame)), + }); + this.chunkedSockets.set(ws, transport); + } + return transport; + } + + private socketBufferedAmount(ws: WebSocketLike): number | null { + const queued = this.chunkedSockets.get(ws)?.queuedByteLength ?? 0; + if (typeof ws.bufferedAmount === "number") return ws.bufferedAmount + queued; + return queued > 0 ? queued : null; + } + + private sendSocketFrame( + ws: WebSocketLike, + frame: string | Uint8Array, + bytes: number, + waitForDrain = false, + ): Promise { + const transport = this.chunkedSocket(ws); + const admittedBytes = transport.isWriting + ? bytes + : Math.min(bytes, ws.maxFrameBytes ?? 32 * 1024 * 1024); + if (!this.ensureOutboundCapacity(ws, admittedBytes)) + return Promise.reject(new Error("Source is closed")); + return transport.send(frame, { waitForDrain, byteLength: bytes }); + } + private async sendMessageToClientAndWait( ws: WebSocketLike, message: WSOutboundMessage, @@ -1280,13 +1346,7 @@ export class VoiceAssistantWebSocketServer { const prepared = this.prepareJsonDelivery([ws], message, true); if (!prepared) throw new Error("Source message delivery unavailable"); try { - const sent = await sendBoundedPhysicalFrameAndWait({ - socket: ws, - frame: prepared.payload, - frameBytes: prepared.bytes, - onHighWater: () => this.closeAtOutboundHighWater(ws), - }); - if (!sent) throw new Error("Source is closed"); + await this.sendSocketFrame(ws, prepared.payload, prepared.bytes, true); this.runtimeMetrics.recordOutboundMessage(message, ws.bufferedAmount); } catch (err) { this.logger.warn({ err }, "ws_send_failed"); @@ -1313,14 +1373,7 @@ export class VoiceAssistantWebSocketServer { return; } try { - const sent = await sendBoundedPhysicalFrameAndWait({ - socket: ws, - frame, - onHighWater: () => this.closeAtOutboundHighWater(ws), - }); - if (!sent) { - throw new Error("Physical WebSocket is not open"); - } + await this.sendSocketFrame(ws, frame, frame.byteLength, true); this.runtimeMetrics.recordOutboundBinaryFrame(ws.bufferedAmount); } catch (err) { this.logger.warn({ err }, "ws_send_failed"); @@ -1335,13 +1388,9 @@ export class VoiceAssistantWebSocketServer { recordSent: () => void, ): void { try { - const sent = sendBoundedPhysicalFrame({ - socket: ws, - frame, - frameBytes, - onHighWater: () => this.closeAtOutboundHighWater(ws), + void this.sendSocketFrame(ws, frame, frameBytes).then(recordSent, (err) => { + this.logger.warn({ err }, "ws_send_failed"); }); - if (sent) recordSent(); } catch (err) { this.logger.warn({ err }, "ws_send_failed"); } @@ -1349,7 +1398,13 @@ export class VoiceAssistantWebSocketServer { private ensureOutboundCapacity(ws: WebSocketLike, frameBytes: number): boolean { if (ws.readyState !== 1) return false; - if (physicalSocketHasCapacity(ws, frameBytes)) return true; + if ( + physicalSocketHasCapacity( + { bufferedAmount: this.socketBufferedAmount(ws) ?? undefined }, + frameBytes, + ) + ) + return true; this.closeAtOutboundHighWater(ws); return false; @@ -1522,16 +1577,15 @@ export class VoiceAssistantWebSocketServer { if (!connection) { return null; } - if (source) return (source as WebSocketLike).bufferedAmount ?? null; + if (source) return this.socketBufferedAmount(source as WebSocketLike); // Relay-attached sockets are a WebSocketLike that doesn't expose // bufferedAmount. Return null when no socket gives a signal so the // terminal fallback can't mistake "no signal" for "client keeping up"; // a direct ws reports its real buffered bytes (0 when drained). let maxBuffered: number | null = null; for (const socket of connection.sockets) { - if (typeof socket.bufferedAmount === "number") { - maxBuffered = Math.max(maxBuffered ?? 0, socket.bufferedAmount); - } + const buffered = this.socketBufferedAmount(socket); + if (buffered !== null) maxBuffered = Math.max(maxBuffered ?? 0, buffered); } return maxBuffered; }, @@ -1568,8 +1622,9 @@ export class VoiceAssistantWebSocketServer { let total = 0; for (const socket of this.sessions.keys()) { if (socket instanceof VirtualHostSocket) continue; - if (typeof socket.bufferedAmount !== "number") return null; - total += socket.bufferedAmount; + const buffered = this.socketBufferedAmount(socket); + if (buffered === null) return null; + total += buffered; } return total; }, @@ -1770,7 +1825,7 @@ export class VoiceAssistantWebSocketServer { this.externalSessionsByKey.set(sessionKey, connection); } pending.identity.sessionId = connection.session.getSessionId(); - this.sendToClient(ws, this.createServerInfoMessage(connection.session)); + this.sendHelloResponse(ws, connection.session, message); connection.connectionLogger.info( { ...toConnectionLogFields(pending.identity), @@ -1906,7 +1961,7 @@ export class VoiceAssistantWebSocketServer { existing.sockets.add(ws); this.sessions.set(ws, existing); pending.identity.sessionId = existing.session.getSessionId(); - this.sendToClient(ws, this.createServerInfoMessage(existing.session)); + this.sendHelloResponse(ws, existing.session, message); pending.connectionLogger.info( { ...toConnectionLogFields(pending.identity), @@ -1952,6 +2007,7 @@ export class VoiceAssistantWebSocketServer { usageSetting: true, usageSources: true, ownedSubscriptions: true, + binaryChunkingV1: true, agentRequestReceipts: true, workspaceRequestReceipts: true, creationLifecycle: true, @@ -2165,10 +2221,16 @@ export class VoiceAssistantWebSocketServer { private bindSocketHandlers(ws: WebSocketLike): void { ws.on("message", (...args: unknown[]) => { const data = args[0] as Buffer | ArrayBuffer | Buffer[] | string; - this.handleRawMessage(ws, data); + try { + this.chunkedSocket(ws).receive(typeof data === "string" ? data : bufferFromWsData(data)); + } catch (error) { + if (!(error instanceof ChunkedTransportProtocolError)) throw error; + ws.close(1002, "Invalid chunked transport"); + } }); ws.on("close", async (...args: unknown[]) => { + this.chunkedSockets.get(ws)?.close(); const code = args[0]; const reason = args[1]; await this.detachSocket(ws, { @@ -2568,7 +2630,7 @@ export class VoiceAssistantWebSocketServer { identity: JSON.stringify([connection.principalId, connection.clientId]), permissions: () => connection.session.getPermissions(), routed: () => connection.session.isRouted(), - bufferedAmount: () => ws.bufferedAmount ?? null, + bufferedAmount: () => this.socketBufferedAmount(ws), delivery: connection.session.delivery, admit: (socket, permissions) => this.attachSocket(socket, undefined, undefined, false, { diff --git a/packages/server/src/server/websocket/encrypted-relay-socket.test.ts b/packages/server/src/server/websocket/encrypted-relay-socket.test.ts index f2a880f15f..3e17f0cbb4 100644 --- a/packages/server/src/server/websocket/encrypted-relay-socket.test.ts +++ b/packages/server/src/server/websocket/encrypted-relay-socket.test.ts @@ -140,3 +140,74 @@ test("encrypted sockets do not double-count bytes already buffered by the transp expect(socket.bufferedAmount).toBe(payload.byteLength + 40); }); + +test("50 MiB logical text and routed binary cross encrypted relay frames in both directions", async () => { + const { ChunkedTransport } = await import("@getpaseo/protocol/chunked-transport"); + const { openClientSecureChannel, openDaemonSecureChannel } = await import("@getpaseo/relay/e2ee"); + const { generateKeyPair, exportPublicKey, maxSecuredPlaintextByteLength } = + await import("@getpaseo/relay"); + const { encodeHostRouteFrame, decodeHostRouteFrame } = + await import("@getpaseo/protocol/binary-frames/index"); + type Transport = import("@getpaseo/relay/e2ee").Transport; + const limit = 32 * 1024 * 1024; + const wireSizes: number[] = []; + const endpoint = (): Transport => ({ + send: () => {}, + close: () => {}, + onmessage: null, + onclose: null, + onerror: null, + }); + const left = endpoint(); + const right = endpoint(); + const wire = (destination: Transport) => (data: string | ArrayBuffer) => { + const size = typeof data === "string" ? Buffer.byteLength(data) : data.byteLength; + expect(size).toBeLessThanOrEqual(limit); + wireSizes.push(size); + queueMicrotask(() => destination.onmessage?.({ data, isBinary: typeof data !== "string" })); + }; + left.send = wire(right); + right.send = wire(left); + const key = generateKeyPair(); + const delivered: (string | Uint8Array)[][] = [[], []]; + let receivers: InstanceType[] = []; + const incoming = (index: number) => (data: string | ArrayBuffer) => + receivers[index]!.receive(typeof data === "string" ? data : new Uint8Array(data)); + const daemonPromise = openDaemonSecureChannel(right, { + daemonKeyPair: key, + events: { onmessage: incoming(1) }, + }); + const client = await openClientSecureChannel(left, { + daemonPublicKeyB64: exportPublicKey(key.publicKey), + events: { onmessage: incoming(0) }, + }); + const daemon = await daemonPromise; + receivers = [client, daemon].map( + (channel, index) => + new ChunkedTransport({ + frameBytes: maxSecuredPlaintextByteLength(limit), + send: (frame) => channel.send(typeof frame === "string" ? frame : frame.slice().buffer), + deliver: (frame) => { + delivered[index]!.push(frame); + }, + }), + ); + for (const receiver of receivers) receiver.enabled = true; + const text = "x".repeat(50 * 1024 * 1024); + wireSizes.length = 0; + await receivers[1]!.send(text); + await new Promise((resolve) => setImmediate(resolve)); + expect(delivered[0]).toEqual([text]); + expect(wireSizes.length).toBeGreaterThan(1); + const routed = encodeHostRouteFrame("route", new Uint8Array(50 * 1024 * 1024).fill(7)); + await receivers[0]!.send(routed); + await new Promise((resolve) => setImmediate(resolve)); + expect(delivered[1]).toHaveLength(1); + const received = decodeHostRouteFrame(delivered[1]![0] as Uint8Array)!; + expect(received.routeId).toBe("route"); + expect( + Buffer.from(received.payload).equals(Buffer.from(new Uint8Array(50 * 1024 * 1024).fill(7))), + ).toBe(true); + client.close(); + daemon.close(); +}, 30_000); diff --git a/packages/server/src/server/websocket/encrypted-relay-socket.ts b/packages/server/src/server/websocket/encrypted-relay-socket.ts index f11162761c..794789718c 100644 --- a/packages/server/src/server/websocket/encrypted-relay-socket.ts +++ b/packages/server/src/server/websocket/encrypted-relay-socket.ts @@ -1,3 +1,5 @@ +import { maxSecuredPlaintextByteLength } from "@getpaseo/relay"; +import { MAX_RELAY_WEBSOCKET_MESSAGE_BYTES } from "@getpaseo/protocol/websocket-message-limit"; import { EventEmitter } from "node:events"; import { MAX_PHYSICAL_SOCKET_BUFFERED_BYTES } from "./physical-socket.js"; @@ -8,6 +10,7 @@ export interface EncryptedRelayChannel { } export interface EncryptedRelaySocket { + readonly maxFrameBytes: number; readonly readyState: number; readonly bufferedAmount: number; send: (data: string | Uint8Array | ArrayBuffer) => void | Promise; @@ -43,6 +46,7 @@ export function createEncryptedRelaySocket(params: { }); return { + maxFrameBytes: maxSecuredPlaintextByteLength(MAX_RELAY_WEBSOCKET_MESSAGE_BYTES), get readyState() { return readyState; }, diff --git a/packages/server/src/server/websocket/physical-socket.test.ts b/packages/server/src/server/websocket/physical-socket.test.ts index c81b1410d6..6dce3e151d 100644 --- a/packages/server/src/server/websocket/physical-socket.test.ts +++ b/packages/server/src/server/websocket/physical-socket.test.ts @@ -120,3 +120,96 @@ test("the awaitable physical send rejects callback errors", async () => { }), ).rejects.toThrow("send failed"); }); + +test("ordinary frames expose a paused real socket's backlog and terminate at the physical backstop", async () => { + const { once } = await import("node:events"); + const { WebSocket, WebSocketServer } = await import("ws"); + const { ChunkedTransport } = await import("@getpaseo/protocol/chunked-transport"); + const server = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + await once(server, "listening"); + const connected = once(server, "connection"); + const address = server.address(); + if (typeof address === "string" || !address) throw new Error("Missing server address"); + const client = new WebSocket(`ws://127.0.0.1:${address.port}`); + await once(client, "open"); + const [socket] = (await connected) as [InstanceType]; + const tcp = (socket as unknown as { _socket: import("node:net").Socket })._socket; + let terminated = false; + const transport = new ChunkedTransport({ + frameBytes: 32 * 1024 * 1024, + deliver: () => {}, + send: (frame, wait) => { + const options = { + socket, + frame, + onHighWater: () => { + terminated = true; + socket.terminate(); + }, + }; + if (wait) return sendBoundedPhysicalFrameAndWait(options).then(() => undefined); + sendBoundedPhysicalFrame(options); + }, + }); + try { + client.pause(); + tcp.cork(); + const frame = new Uint8Array(256 * 1024); + for (let count = 0; count < 200; count++) await transport.send(frame); + expect(socket.bufferedAmount).toBeGreaterThan(4 * 1024 * 1024); + expect(transport.queuedByteLength).toBe(0); + for (let count = 0; count < 100; count++) { + if (terminated) break; + await transport.send(frame); + } + expect(terminated).toBe(true); + } finally { + transport.close(); + client.terminate(); + socket.terminate(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); + +test("an awaited framing failure terminates its real socket immediately", async () => { + const { once } = await import("node:events"); + const { WebSocket, WebSocketServer } = await import("ws"); + const { ChunkedTransport } = await import("@getpaseo/protocol/chunked-transport"); + const server = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + await once(server, "listening"); + const connected = once(server, "connection"); + const address = server.address(); + if (typeof address === "string" || !address) throw new Error("Missing server address"); + const client = new WebSocket(`ws://127.0.0.1:${address.port}`); + await once(client, "open"); + const [socket] = (await connected) as [InstanceType]; + const tcp = (socket as unknown as { _socket: import("node:net").Socket })._socket; + const transport = new ChunkedTransport({ + frameBytes: 32 * 1024 * 1024, + deliver: () => {}, + onFailure: () => socket.terminate(), + send: (frame) => + sendBoundedPhysicalFrameAndWait({ + socket, + frame, + onHighWater: () => socket.terminate(), + }).then(() => undefined), + }); + transport.enabled = true; + try { + client.pause(); + tcp.cork(); + const closed = once(socket, "close"); + const first = transport.send(new Uint8Array(100 * 1024 * 1024), { waitForDrain: true }); + const overflow = transport.send(new Uint8Array([1]), { waitForDrain: true }); + const settled = Promise.allSettled([first, overflow]); + await closed; + expect(socket.readyState).toBe(WebSocket.CLOSED); + expect((await settled).map((result) => result.status)).toEqual(["rejected", "rejected"]); + } finally { + transport.close(); + client.terminate(); + socket.terminate(); + await new Promise((resolve) => server.close(() => resolve())); + } +});