mirror of
https://github.com/getpaseo/paseo.git
synced 2026-10-30 21:23:04 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
41d9bb4294 | ||
|
|
b044201f62 | ||
|
|
cacf186a94 | ||
|
|
47b18ffb50 | ||
|
|
ab55377458 | ||
|
|
f3d8718c6a | ||
|
|
26db1a4672 | ||
|
|
f28f0fee3d | ||
|
|
98f667f93e | ||
|
|
8c2a2a9a88 |
@@ -52,6 +52,33 @@ On capable daemons, connecting creates no timeline or event demand. Client subsc
|
||||
release it on unsubscribe, and restore it after reconnect. Raw message observers inspect traffic
|
||||
without requesting streams. Application caches and which agents are visible remain caller-owned.
|
||||
|
||||
## Binary chunking
|
||||
|
||||
`binary_chunking_v1` in hello and optional `server_info.features.binaryChunkingV1`
|
||||
negotiate connection-scoped framing. Application serializers and routed envelopes use
|
||||
the same boundary. Small messages and handshakes keep their existing representation.
|
||||
Older peers retain existing frame limits.
|
||||
|
||||
The reserved binary opcode `0x31` is followed by version `1`, original kind
|
||||
(`0` text, `1` binary), and START/END flag bits `0`/`1`. Remaining bytes are an
|
||||
unchanged slice. There is no total length or chunk count; a future receiver can
|
||||
consume slices incrementally. Today the receiver buffers up to the existing
|
||||
100 MiB logical limit and decodes UTF-8 only after END. Transfers cannot interleave.
|
||||
Disconnect clears partial state and negotiation.
|
||||
|
||||
The lower transport supplies its plaintext frame budget, including encryption
|
||||
overhead. The sender waits for drain between chunks while ordinary messages keep
|
||||
the existing delivery path. Slow-client protections count messages queued behind
|
||||
an active transfer as well as bytes already in the socket.
|
||||
The framing module never selects an encryption construction.
|
||||
|
||||
Native uploads retain the platform's existing queue limits. React Native's
|
||||
WebSocket does not report its queued bytes, so the client cannot wait for native
|
||||
drain. Android's installed OkHttp transport closes when a send would put more
|
||||
than 16 MiB in its queue. The native transport disables outbound chunking, including
|
||||
on routed channels, and retains its existing single-frame admission errors.
|
||||
Inbound chunking remains enabled; daemon-to-native responses use the daemon's paced writes.
|
||||
|
||||
## Owned observations
|
||||
|
||||
`owned_subscriptions` and `server_info.features.ownedSubscriptions` negotiate the source-owned
|
||||
|
||||
@@ -18,7 +18,7 @@ function visibleMainPane(page: Page) {
|
||||
}
|
||||
|
||||
function composerChangesPill(page: Page) {
|
||||
return page.getByTestId("composer-diff-stat-pill");
|
||||
return page.getByRole("button", { name: "Open Changes tab", exact: true });
|
||||
}
|
||||
|
||||
async function revealComposerChangesInExplorer(page: Page) {
|
||||
@@ -498,6 +498,7 @@ test.describe("send review feedback", () => {
|
||||
test("only open tabs in this workspace appear in compact and wide recipient menus", async ({
|
||||
page,
|
||||
}) => {
|
||||
test.setTimeout(120000);
|
||||
const workspace = await seedChangedAgent("review-feedback-multiple-");
|
||||
const other = await seedChangedAgent("review-feedback-other-");
|
||||
try {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { chooseModel, startWithoutRememberedModel } from "../support/helpers/model-memory";
|
||||
import {
|
||||
composerLocator,
|
||||
enterComposerFullscreen,
|
||||
@@ -97,6 +98,7 @@ test.describe("New workspace composer draft", () => {
|
||||
const project = await seedWorkspace({ repoPrefix: "new-workspace-submitted-draft-" });
|
||||
|
||||
try {
|
||||
await startWithoutRememberedModel(page);
|
||||
await gotoAppShell(page);
|
||||
await waitForSidebarHydration(page);
|
||||
await openNewWorkspaceComposer(page, {
|
||||
@@ -104,6 +106,7 @@ test.describe("New workspace composer draft", () => {
|
||||
projectDisplayName: project.projectDisplayName,
|
||||
});
|
||||
|
||||
await chooseModel(page, "mock", "Ten second stream");
|
||||
const composer = page.getByRole("textbox", { name: "Message agent..." });
|
||||
const createButton = page.getByTestId("workspace-create-submit");
|
||||
await expect(composer).toBeEditable({ timeout: 30_000 });
|
||||
|
||||
@@ -76,6 +76,25 @@ describe("test-daemon-connection connectToDaemon", () => {
|
||||
probe = new FakeDaemonProbe();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ id: "direct:lan:6767", type: "directTcp" as const, endpoint: "lan:6767" },
|
||||
{
|
||||
id: "relay:test",
|
||||
type: "relay" as const,
|
||||
relayEndpoint: "relay.paseo.sh:443",
|
||||
daemonPublicKeyB64: "pubkey",
|
||||
},
|
||||
])("adoptable $type probe clients use the app WebSocket factory", async (connection) => {
|
||||
const { connectToDaemon } = await import("./test-daemon-connection");
|
||||
const { createAppWebSocketFactory } = await import("@/runtime/websocket-factory");
|
||||
const result = await connectToDaemon(connection, { serverId: "srv_probe_test" }, probe.deps);
|
||||
try {
|
||||
expect(result.client.config.webSocketFactory).toBe(createAppWebSocketFactory());
|
||||
} finally {
|
||||
await result.client.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("reuses the app clientId for direct connections", async () => {
|
||||
const { connectToDaemon } = await import("./test-daemon-connection");
|
||||
const first = await connectToDaemon(
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
} from "@getpaseo/client/internal/daemon-client";
|
||||
import type { DaemonClientConfig } from "@getpaseo/client/internal/daemon-client";
|
||||
import type { HostConnection } from "@/types/host-connection";
|
||||
import { createAppWebSocketFactory } from "@/runtime/websocket-factory";
|
||||
import { getOrCreateClientId } from "./client-id";
|
||||
import { resolveAppVersion } from "./app-version";
|
||||
import {
|
||||
@@ -156,6 +157,7 @@ export async function buildClientConfig(
|
||||
clientType: "mobile" as const,
|
||||
appVersion: deps.resolveAppVersion() ?? undefined,
|
||||
suppressSendErrors: true,
|
||||
webSocketFactory: createAppWebSocketFactory(),
|
||||
reconnect: { enabled: false },
|
||||
...resolveConnectionCredentials(connection, options),
|
||||
...(options.capabilities ? { capabilities: options.capabilities } : {}),
|
||||
|
||||
@@ -125,6 +125,7 @@ export class ConnectionSubscriptions extends OwnedSubscriptions {
|
||||
// Protocol support belongs to the installed client. Only browser hosting needs
|
||||
// a resource supplied by the caller. Keep this exhaustive as the protocol evolves.
|
||||
export const DEFAULT_CLIENT_CAPABILITIES = {
|
||||
[CLIENT_CAPS.binaryChunkingV1]: true,
|
||||
[CLIENT_CAPS.helloRejection]: true,
|
||||
[CLIENT_CAPS.ownedSubscriptions]: true,
|
||||
[CLIENT_CAPS.allProviders]: true,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { maxSecuredPlaintextByteLength } from "@getpaseo/relay";
|
||||
import { MAX_RELAY_WEBSOCKET_MESSAGE_BYTES } from "@getpaseo/protocol/websocket-message-limit";
|
||||
import {
|
||||
openClientSecureChannel,
|
||||
@@ -155,18 +156,15 @@ export function createEncryptedTransport(
|
||||
const relayTransport: RelayTransport = {
|
||||
send: (data) => {
|
||||
if (typeof data === "string") {
|
||||
base.send(data);
|
||||
return;
|
||||
return base.send(data);
|
||||
}
|
||||
if (ArrayBuffer.isView(data)) {
|
||||
base.send(normalizeTransportPayload(data));
|
||||
return;
|
||||
return base.send(normalizeTransportPayload(data));
|
||||
}
|
||||
if (data instanceof ArrayBuffer) {
|
||||
base.send(data);
|
||||
return;
|
||||
return base.send(data);
|
||||
}
|
||||
base.send(String(data));
|
||||
return base.send(String(data));
|
||||
},
|
||||
// The channel decides the close code. Report it now: a socket close event
|
||||
// arrives asynchronously, and the consumer must not act on an earlier
|
||||
@@ -226,6 +224,8 @@ export function createEncryptedTransport(
|
||||
});
|
||||
|
||||
return {
|
||||
supportsDrain: base.supportsDrain,
|
||||
maxFrameBytes: maxSecuredPlaintextByteLength(MAX_RELAY_WEBSOCKET_MESSAGE_BYTES),
|
||||
send: (data) => {
|
||||
if (!channel) {
|
||||
throw new Error("Encrypted channel not ready");
|
||||
@@ -239,8 +239,9 @@ export function createEncryptedTransport(
|
||||
// Encryption and its transport handoff are synchronous up to send's first await.
|
||||
// Reject capacity failures here so a routed sender loses only its own channel.
|
||||
assertOutboundCapacity(wireBytes);
|
||||
void channel.send(payload).catch((error) => {
|
||||
return channel.send(payload).catch((error) => {
|
||||
emitError(error);
|
||||
throw error;
|
||||
});
|
||||
},
|
||||
close: (code?: number, reason?: string) => {
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
export interface DaemonTransport {
|
||||
send: (data: string | Uint8Array | ArrayBuffer) => void;
|
||||
/** False when the underlying socket cannot report write drain. */
|
||||
supportsDrain?: boolean;
|
||||
maxFrameBytes?: number;
|
||||
send: (data: string | Uint8Array | ArrayBuffer) => void | Promise<void>;
|
||||
close: (code?: number, reason?: string) => void;
|
||||
onMessage: (handler: (data: unknown, isBinary: boolean) => void) => () => void;
|
||||
onOpen: (handler: () => void) => () => void;
|
||||
@@ -13,12 +16,16 @@ export type DaemonTransportFactory = (options: {
|
||||
protocols?: string[];
|
||||
}) => DaemonTransport;
|
||||
|
||||
export type WebSocketFactory = (
|
||||
url: string,
|
||||
options?: { headers?: Record<string, string>; protocols?: string[] },
|
||||
) => WebSocketLike;
|
||||
export interface WebSocketFactory {
|
||||
(
|
||||
url: string,
|
||||
options?: { headers?: Record<string, string>; protocols?: string[] },
|
||||
): WebSocketLike;
|
||||
supportsDrain?: boolean;
|
||||
}
|
||||
|
||||
export interface WebSocketLike {
|
||||
bufferedAmount?: number;
|
||||
readyState: number;
|
||||
send: (data: string | Uint8Array | ArrayBuffer) => void;
|
||||
close: (code?: number, reason?: string) => void;
|
||||
|
||||
@@ -175,3 +175,18 @@ describe("daemon-client transport helpers", () => {
|
||||
expect(decodeMessageData(bytes)).toBe("hello");
|
||||
});
|
||||
});
|
||||
|
||||
test("native drain capability survives WebSocket and encryption wrappers", async () => {
|
||||
const { nativeWebSocketFactory } = await import("./daemon-client-websocket-transport.js");
|
||||
expect(nativeWebSocketFactory.supportsDrain).toBe(false);
|
||||
const factory = Object.assign(() => ({ readyState: 1, send() {}, close() {} }), {
|
||||
supportsDrain: false,
|
||||
});
|
||||
const base = createWebSocketTransportFactory(factory)({ url: "ws://test" });
|
||||
expect(base.supportsDrain).toBe(false);
|
||||
const encrypted = createEncryptedTransport(base, {
|
||||
daemonPublicKeyB64: "unused-key",
|
||||
logger: { warn() {} },
|
||||
});
|
||||
expect(encrypted.supportsDrain).toBe(false);
|
||||
});
|
||||
|
||||
@@ -39,6 +39,9 @@ export function nativeWebSocketFactory(
|
||||
return new (getGlobalWebSocket())(url, options?.protocols, { headers: options?.headers });
|
||||
}
|
||||
|
||||
// React Native leaves bufferedAmount at zero, including while native output is queued.
|
||||
nativeWebSocketFactory.supportsDrain = false;
|
||||
|
||||
export function createWebSocketTransportFactory(factory: WebSocketFactory): DaemonTransportFactory {
|
||||
return ({ url, headers, protocols }) => {
|
||||
const ws = factory(url, { headers, protocols });
|
||||
@@ -50,11 +53,28 @@ export function createWebSocketTransportFactory(factory: WebSocketFactory): Daem
|
||||
}
|
||||
}
|
||||
return {
|
||||
supportsDrain: factory.supportsDrain !== false,
|
||||
send: (data) => {
|
||||
if (typeof ws.readyState === "number" && ws.readyState !== 1) {
|
||||
throw new Error(`WebSocket not open (readyState=${ws.readyState})`);
|
||||
}
|
||||
ws.send(data);
|
||||
if ((ws.bufferedAmount ?? 0) > 1024 * 1024) {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const drain = () => {
|
||||
if (ws.readyState !== 1) {
|
||||
reject(new Error("WebSocket closed while draining"));
|
||||
return;
|
||||
}
|
||||
if ((ws.bufferedAmount ?? 0) <= 1024 * 1024) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
setTimeout(drain, 10);
|
||||
};
|
||||
drain();
|
||||
});
|
||||
}
|
||||
},
|
||||
close: (code?: number, reason?: string) => {
|
||||
// Node's `ws` may emit an `error` when a connecting socket is closed before the
|
||||
|
||||
@@ -1094,6 +1094,7 @@ test("advertises client capabilities in hello", async () => {
|
||||
provider_snapshot_references: true,
|
||||
explicit_event_subscriptions: true,
|
||||
owned_subscriptions: true,
|
||||
binary_chunking_v1: true,
|
||||
compact_provider_snapshots: true,
|
||||
custom_mode_icons: true,
|
||||
project_updates: true,
|
||||
@@ -6817,91 +6818,102 @@ test("drops legacy agent_stream tool_call payloads and logs validation warning",
|
||||
expect(logger.warn).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("parses canonical fetch_agent_timeline_response payloads without crashing", async () => {
|
||||
const logger = createMockLogger();
|
||||
const mock = createMockTransport();
|
||||
test.each([false, true])(
|
||||
"parses canonical timeline responses with chunking=%s",
|
||||
async (chunked) => {
|
||||
const logger = createMockLogger();
|
||||
const mock = createMockTransport();
|
||||
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "clsk_unit_test",
|
||||
logger,
|
||||
reconnect: { enabled: false },
|
||||
transportFactory: () => mock.transport,
|
||||
});
|
||||
clients.push(client);
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "clsk_unit_test",
|
||||
logger,
|
||||
reconnect: { enabled: false },
|
||||
transportFactory: () => mock.transport,
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
const connectPromise = client.connect();
|
||||
mock.triggerOpen();
|
||||
await connectPromise;
|
||||
const connectPromise = client.connect();
|
||||
mock.triggerOpen({ features: { ownedSubscriptions: true, binaryChunkingV1: chunked } });
|
||||
await connectPromise;
|
||||
|
||||
const received: unknown[] = [];
|
||||
const unsubscribe = client.on("fetch_agent_timeline_response", (msg) => {
|
||||
received.push(msg);
|
||||
});
|
||||
const received: unknown[] = [];
|
||||
const unsubscribe = client.on("fetch_agent_timeline_response", (msg) => {
|
||||
received.push(msg);
|
||||
});
|
||||
|
||||
mock.triggerMessage(
|
||||
wrapSessionMessage({
|
||||
type: "fetch_agent_timeline_response",
|
||||
const command = chunked ? "x".repeat(50 * 1024 * 1024) : "pwd";
|
||||
const { ChunkedTransport } = await import("@getpaseo/protocol/chunked-transport");
|
||||
const sender = new ChunkedTransport({
|
||||
frameBytes: 24 * 1024 * 1024,
|
||||
send: (frame) => mock.triggerMessage(frame),
|
||||
deliver: () => {},
|
||||
});
|
||||
sender.enabled = chunked;
|
||||
await sender.send(
|
||||
wrapSessionMessage({
|
||||
type: "fetch_agent_timeline_response",
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
agentId: "agent_cli",
|
||||
agent: null,
|
||||
direction: "tail",
|
||||
projection: "projected",
|
||||
epoch: "epoch-1",
|
||||
reset: false,
|
||||
staleCursor: false,
|
||||
gap: false,
|
||||
window: { minSeq: 1, maxSeq: 1, nextSeq: 2 },
|
||||
startCursor: { epoch: "epoch-1", seq: 1 },
|
||||
endCursor: { epoch: "epoch-1", seq: 1 },
|
||||
hasOlder: false,
|
||||
hasNewer: false,
|
||||
entries: [
|
||||
{
|
||||
timestamp: "2026-02-08T20:20:00.000Z",
|
||||
provider: "codex",
|
||||
seqStart: 1,
|
||||
seqEnd: 1,
|
||||
sourceSeqRanges: [{ startSeq: 1, endSeq: 1 }],
|
||||
collapsed: [],
|
||||
item: {
|
||||
type: "tool_call",
|
||||
callId: "call_cli_snapshot",
|
||||
name: "shell",
|
||||
status: "running",
|
||||
detail: {
|
||||
type: "shell",
|
||||
command,
|
||||
},
|
||||
error: null,
|
||||
},
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
unsubscribe();
|
||||
|
||||
expect(received).toHaveLength(1);
|
||||
expect(received[0]).toMatchObject({
|
||||
payload: {
|
||||
requestId: "req-1",
|
||||
agentId: "agent_cli",
|
||||
agent: null,
|
||||
direction: "tail",
|
||||
projection: "projected",
|
||||
epoch: "epoch-1",
|
||||
reset: false,
|
||||
staleCursor: false,
|
||||
gap: false,
|
||||
window: { minSeq: 1, maxSeq: 1, nextSeq: 2 },
|
||||
startCursor: { epoch: "epoch-1", seq: 1 },
|
||||
endCursor: { epoch: "epoch-1", seq: 1 },
|
||||
hasOlder: false,
|
||||
hasNewer: false,
|
||||
entries: [
|
||||
{
|
||||
timestamp: "2026-02-08T20:20:00.000Z",
|
||||
provider: "codex",
|
||||
seqStart: 1,
|
||||
seqEnd: 1,
|
||||
sourceSeqRanges: [{ startSeq: 1, endSeq: 1 }],
|
||||
collapsed: [],
|
||||
item: {
|
||||
type: "tool_call",
|
||||
callId: "call_cli_snapshot",
|
||||
name: "shell",
|
||||
status: "running",
|
||||
detail: {
|
||||
type: "shell",
|
||||
command: "pwd",
|
||||
},
|
||||
error: null,
|
||||
detail: { type: "shell", command },
|
||||
},
|
||||
},
|
||||
],
|
||||
error: null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
unsubscribe();
|
||||
|
||||
expect(received).toHaveLength(1);
|
||||
expect(received[0]).toMatchObject({
|
||||
payload: {
|
||||
entries: [
|
||||
{
|
||||
item: {
|
||||
type: "tool_call",
|
||||
status: "running",
|
||||
error: null,
|
||||
detail: { type: "shell" },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(logger.warn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
expect(logger.warn).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
test("rejects and logs a correlated response that violates the protocol schema", async () => {
|
||||
const mock = createMockTransport();
|
||||
@@ -8471,3 +8483,302 @@ test("closes before admission when a protected-capable daemon is reached in lega
|
||||
expect(daemonReceived).toHaveLength(1);
|
||||
expect(JSON.parse(daemonReceived[0] as string)).toMatchObject({ type: "hello" });
|
||||
});
|
||||
|
||||
test("uses the frame budget selected by an asynchronously opened transport", async () => {
|
||||
const mock = createMockTransport();
|
||||
mock.transport.maxFrameBytes = 1024;
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "dynamic-budget",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.transport.maxFrameBytes = 16;
|
||||
mock.triggerOpen({ features: { binaryChunkingV1: true } });
|
||||
await connecting;
|
||||
client.sendHostFrame("route", new Uint8Array(100));
|
||||
await vi.waitFor(() => expect(mock.sent.length).toBeGreaterThan(1));
|
||||
expect(mock.sent.every((frame) => typeof frame !== "string" && frame.byteLength <= 16)).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test.each([false, true])("routed sends wait for origin drain (binary=%s)", async (binary) => {
|
||||
const mock = createMockTransport();
|
||||
let drain!: () => void;
|
||||
let blocked = false;
|
||||
const send = mock.transport.send;
|
||||
mock.transport.send = (frame) => {
|
||||
send(frame);
|
||||
if (blocked)
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
};
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "routed-drain",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.triggerOpen({ features: { binaryChunkingV1: true } });
|
||||
await connecting;
|
||||
blocked = true;
|
||||
let finished = false;
|
||||
const write = Promise.resolve(
|
||||
client.sendHostFrame("route", binary ? new Uint8Array([1]) : "hello"),
|
||||
).then(() => {
|
||||
finished = true;
|
||||
return undefined;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(finished).toBe(false);
|
||||
drain();
|
||||
await write;
|
||||
expect(finished).toBe(true);
|
||||
});
|
||||
|
||||
test("a queued admission failure rejects its RPC without closing the client", async () => {
|
||||
const mock = createMockTransport();
|
||||
const send = mock.transport.send;
|
||||
let block = false;
|
||||
let drain!: () => void;
|
||||
mock.transport.send = (frame) => {
|
||||
if (!block) return send(frame);
|
||||
if (typeof frame !== "string")
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
throw new Error("capacity");
|
||||
};
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "queued-admission",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.triggerOpen();
|
||||
await connecting;
|
||||
block = true;
|
||||
const first = client.sendHostFrame("route", new Uint8Array([1]));
|
||||
const ping = client.ping({ timeoutMs: 1000 });
|
||||
void ping.catch(() => {});
|
||||
drain();
|
||||
await first;
|
||||
await expect(ping).rejects.toThrow("capacity");
|
||||
expect(client.isConnected).toBe(true);
|
||||
});
|
||||
|
||||
test("RPC deadlines remain active while a preceding frame waits for drain", async () => {
|
||||
const mock = createMockTransport();
|
||||
const send = mock.transport.send;
|
||||
let blocked = false;
|
||||
let drain!: () => void;
|
||||
mock.transport.send = (frame) => {
|
||||
if (!blocked) return send(frame);
|
||||
if (typeof frame !== "string")
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
};
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "drain-deadline",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.triggerOpen();
|
||||
await connecting;
|
||||
blocked = true;
|
||||
const first = client.sendHostFrame("route", new Uint8Array([1]));
|
||||
try {
|
||||
await expect(client.ping({ timeoutMs: 20 })).rejects.toThrow("Timeout waiting for message");
|
||||
expect(client.isConnected).toBe(true);
|
||||
} finally {
|
||||
drain();
|
||||
await first;
|
||||
}
|
||||
});
|
||||
|
||||
test.each([false, true])(
|
||||
"event listener errors do not close the transport (chunked=%s)",
|
||||
async (chunked) => {
|
||||
const mock = createMockTransport();
|
||||
const closed = vi.fn();
|
||||
mock.transport.close = closed;
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "listener-error",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.triggerOpen({ features: { binaryChunkingV1: chunked } });
|
||||
await connecting;
|
||||
const unsubscribe = client.subscribe(() => {
|
||||
throw new Error("listener failed");
|
||||
});
|
||||
const text = wrapSessionMessage({
|
||||
type: "agent_deleted",
|
||||
payload: { agentId: "agent-test", requestId: "listener-test" },
|
||||
});
|
||||
const payload = new TextEncoder().encode(text);
|
||||
const frame = chunked ? new Uint8Array([0x31, 1, 0, 3, ...payload]) : text;
|
||||
expect(() => mock.triggerMessage(frame)).toThrow("listener failed");
|
||||
expect(closed).not.toHaveBeenCalled();
|
||||
unsubscribe();
|
||||
const events: string[] = [];
|
||||
client.subscribe((event) => {
|
||||
events.push(event.type);
|
||||
});
|
||||
mock.triggerMessage(frame);
|
||||
expect(events).toEqual(["agent_deleted"]);
|
||||
await client.sendHostFrame("route", "still connected");
|
||||
expect(client.isConnected).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
test("malformed chunk metadata still closes the client transport", async () => {
|
||||
const mock = createMockTransport();
|
||||
const closed = vi.fn();
|
||||
mock.transport.close = closed;
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "malformed-chunk",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.triggerOpen({ features: { binaryChunkingV1: true } });
|
||||
await connecting;
|
||||
mock.triggerMessage(new Uint8Array([0x31, 2, 0, 3, 65]));
|
||||
expect(closed).toHaveBeenCalledWith(4003, "Invalid chunked transport");
|
||||
});
|
||||
|
||||
test.each([FileTransferOpcode.FileBegin, FileTransferOpcode.FileChunk, FileTransferOpcode.FileEnd])(
|
||||
"upload stops at a queued pre-write rejection (opcode=%s)",
|
||||
async (opcode) => {
|
||||
const mock = createMockTransport();
|
||||
const send = mock.transport.send;
|
||||
let drain!: () => void;
|
||||
let blocked = false;
|
||||
const attempts: number[] = [];
|
||||
mock.transport.send = (frame) => {
|
||||
if (typeof frame !== "string") {
|
||||
const decoded = decodeFileTransferFrame(assertUint8Array(frame));
|
||||
if (decoded) {
|
||||
attempts.push(decoded.opcode);
|
||||
if (decoded.opcode === opcode) throw new Error("upload capacity");
|
||||
} else if (blocked)
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
}
|
||||
return send(frame);
|
||||
};
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "upload-admission",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.triggerOpen();
|
||||
await connecting;
|
||||
blocked = true;
|
||||
const first = client.sendHostFrame("route", new Uint8Array([1]));
|
||||
const upload = client.uploadFile({
|
||||
fileName: "file.bin",
|
||||
mimeType: "application/octet-stream",
|
||||
bytes: new Uint8Array([1, 2]),
|
||||
chunkSize: 1,
|
||||
});
|
||||
let rejected: unknown;
|
||||
void upload.catch((error) => {
|
||||
rejected = error;
|
||||
});
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
drain();
|
||||
await first;
|
||||
await vi.waitFor(() => expect(rejected).toMatchObject({ message: "upload capacity" }), {
|
||||
timeout: 500,
|
||||
});
|
||||
expect(attempts.at(-1)).toBe(opcode);
|
||||
expect(client.isConnected).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
test.each(["latency", "dictation start", "dictation finish"])(
|
||||
"queued control send rejection reaches its caller (%s)",
|
||||
async (operation) => {
|
||||
const mock = createMockTransport();
|
||||
const send = mock.transport.send;
|
||||
let blocked = false;
|
||||
let drain!: () => void;
|
||||
mock.transport.send = (frame) => {
|
||||
if (!blocked) return send(frame);
|
||||
if (typeof frame !== "string")
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
throw new Error("control capacity");
|
||||
};
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "control-admission",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.triggerOpen();
|
||||
await connecting;
|
||||
blocked = true;
|
||||
const first = client.sendHostFrame("route", new Uint8Array([1]));
|
||||
let result: Promise<unknown>;
|
||||
if (operation === "latency") result = client.measureLatency({ timeoutMs: 1000 });
|
||||
else if (operation === "dictation start")
|
||||
result = client.startDictationStream("dictation", "pcm");
|
||||
else result = client.finishDictationStream("dictation", 0);
|
||||
const rejected = expect(result).rejects.toThrow("control capacity");
|
||||
drain();
|
||||
await first;
|
||||
await rejected;
|
||||
expect(client.isConnected).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
test("routed channels inherit an origin without drain support", async () => {
|
||||
const { createHostChannelTransport } = await import("./hosts/index.js");
|
||||
const mock = createMockTransport();
|
||||
mock.transport.supportsDrain = false;
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "unpaced-route",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
clients.push(client);
|
||||
const connecting = client.connect();
|
||||
mock.triggerOpen();
|
||||
await connecting;
|
||||
const route = createHostChannelTransport(
|
||||
client,
|
||||
"srv_destination",
|
||||
"scope",
|
||||
"route",
|
||||
)({ url: "ws://route" });
|
||||
expect(route.supportsDrain).toBe(false);
|
||||
route.close();
|
||||
});
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import {
|
||||
ChunkedTransport,
|
||||
ChunkedTransportProtocolError,
|
||||
} from "@getpaseo/protocol/chunked-transport";
|
||||
import {
|
||||
HOST_ROUTE_OPCODE,
|
||||
encodeHostRouteFrame,
|
||||
@@ -1266,7 +1270,7 @@ function toReasonCode(reason: string | null | undefined): string | null {
|
||||
}
|
||||
|
||||
interface PendingSend {
|
||||
send: () => void;
|
||||
send: () => void | Promise<void>;
|
||||
resolve: () => void;
|
||||
reject: (error: Error) => void;
|
||||
timeoutHandle: ReturnType<typeof setTimeout>;
|
||||
@@ -1366,6 +1370,7 @@ export class DaemonClient {
|
||||
failed: (error) => this.logger.error({ err: error }, "Subscription failed"),
|
||||
});
|
||||
private transport: DaemonTransport | null = null;
|
||||
private chunkedTransport: ChunkedTransport | null = null;
|
||||
private helloAuth: HelloAuth;
|
||||
private transportCleanup: Array<() => void> = [];
|
||||
private rawMessageListeners: Set<(message: SessionOutboundMessage) => void> = new Set();
|
||||
@@ -1560,6 +1565,17 @@ export class DaemonClient {
|
||||
});
|
||||
this.traceConnectionAttempt += 1;
|
||||
this.transport = transport;
|
||||
this.chunkedTransport = new ChunkedTransport({
|
||||
get supportsDrain() {
|
||||
return transport.supportsDrain;
|
||||
},
|
||||
get frameBytes() {
|
||||
return transport.maxFrameBytes ?? 32 * 1024 * 1024;
|
||||
},
|
||||
send: (frame) => transport.send(frame),
|
||||
onFailure: () => transport.close(4003, "Transport send failed"),
|
||||
deliver: (frame) => this.handleTransportMessage(frame),
|
||||
});
|
||||
this.lastServerInfoMessage = null;
|
||||
this.authFailureReasonValue = null;
|
||||
|
||||
@@ -1666,7 +1682,16 @@ export class DaemonClient {
|
||||
});
|
||||
}),
|
||||
transport.onMessage((data) => {
|
||||
if (this.transport === transport) this.handleTransportMessage(data);
|
||||
if (this.transport !== transport) return;
|
||||
const bytes = asUint8Array(data);
|
||||
try {
|
||||
if (typeof data === "string" || bytes)
|
||||
this.chunkedTransport!.receive(typeof data === "string" ? data : bytes!);
|
||||
else this.handleTransportMessage(data);
|
||||
} catch (error) {
|
||||
if (!(error instanceof ChunkedTransportProtocolError)) throw error;
|
||||
transport.close(4003, "Invalid chunked transport");
|
||||
}
|
||||
}),
|
||||
];
|
||||
} catch (error) {
|
||||
@@ -1867,9 +1892,9 @@ export class DaemonClient {
|
||||
})
|
||||
).routeId;
|
||||
}
|
||||
sendHostFrame(routeId: string, data: string | Uint8Array): void {
|
||||
sendHostFrame(routeId: string, data: string | Uint8Array): Promise<void> {
|
||||
if (typeof data === "string") {
|
||||
this.sendSessionMessageStrict({
|
||||
return this.sendSessionMessageStrict({
|
||||
type: "hosts.route.frame.request",
|
||||
routeId,
|
||||
data,
|
||||
@@ -1877,7 +1902,7 @@ export class DaemonClient {
|
||||
});
|
||||
} else {
|
||||
if (!this.isConnected || !this.transport) throw this.unavailableError();
|
||||
this.sendTransportFrame(encodeHostRouteFrame(routeId, data));
|
||||
return this.sendTransportFrame(encodeHostRouteFrame(routeId, data));
|
||||
}
|
||||
}
|
||||
private readonly hostFrameListeners = new Set<(frame: HostRouteFrame) => void>();
|
||||
@@ -1935,6 +1960,11 @@ export class DaemonClient {
|
||||
// Message Subscription
|
||||
// ============================================================================
|
||||
|
||||
/** Drain capability inherited by transports routed through this connection. */
|
||||
get supportsDrain(): boolean {
|
||||
return this.transport?.supportsDrain !== false;
|
||||
}
|
||||
|
||||
subscribe(handler: DaemonEventHandler): () => void {
|
||||
this.eventListeners.add(handler);
|
||||
return () => {
|
||||
@@ -2043,7 +2073,7 @@ export class DaemonClient {
|
||||
messageType: string,
|
||||
message: unknown,
|
||||
submitted?: () => void,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
const isOpen =
|
||||
this.config.trace?.isEnabled() === true &&
|
||||
this.beginTraceSection("paseo.ws.message.outbound", {
|
||||
@@ -2052,25 +2082,30 @@ export class DaemonClient {
|
||||
...describeRequestIdentity(message),
|
||||
});
|
||||
try {
|
||||
this.sendTransportFrame(JSON.stringify(message), submitted);
|
||||
return this.sendTransportFrame(JSON.stringify(message), submitted);
|
||||
} finally {
|
||||
this.endTraceSection(isOpen);
|
||||
}
|
||||
}
|
||||
|
||||
private sendSessionPayload(payload: SessionInboundMessage): void {
|
||||
this.sendJsonMessage("session", payload.type, { type: "session", message: payload }, () => {
|
||||
if (!("requestId" in payload) || typeof payload.requestId !== "string") return;
|
||||
for (const waiter of this.waiters) {
|
||||
if (waiter.requestId === payload.requestId) waiter.submitted = true;
|
||||
}
|
||||
});
|
||||
private sendSessionPayload(payload: SessionInboundMessage): Promise<void> {
|
||||
return this.sendJsonMessage(
|
||||
"session",
|
||||
payload.type,
|
||||
{ type: "session", message: payload },
|
||||
() => {
|
||||
if (!("requestId" in payload) || typeof payload.requestId !== "string") return;
|
||||
for (const waiter of this.waiters) {
|
||||
if (waiter.requestId === payload.requestId) waiter.submitted = true;
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private sendTransportFrame(
|
||||
frame: string | Uint8Array | ArrayBuffer,
|
||||
submitted?: () => void,
|
||||
): void {
|
||||
): Promise<void> {
|
||||
if (!this.transport) {
|
||||
throw this.unavailableError();
|
||||
}
|
||||
@@ -2083,7 +2118,8 @@ export class DaemonClient {
|
||||
});
|
||||
try {
|
||||
submitted?.();
|
||||
this.transport.send(frame);
|
||||
const payload = typeof frame === "string" ? frame : asUint8Array(frame)!;
|
||||
return this.chunkedTransport!.send(payload);
|
||||
} finally {
|
||||
this.endTraceSection(isOpen);
|
||||
}
|
||||
@@ -2112,10 +2148,10 @@ export class DaemonClient {
|
||||
}
|
||||
}
|
||||
|
||||
private sendBinaryFrame(frame: Uint8Array): void {
|
||||
private sendBinaryFrame(frame: Uint8Array): Promise<void> {
|
||||
if (!this.transport || this.connectionState.status !== "connected") {
|
||||
if (this.config.suppressSendErrors) {
|
||||
return;
|
||||
return Promise.resolve();
|
||||
}
|
||||
throw new Error(`Transport not connected (status: ${this.connectionState.status})`);
|
||||
}
|
||||
@@ -2124,10 +2160,10 @@ export class DaemonClient {
|
||||
envelopeType: "binary",
|
||||
messageType: "binary",
|
||||
});
|
||||
this.sendTransportFrame(frame);
|
||||
return this.sendTransportFrame(frame);
|
||||
} catch (error) {
|
||||
if (this.config.suppressSendErrors) {
|
||||
return;
|
||||
return Promise.resolve();
|
||||
}
|
||||
throw error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
@@ -2142,7 +2178,7 @@ export class DaemonClient {
|
||||
private sendSessionMessageOrThrow(message: SessionInboundMessage): Promise<void> {
|
||||
return this.sendWhenConnected(() => {
|
||||
const payload = SessionInboundMessageSchema.parse(message);
|
||||
this.sendSessionPayload(payload);
|
||||
return this.sendSessionPayload(payload);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2151,13 +2187,12 @@ export class DaemonClient {
|
||||
return this.sendWhenConnected(() => undefined);
|
||||
}
|
||||
|
||||
private sendWhenConnected(send: () => void): Promise<void> {
|
||||
private sendWhenConnected(send: () => void | Promise<void>): Promise<void> {
|
||||
const status = this.connectionState.status;
|
||||
|
||||
// If connected, send immediately
|
||||
if (this.transport && status === "connected") {
|
||||
send();
|
||||
return Promise.resolve();
|
||||
return Promise.resolve(send());
|
||||
}
|
||||
|
||||
// If connecting, queue the message to be sent once connected
|
||||
@@ -2196,8 +2231,7 @@ export class DaemonClient {
|
||||
clearTimeout(pending.timeoutHandle);
|
||||
try {
|
||||
if (this.transport && this.connectionState.status === "connected") {
|
||||
pending.send();
|
||||
pending.resolve();
|
||||
void Promise.resolve(pending.send()).then(pending.resolve, pending.reject);
|
||||
} else {
|
||||
pending.reject(new DaemonConnectionError("Connection lost before message could be sent"));
|
||||
}
|
||||
@@ -2253,19 +2287,17 @@ export class DaemonClient {
|
||||
{ ...params.options, requestId: params.requestId },
|
||||
);
|
||||
|
||||
// The response deadline still applies while the transport waits for drain.
|
||||
void promise.catch(() => undefined);
|
||||
try {
|
||||
await this.sendSessionMessageOrThrow(wire.message);
|
||||
const sent = this.sendSessionMessageOrThrow(wire.message);
|
||||
const result = await Promise.race([promise, sent.then(() => promise)]);
|
||||
if (result.kind === "error") throw result.error;
|
||||
return result.value;
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
cancel(err);
|
||||
void promise.catch(() => undefined);
|
||||
throw err;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await promise;
|
||||
if (result.kind === "error") throw result.error;
|
||||
return result.value;
|
||||
} finally {
|
||||
await wire.finish();
|
||||
}
|
||||
@@ -2355,13 +2387,13 @@ export class DaemonClient {
|
||||
});
|
||||
}
|
||||
|
||||
private sendSessionMessageStrict(message: SessionInboundMessage): void {
|
||||
private sendSessionMessageStrict(message: SessionInboundMessage): Promise<void> {
|
||||
if (!this.transport || this.connectionState.status !== "connected") {
|
||||
throw this.unavailableError();
|
||||
}
|
||||
const payload = SessionInboundMessageSchema.parse(message);
|
||||
try {
|
||||
this.sendSessionPayload(payload);
|
||||
return this.sendSessionPayload(payload);
|
||||
} catch (error) {
|
||||
throw error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
@@ -2559,15 +2591,16 @@ export class DaemonClient {
|
||||
};
|
||||
this.pingProbe = probe;
|
||||
|
||||
try {
|
||||
this.sendJsonMessage("ping", "ping", { type: "ping" });
|
||||
} catch (error) {
|
||||
this.clearPingProbe();
|
||||
const rejectSend = (error: unknown) => {
|
||||
if (this.pingProbe !== probe) return;
|
||||
const sendError = error instanceof Error ? error : new Error(String(error));
|
||||
if (probe.drivesLivenessFailure) {
|
||||
this.recordLivenessFailure(sendError);
|
||||
}
|
||||
return Promise.reject(sendError);
|
||||
this.rejectPingProbe(sendError);
|
||||
if (probe.drivesLivenessFailure) this.recordLivenessFailure(sendError);
|
||||
};
|
||||
try {
|
||||
void this.sendJsonMessage("ping", "ping", { type: "ping" }).catch(rejectSend);
|
||||
} catch (error) {
|
||||
rejectSend(error);
|
||||
}
|
||||
|
||||
return promise;
|
||||
@@ -4222,7 +4255,7 @@ export class DaemonClient {
|
||||
}
|
||||
|
||||
async sendVoiceAudioChunk(audio: string, format: string, isLast = false): Promise<void> {
|
||||
this.sendSessionMessage({ type: "voice_audio_chunk", audio, format, isLast });
|
||||
return this.sendSessionMessageStrict({ type: "voice_audio_chunk", audio, format, isLast });
|
||||
}
|
||||
|
||||
async startDictationStream(dictationId: string, format: string): Promise<void> {
|
||||
@@ -4263,8 +4296,13 @@ export class DaemonClient {
|
||||
|
||||
const cleanupError = new Error("Cancelled dictation start waiter");
|
||||
try {
|
||||
this.sendSessionMessageStrict({ type: "dictation_stream_start", dictationId, format });
|
||||
await Promise.race([ackPromise, errorPromise]);
|
||||
const sent = this.sendSessionMessageStrict({
|
||||
type: "dictation_stream_start",
|
||||
dictationId,
|
||||
format,
|
||||
});
|
||||
const response = Promise.race([ackPromise, errorPromise]);
|
||||
await Promise.race([response, sent.then(() => response)]);
|
||||
} finally {
|
||||
ack.cancel(cleanupError);
|
||||
streamError.cancel(cleanupError);
|
||||
@@ -4399,12 +4437,17 @@ export class DaemonClient {
|
||||
|
||||
const cleanupError = new Error("Cancelled dictation finish waiter");
|
||||
try {
|
||||
this.sendSessionMessageStrict({ type: "dictation_stream_finish", dictationId, finalSeq });
|
||||
const firstOutcome = await Promise.race([
|
||||
const sent = this.sendSessionMessageStrict({
|
||||
type: "dictation_stream_finish",
|
||||
dictationId,
|
||||
finalSeq,
|
||||
});
|
||||
const response = Promise.race([
|
||||
finalOutcomePromise,
|
||||
errorOutcomePromise,
|
||||
finishAcceptedOutcomePromise,
|
||||
]);
|
||||
const firstOutcome = await Promise.race([response, sent.then(() => response)]);
|
||||
|
||||
if (firstOutcome.kind === "final") {
|
||||
return firstOutcome.payload;
|
||||
@@ -5320,8 +5363,10 @@ export class DaemonClient {
|
||||
return undefined;
|
||||
},
|
||||
);
|
||||
const sendFrame = (frame: Uint8Array) =>
|
||||
Promise.race([this.sendBinaryFrame(frame), responsePromise]);
|
||||
try {
|
||||
this.sendBinaryFrame(
|
||||
await sendFrame(
|
||||
encodeFileTransferFrame({
|
||||
opcode: FileTransferOpcode.FileBegin,
|
||||
requestId: resolvedRequestId,
|
||||
@@ -5335,6 +5380,7 @@ export class DaemonClient {
|
||||
}),
|
||||
);
|
||||
|
||||
if (settled) return await responsePromise;
|
||||
const chunkSize = input.chunkSize ?? 128 * 1024;
|
||||
for (let offset = 0; offset < bytes.byteLength; offset += chunkSize) {
|
||||
// Native WebSocket.send encodes binary synchronously. Let rendering and
|
||||
@@ -5344,7 +5390,7 @@ export class DaemonClient {
|
||||
if (this.transport !== uploadTransport || this.connectionState.status !== "connected") {
|
||||
throw new DaemonConnectionError("Connection changed during file upload");
|
||||
}
|
||||
this.sendBinaryFrame(
|
||||
await sendFrame(
|
||||
encodeFileTransferFrame({
|
||||
opcode: FileTransferOpcode.FileChunk,
|
||||
requestId: resolvedRequestId,
|
||||
@@ -5353,7 +5399,7 @@ export class DaemonClient {
|
||||
);
|
||||
}
|
||||
|
||||
this.sendBinaryFrame(
|
||||
await sendFrame(
|
||||
encodeFileTransferFrame({
|
||||
opcode: FileTransferOpcode.FileEnd,
|
||||
requestId: resolvedRequestId,
|
||||
@@ -6763,6 +6809,8 @@ export class DaemonClient {
|
||||
}
|
||||
|
||||
private disposeTransport(code = 1001, reason = "Reconnecting"): void {
|
||||
this.chunkedTransport?.close();
|
||||
this.chunkedTransport = null;
|
||||
this.owned.disconnected();
|
||||
this.providerSnapshotUpdates.clear();
|
||||
this.stopLivenessHeartbeat();
|
||||
@@ -7232,15 +7280,6 @@ export class DaemonClient {
|
||||
probe.resolve(perfNow() - probe.startedAt);
|
||||
}
|
||||
|
||||
private clearPingProbe(): void {
|
||||
const probe = this.pingProbe;
|
||||
if (!probe) {
|
||||
return;
|
||||
}
|
||||
this.pingProbe = null;
|
||||
clearTimeout(probe.timeoutHandle);
|
||||
}
|
||||
|
||||
private rejectPingProbe(error: Error): void {
|
||||
const probe = this.pingProbe;
|
||||
if (!probe) {
|
||||
@@ -7289,6 +7328,10 @@ export class DaemonClient {
|
||||
if (consumerMessage.type === "status") {
|
||||
const serverInfo = parseServerInfoStatusPayload(consumerMessage.payload);
|
||||
if (serverInfo) {
|
||||
if (this.chunkedTransport)
|
||||
this.chunkedTransport.enabled =
|
||||
serverInfo.features?.binaryChunkingV1 === true &&
|
||||
this.config.capabilities?.[CLIENT_CAPS.binaryChunkingV1] !== false;
|
||||
this.lastServerInfoMessage = serverInfo;
|
||||
this.destinationServerId = serverInfo.serverId;
|
||||
this.connectionFailure = undefined;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { MAX_HOST_CHANNEL_FRAME_BYTES } from "@getpaseo/protocol/binary-frames/index";
|
||||
import { safeRandomId } from "../daemon-client-transport-utils.js";
|
||||
import { DaemonClient } from "../daemon-client.js";
|
||||
import type { DaemonTransportFactory } from "../daemon-client-transport-types.js";
|
||||
@@ -84,13 +85,17 @@ export function createHostChannelTransport(
|
||||
})
|
||||
.catch(close);
|
||||
return {
|
||||
get supportsDrain() {
|
||||
return origin.supportsDrain;
|
||||
},
|
||||
maxFrameBytes: MAX_HOST_CHANNEL_FRAME_BYTES,
|
||||
send(data) {
|
||||
if (closed || !routeId)
|
||||
throw Object.assign(new Error(`Host ${serverId} is offline`), {
|
||||
code: "host_offline",
|
||||
serverId,
|
||||
});
|
||||
origin.sendHostFrame(
|
||||
return origin.sendHostFrame(
|
||||
routeId,
|
||||
typeof data === "string" || data instanceof Uint8Array ? data : new Uint8Array(data),
|
||||
);
|
||||
|
||||
@@ -44,12 +44,14 @@ test("a slow consumer cannot exceed the shared byte budget and close releases re
|
||||
socket.pause();
|
||||
await ready;
|
||||
const frame = new Uint8Array(1024 * 1024);
|
||||
for (let index = 0; index < 4; index++) transport.send(frame);
|
||||
const writes = Array.from({ length: 4 }, () => transport.send(frame));
|
||||
const drained = Promise.allSettled(writes);
|
||||
expect(budget.bufferedBytes).toBe(4 * 1024 * 1024);
|
||||
expect(() => transport.send(frame)).toThrow("Connected host buffer limit reached");
|
||||
expect(budget.bufferedBytes).toBeLessThanOrEqual(budget.limit);
|
||||
transport.close();
|
||||
await expect.poll(() => budget.bufferedBytes).toBe(0);
|
||||
await drained;
|
||||
});
|
||||
test("direct connector preserves 25 MiB binary frames in both directions", async () => {
|
||||
const server = await endpoint();
|
||||
@@ -65,12 +67,13 @@ test("direct connector preserves 25 MiB binary frames in both directions", async
|
||||
await ready;
|
||||
const bytes = new Uint8Array(25 * 1024 * 1024).fill(173);
|
||||
const echoed = once(socket, "message");
|
||||
transport.send(bytes);
|
||||
const sending = transport.send(bytes);
|
||||
const [sent, binary] = await echoed;
|
||||
expect(binary).toBe(true);
|
||||
expect(Buffer.from(sent as Uint8Array).equals(bytes)).toBe(true);
|
||||
socket.send(bytes);
|
||||
expect(Buffer.from(await received).equals(bytes)).toBe(true);
|
||||
await sending;
|
||||
});
|
||||
|
||||
import {
|
||||
@@ -124,24 +127,29 @@ test("encrypted connector rejects overload synchronously and preserves the physi
|
||||
});
|
||||
await new Promise<void>((resolve) => transport.onOpen(resolve));
|
||||
await ready;
|
||||
expect(transport.maxFrameBytes).toBeLessThan(32 * 1024 * 1024);
|
||||
peer.pause();
|
||||
const frame = new Uint8Array(1024 * 1024).fill(173);
|
||||
let rejected: unknown;
|
||||
const writes: (void | Promise<void>)[] = [];
|
||||
for (let index = 0; index < 8; index++) {
|
||||
try {
|
||||
transport.send(frame);
|
||||
writes.push(transport.send(frame));
|
||||
} catch (error) {
|
||||
rejected = error;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const drained = Promise.allSettled(writes);
|
||||
expect(rejected).toMatchObject({ code: "resource_limit" });
|
||||
expect(budget.bufferedBytes).toBeLessThanOrEqual(budget.limit);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(errors).toEqual([]);
|
||||
expect(closed).toBe(false);
|
||||
transport.send("sibling control");
|
||||
const sibling = transport.send("sibling control");
|
||||
peer.resume();
|
||||
await drained;
|
||||
await sibling;
|
||||
await expect.poll(() => received.includes("sibling control")).toBe(true);
|
||||
expect(received.filter((data) => data instanceof ArrayBuffer).length).toBeGreaterThan(0);
|
||||
expect(
|
||||
|
||||
@@ -172,9 +172,15 @@ export function createHostTransportFactory(
|
||||
close();
|
||||
});
|
||||
return {
|
||||
get supportsDrain() {
|
||||
return transport?.supportsDrain;
|
||||
},
|
||||
get maxFrameBytes() {
|
||||
return transport?.maxFrameBytes;
|
||||
},
|
||||
send(data) {
|
||||
if (closed || !transport) throw new Error("Host transport is unavailable");
|
||||
transport.send(data);
|
||||
return transport.send(data);
|
||||
},
|
||||
close,
|
||||
onOpen(handler) {
|
||||
|
||||
@@ -3,6 +3,11 @@ import { MAX_WEBSOCKET_MESSAGE_BYTES } from "../websocket-message-limit.js";
|
||||
export const HOST_ROUTE_OPCODE = 0x30;
|
||||
export const MAX_HOST_ROUTE_ID_BYTES = 256;
|
||||
export const MAX_HOST_ROUTE_HEADER_BYTES = 3 + MAX_HOST_ROUTE_ID_BYTES;
|
||||
// A logical host channel crosses the 32 MiB route admission boundary. Text can
|
||||
// expand sixfold when enclosed in JSON; binary adds the bounded route header.
|
||||
export const MAX_HOST_CHANNEL_FRAME_BYTES = Math.floor(
|
||||
(32 * 1024 * 1024 - MAX_HOST_ROUTE_HEADER_BYTES) / 6,
|
||||
);
|
||||
export const MAX_ROUTED_PAYLOAD_BYTES = MAX_WEBSOCKET_MESSAGE_BYTES - MAX_HOST_ROUTE_HEADER_BYTES;
|
||||
export interface HostRouteFrame {
|
||||
routeId: string;
|
||||
|
||||
@@ -0,0 +1,392 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ChunkedTransport } from "./chunked-transport.js";
|
||||
|
||||
describe("chunked transport", () => {
|
||||
it("delivers a large UTF-8 logical message once, preserving order", async () => {
|
||||
const delivered: (string | Uint8Array)[] = [];
|
||||
const receiver = new ChunkedTransport({
|
||||
frameBytes: 17,
|
||||
send: () => {},
|
||||
deliver: (v) => delivered.push(v),
|
||||
});
|
||||
receiver.enabled = true;
|
||||
const frames: (string | Uint8Array)[] = [];
|
||||
const sender = new ChunkedTransport({
|
||||
frameBytes: 17,
|
||||
send: (v) => {
|
||||
frames.push(v);
|
||||
receiver.receive(v);
|
||||
},
|
||||
deliver: () => {},
|
||||
});
|
||||
sender.enabled = true;
|
||||
const text = "🌍".repeat(100);
|
||||
await sender.send(text);
|
||||
expect(delivered).toEqual([text]);
|
||||
expect(frames.length).toBeGreaterThan(1);
|
||||
expect(frames.every((v) => v instanceof Uint8Array && v.byteLength <= 17)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves binary kind and leaves legacy frames unchanged", async () => {
|
||||
const delivered: (string | Uint8Array)[] = [];
|
||||
const receiver = new ChunkedTransport({
|
||||
frameBytes: 8,
|
||||
send: () => {},
|
||||
deliver: (v) => delivered.push(v),
|
||||
});
|
||||
const frames: (string | Uint8Array)[] = [];
|
||||
const sender = new ChunkedTransport({
|
||||
frameBytes: 8,
|
||||
send: (v) => {
|
||||
frames.push(v);
|
||||
receiver.receive(v);
|
||||
},
|
||||
deliver: () => {},
|
||||
});
|
||||
const data = new Uint8Array(40).fill(5);
|
||||
await sender.send(data);
|
||||
expect(frames).toEqual([data]);
|
||||
frames.length = 0;
|
||||
sender.enabled = receiver.enabled = true;
|
||||
await sender.send(data);
|
||||
expect(frames).toHaveLength(10);
|
||||
expect(delivered).toEqual([data, data]);
|
||||
});
|
||||
|
||||
it("rejects malformed sequences and drops incomplete transfers on close", () => {
|
||||
const delivered: unknown[] = [];
|
||||
const receiver = new ChunkedTransport({
|
||||
frameBytes: 8,
|
||||
send: () => {},
|
||||
deliver: (v) => delivered.push(v),
|
||||
});
|
||||
receiver.enabled = true;
|
||||
receiver.receive(new Uint8Array([0x31, 1, 0, 1, 65]));
|
||||
expect(() => receiver.receive("interrupted")).toThrow("Interrupted");
|
||||
receiver.receive(new Uint8Array([0x31, 1, 0, 2, 66]));
|
||||
expect(delivered).toEqual([]);
|
||||
});
|
||||
|
||||
it("waits for drain before the next chunk and preserves logical ordering", async () => {
|
||||
const frames: (string | Uint8Array)[] = [];
|
||||
let drain!: () => void;
|
||||
const sender = new ChunkedTransport({
|
||||
frameBytes: 8,
|
||||
send: (v) => {
|
||||
frames.push(v);
|
||||
if (frames.length === 1)
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
},
|
||||
deliver: () => {},
|
||||
});
|
||||
sender.enabled = true;
|
||||
const first = sender.send("abcdefghij");
|
||||
const next = new Uint8Array([1, 2, 3, 4]);
|
||||
const second = sender.send(next);
|
||||
await Promise.resolve();
|
||||
expect(frames).toHaveLength(1);
|
||||
expect(sender.queuedByteLength).toBe(4);
|
||||
drain();
|
||||
await Promise.all([first, second]);
|
||||
expect(frames).toHaveLength(4);
|
||||
expect(frames[3]).toEqual(next);
|
||||
expect(sender.queuedByteLength).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[0x31, 2, 0, 3, 65],
|
||||
[0x31, 1, 2, 3, 65],
|
||||
[0x31, 1, 0, 4, 65],
|
||||
[0x31, 1, 0, 2, 65],
|
||||
[0x31, 1, 0, 1],
|
||||
])("rejects malformed chunk metadata %j", (...bytes) => {
|
||||
const receiver = new ChunkedTransport({
|
||||
frameBytes: 8,
|
||||
send: () => {},
|
||||
deliver: () => {
|
||||
throw new Error("Unexpected delivery");
|
||||
},
|
||||
});
|
||||
receiver.enabled = true;
|
||||
expect(() => receiver.receive(new Uint8Array(bytes))).toThrow();
|
||||
});
|
||||
|
||||
it("reconnect discards partial state and requires negotiation again", () => {
|
||||
const delivered: unknown[] = [];
|
||||
const create = () =>
|
||||
new ChunkedTransport({ frameBytes: 8, send: () => {}, deliver: (v) => delivered.push(v) });
|
||||
const old = create();
|
||||
old.enabled = true;
|
||||
old.receive(new Uint8Array([0x31, 1, 0, 1, 65]));
|
||||
old.close();
|
||||
old.receive(new Uint8Array([0x31, 1, 0, 2, 66]));
|
||||
const fresh = create();
|
||||
expect(fresh.enabled).toBe(false);
|
||||
fresh.enabled = true;
|
||||
fresh.receive(new Uint8Array([0x31, 1, 0, 3, 67]));
|
||||
expect(delivered).toEqual(["C"]);
|
||||
});
|
||||
|
||||
it("bounds logical messages waiting behind a blocked writer", async () => {
|
||||
let drain!: () => void;
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 32 * 1024 * 1024,
|
||||
send: () =>
|
||||
new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
}),
|
||||
deliver: () => {},
|
||||
});
|
||||
const first = transport.send(new Uint8Array(100 * 1024 * 1024));
|
||||
const overflow = transport.send("one more byte");
|
||||
try {
|
||||
await expect(overflow).rejects.toThrow("queue");
|
||||
} finally {
|
||||
drain();
|
||||
await first;
|
||||
}
|
||||
});
|
||||
|
||||
it("requests drain for chunks while ordinary frames keep the immediate write path", async () => {
|
||||
const waits: boolean[] = [];
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 8,
|
||||
send: (_frame, wait) => {
|
||||
waits.push(wait);
|
||||
},
|
||||
deliver: () => {},
|
||||
});
|
||||
transport.enabled = true;
|
||||
await transport.send("small");
|
||||
await transport.send("abcdefghij");
|
||||
expect(waits).toEqual([false, true, true, true]);
|
||||
});
|
||||
|
||||
it("charges queued ASCII messages by their UTF-8 bytes", async () => {
|
||||
let drain!: () => void;
|
||||
let blocked = true;
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 32 * 1024 * 1024,
|
||||
send: () => {
|
||||
if (blocked) {
|
||||
blocked = false;
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
}
|
||||
},
|
||||
deliver: () => {},
|
||||
});
|
||||
transport.enabled = true;
|
||||
const first = transport.send(new Uint8Array([1]));
|
||||
const text = "x".repeat(10 * 1024 * 1024);
|
||||
const queued = Array.from({ length: 4 }, () => transport.send(text));
|
||||
const settled = Promise.allSettled(queued);
|
||||
expect(transport.queuedByteLength).toBe(40 * 1024 * 1024);
|
||||
drain();
|
||||
await first;
|
||||
expect((await settled).map((result) => result.status)).toEqual([
|
||||
"fulfilled",
|
||||
"fulfilled",
|
||||
"fulfilled",
|
||||
"fulfilled",
|
||||
]);
|
||||
});
|
||||
|
||||
it("notifies the socket owner when an awaited send overflows the queue", async () => {
|
||||
let drain!: () => void;
|
||||
const failures: unknown[] = [];
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 32 * 1024 * 1024,
|
||||
send: () =>
|
||||
new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
}),
|
||||
deliver: () => {},
|
||||
onFailure: (error) => {
|
||||
failures.push(error);
|
||||
},
|
||||
});
|
||||
const first = transport.send(new Uint8Array(100 * 1024 * 1024));
|
||||
const overflow = transport.send("extra");
|
||||
await expect(overflow).rejects.toThrow("queue");
|
||||
drain();
|
||||
await first;
|
||||
expect(failures).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("bounds retained reassembly memory for many one-byte chunks", async () => {
|
||||
const { spawnSync } = await import("node:child_process");
|
||||
// Exercise the built package as a Node consumer; CI builds it before protocol tests.
|
||||
const entry = new URL("../dist/chunked-transport.js", import.meta.url).href;
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--expose-gc",
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
`
|
||||
import { ChunkedTransport } from ${JSON.stringify(entry)};
|
||||
let delivered = 0;
|
||||
const receiver = new ChunkedTransport({ frameBytes: 8, send() {}, deliver(frame) { delivered = frame.byteLength; } });
|
||||
receiver.enabled = true;
|
||||
global.gc();
|
||||
const before = process.memoryUsage().heapUsed;
|
||||
receiver.receive(new Uint8Array([0x31, 1, 1, 1, 65]));
|
||||
for (let index = 0; index < 300000; index++) receiver.receive(new Uint8Array([0x31, 1, 1, 0, 65]));
|
||||
global.gc();
|
||||
const growth = process.memoryUsage().heapUsed - before;
|
||||
receiver.receive(new Uint8Array([0x31, 1, 1, 2, 66]));
|
||||
console.log(JSON.stringify({ growth, delivered }));
|
||||
`,
|
||||
],
|
||||
{ encoding: "utf8", timeout: 15000 },
|
||||
);
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const observed = JSON.parse(result.stdout);
|
||||
expect(observed.delivered).toBe(300002);
|
||||
expect(observed.growth).toBeLessThan(8 * 1024 * 1024);
|
||||
}, 20000);
|
||||
|
||||
it.each(["ASCII", "中", "🌍", "\ud800", "\udc00", "\ud800x"])(
|
||||
"accounts for queued UTF-8 text %j exactly",
|
||||
async (text) => {
|
||||
let drain!: () => void;
|
||||
let blocked = true;
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 1024,
|
||||
send: () => {
|
||||
if (blocked) {
|
||||
blocked = false;
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
}
|
||||
},
|
||||
deliver: () => {},
|
||||
});
|
||||
const first = transport.send(new Uint8Array([1]));
|
||||
const second = transport.send(text);
|
||||
expect(transport.queuedByteLength).toBe(new TextEncoder().encode(text).byteLength);
|
||||
drain();
|
||||
await Promise.all([first, second]);
|
||||
},
|
||||
);
|
||||
|
||||
it("notifies the owner once when an accepted writer fails", async () => {
|
||||
const failures: unknown[] = [];
|
||||
const error = new Error("write failed");
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 1024,
|
||||
deliver: () => {},
|
||||
onFailure: (failure) => {
|
||||
failures.push(failure);
|
||||
},
|
||||
send: () => Promise.reject(error),
|
||||
});
|
||||
await expect(transport.send("hello", { waitForDrain: true })).rejects.toThrow("write failed");
|
||||
expect(failures).toEqual([error]);
|
||||
expect(() => transport.send("again")).toThrow("Transport closed");
|
||||
expect(failures).toEqual([error]);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"a pre-write rejection preserves the sender (chunked=%s)",
|
||||
async (chunked) => {
|
||||
const failures: unknown[] = [];
|
||||
let reject = true;
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 8,
|
||||
deliver: () => {},
|
||||
onFailure: (error) => {
|
||||
failures.push(error);
|
||||
},
|
||||
send: () => {
|
||||
if (reject) throw new Error("capacity");
|
||||
},
|
||||
});
|
||||
transport.enabled = chunked;
|
||||
expect(() => transport.send("abcdefghij")).toThrow("capacity");
|
||||
expect(failures).toEqual([]);
|
||||
reject = false;
|
||||
await transport.send("sibling");
|
||||
},
|
||||
);
|
||||
|
||||
it("a queued pre-write rejection does not poison the next logical send", async () => {
|
||||
let drain!: () => void;
|
||||
const failures: unknown[] = [];
|
||||
const delivered: string[] = [];
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 1024,
|
||||
deliver: () => {},
|
||||
onFailure: (error) => {
|
||||
failures.push(error);
|
||||
},
|
||||
send: (frame) => {
|
||||
if (frame === "blocked")
|
||||
return new Promise<void>((resolve) => {
|
||||
drain = resolve;
|
||||
});
|
||||
if (frame === "reject") throw new Error("capacity");
|
||||
delivered.push(frame as string);
|
||||
},
|
||||
});
|
||||
const first = transport.send("blocked");
|
||||
const rejected = transport.send("reject");
|
||||
const sibling = transport.send("sibling");
|
||||
const results = Promise.allSettled([first, rejected, sibling]);
|
||||
drain();
|
||||
expect((await results).map((result) => result.status)).toEqual([
|
||||
"fulfilled",
|
||||
"rejected",
|
||||
"fulfilled",
|
||||
]);
|
||||
expect(delivered).toEqual(["sibling"]);
|
||||
expect(failures).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([false, true])("delivery errors leave framing usable (chunked=%s)", async (chunked) => {
|
||||
let fail = true;
|
||||
const delivered: (string | Uint8Array)[] = [];
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 64,
|
||||
send: () => {},
|
||||
deliver: (frame) => {
|
||||
if (fail) throw new Error("listener failed");
|
||||
delivered.push(frame);
|
||||
},
|
||||
});
|
||||
transport.enabled = chunked;
|
||||
const frame = chunked ? new Uint8Array([0x31, 1, 0, 3, 65]) : "A";
|
||||
expect(() => transport.receive(frame)).toThrow("listener failed");
|
||||
fail = false;
|
||||
transport.receive(frame);
|
||||
expect(delivered).toEqual(["A"]);
|
||||
await transport.send("still connected");
|
||||
});
|
||||
|
||||
it("unpaced transports retain ordinary outbound admission and accept inbound chunks", async () => {
|
||||
const frames: unknown[] = [];
|
||||
const delivered: unknown[] = [];
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 8,
|
||||
supportsDrain: false,
|
||||
send: (frame) => {
|
||||
frames.push(frame);
|
||||
if (typeof frame === "string" && frame.length > 8) throw new Error("single frame limit");
|
||||
},
|
||||
deliver: (frame) => {
|
||||
delivered.push(frame);
|
||||
},
|
||||
});
|
||||
transport.enabled = true;
|
||||
expect(() => transport.send("too large for one frame")).toThrow("single frame limit");
|
||||
expect(frames).toEqual(["too large for one frame"]);
|
||||
transport.receive(new Uint8Array([0x31, 1, 0, 3, 65]));
|
||||
expect(delivered).toEqual(["A"]);
|
||||
await transport.send("small");
|
||||
});
|
||||
@@ -0,0 +1,214 @@
|
||||
import { MAX_WEBSOCKET_MESSAGE_BYTES } from "./websocket-message-limit.js";
|
||||
|
||||
type Frame = string | Uint8Array;
|
||||
|
||||
/** A malformed wire frame, distinct from a consumer delivery failure. */
|
||||
export class ChunkedTransportProtocolError extends Error {}
|
||||
// Reserved outside the application binary opcode space. Version, kind, flags follow.
|
||||
const OPCODE = 0x31;
|
||||
const HEADER = 4;
|
||||
|
||||
/** Connection-scoped ordered framing, below serialization and routing, above encryption. */
|
||||
export class ChunkedTransport {
|
||||
enabled = false;
|
||||
private assembly: Uint8Array | undefined;
|
||||
private size = 0;
|
||||
private kind = 0;
|
||||
private closed = false;
|
||||
private pending: Promise<void> | undefined;
|
||||
private queuedBytes = 0;
|
||||
private activeBytes = 0;
|
||||
|
||||
constructor(
|
||||
private readonly io: {
|
||||
frameBytes: number;
|
||||
supportsDrain?: boolean;
|
||||
send: (frame: Frame, waitForDrain: boolean) => void | Promise<void>;
|
||||
deliver: (frame: Frame) => void;
|
||||
onFailure?: (error: unknown) => void;
|
||||
},
|
||||
) {
|
||||
if (io.frameBytes <= HEADER) throw new Error("Invalid transport frame budget");
|
||||
}
|
||||
|
||||
/** Bytes waiting behind the active write; physical buffering owns the active frame. */
|
||||
get queuedByteLength(): number {
|
||||
return this.queuedBytes - this.activeBytes;
|
||||
}
|
||||
get isWriting(): boolean {
|
||||
return this.pending !== undefined;
|
||||
}
|
||||
|
||||
send(frame: Frame, options: { waitForDrain?: boolean; byteLength?: number } = {}): Promise<void> {
|
||||
const byteLength = options.byteLength ?? frameByteLength(frame);
|
||||
if (byteLength > MAX_WEBSOCKET_MESSAGE_BYTES) throw new Error("Logical message too large");
|
||||
const chunk =
|
||||
this.enabled && this.io.supportsDrain !== false && byteLength > this.io.frameBytes;
|
||||
if (this.closed) throw new Error("Transport closed");
|
||||
const write = () => {
|
||||
if (this.closed) throw new Error("Transport closed");
|
||||
this.activeBytes = byteLength;
|
||||
// A synchronous rejection did not admit a frame. Only accepted writes can
|
||||
// fail the connection; queued admission failures still belong to their sender.
|
||||
const result = chunk
|
||||
? this.sendChunks(
|
||||
typeof frame === "string" ? new TextEncoder().encode(frame) : frame,
|
||||
typeof frame === "string",
|
||||
)
|
||||
: this.io.send(frame, options.waitForDrain === true);
|
||||
if (!result) return;
|
||||
return result.catch((error) => {
|
||||
this.fail(error);
|
||||
throw error;
|
||||
});
|
||||
};
|
||||
if (this.queuedBytes + byteLength > MAX_WEBSOCKET_MESSAGE_BYTES) {
|
||||
const error = new Error("Logical output queue exceeded its byte limit");
|
||||
this.fail(error);
|
||||
const rejected = Promise.reject<void>(error);
|
||||
void rejected.catch(() => {});
|
||||
return rejected;
|
||||
}
|
||||
this.queuedBytes += byteLength;
|
||||
let result: void | Promise<void>;
|
||||
try {
|
||||
result = this.pending ? this.pending.then(write) : write();
|
||||
} catch (error) {
|
||||
this.queuedBytes -= byteLength;
|
||||
this.activeBytes = 0;
|
||||
throw error;
|
||||
}
|
||||
if (!result) {
|
||||
this.queuedBytes -= byteLength;
|
||||
this.activeBytes = 0;
|
||||
return Promise.resolve();
|
||||
}
|
||||
const completion = Promise.resolve(result).finally(() => {
|
||||
this.queuedBytes -= byteLength;
|
||||
this.activeBytes = 0;
|
||||
});
|
||||
const tail = completion.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
this.pending = tail;
|
||||
void tail.then(() => {
|
||||
if (this.pending === tail) this.pending = undefined;
|
||||
return undefined;
|
||||
});
|
||||
return completion;
|
||||
}
|
||||
|
||||
private sendChunks(bytes: Uint8Array, text: boolean): Promise<void> {
|
||||
const capacity = this.io.frameBytes - HEADER;
|
||||
const writeChunk = (offset: number) => {
|
||||
if (this.closed) throw new Error("Transport closed");
|
||||
const end = Math.min(bytes.byteLength, offset + capacity);
|
||||
const wire = new Uint8Array(HEADER + end - offset);
|
||||
wire.set([
|
||||
OPCODE,
|
||||
1,
|
||||
text ? 0 : 1,
|
||||
(offset === 0 ? 1 : 0) | (end === bytes.byteLength ? 2 : 0),
|
||||
]);
|
||||
wire.set(bytes.subarray(offset, end), HEADER);
|
||||
return this.io.send(wire, true);
|
||||
};
|
||||
const first = writeChunk(0);
|
||||
return (async () => {
|
||||
await first;
|
||||
for (let offset = capacity; offset < bytes.byteLength; offset += capacity)
|
||||
await writeChunk(offset);
|
||||
})();
|
||||
}
|
||||
|
||||
receive(frame: Frame): void {
|
||||
if (this.closed) return;
|
||||
let message: Frame | undefined;
|
||||
try {
|
||||
message = this.assemble(frame);
|
||||
} catch (error) {
|
||||
this.close();
|
||||
throw new ChunkedTransportProtocolError(
|
||||
error instanceof Error ? error.message : String(error),
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
if (message !== undefined) this.io.deliver(message);
|
||||
}
|
||||
|
||||
private assemble(frame: Frame): Frame | undefined {
|
||||
if (typeof frame === "string" || frame[0] !== OPCODE) {
|
||||
if (this.assembly) throw new Error("Interrupted chunked message");
|
||||
return frame;
|
||||
}
|
||||
if (
|
||||
!this.enabled ||
|
||||
frame.length <= HEADER ||
|
||||
frame[1] !== 1 ||
|
||||
frame[2]! > 1 ||
|
||||
(frame[3]! & ~3) !== 0
|
||||
)
|
||||
throw new Error("Invalid chunk header");
|
||||
const start = (frame[3]! & 1) !== 0;
|
||||
const end = (frame[3]! & 2) !== 0;
|
||||
if (start === !!this.assembly) throw new Error("Invalid chunk sequence");
|
||||
if (start) {
|
||||
this.assembly = new Uint8Array(0);
|
||||
this.kind = frame[2]!;
|
||||
this.size = 0;
|
||||
}
|
||||
if (frame[2] !== this.kind) throw new Error("Chunk kind changed");
|
||||
this.size += frame.length - HEADER;
|
||||
if (this.size > MAX_WEBSOCKET_MESSAGE_BYTES) throw new Error("Logical message too large");
|
||||
const payload = frame.subarray(HEADER);
|
||||
if (this.assembly!.byteLength < this.size) {
|
||||
const capacity = Math.min(
|
||||
MAX_WEBSOCKET_MESSAGE_BYTES,
|
||||
Math.max(this.size, this.assembly!.byteLength * 2, 16384),
|
||||
);
|
||||
const grown = new Uint8Array(capacity);
|
||||
grown.set(this.assembly!.subarray(0, this.size - payload.byteLength));
|
||||
this.assembly = grown;
|
||||
}
|
||||
this.assembly!.set(payload, this.size - payload.byteLength);
|
||||
if (!end) return;
|
||||
const data = this.assembly!.subarray(0, this.size);
|
||||
this.assembly = undefined;
|
||||
this.size = 0;
|
||||
return this.kind === 0 ? new TextDecoder("utf-8", { fatal: true }).decode(data) : data.slice();
|
||||
}
|
||||
|
||||
private fail(error: unknown): void {
|
||||
if (this.closed) return;
|
||||
this.close();
|
||||
this.io.onFailure?.(error);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.closed = true;
|
||||
this.enabled = false;
|
||||
this.assembly = undefined;
|
||||
this.size = 0;
|
||||
}
|
||||
}
|
||||
|
||||
function frameByteLength(frame: Frame): number {
|
||||
if (typeof frame !== "string") return frame.byteLength;
|
||||
let bytes = 0;
|
||||
for (let index = 0; index < frame.length; index++) {
|
||||
const code = frame.charCodeAt(index);
|
||||
if (code <= 0x7f) bytes++;
|
||||
else if (code <= 0x7ff) bytes += 2;
|
||||
else if (
|
||||
code >= 0xd800 &&
|
||||
code <= 0xdbff &&
|
||||
frame.charCodeAt(index + 1) >= 0xdc00 &&
|
||||
frame.charCodeAt(index + 1) <= 0xdfff
|
||||
) {
|
||||
bytes += 4;
|
||||
index++;
|
||||
} else bytes += 3;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
export const CLIENT_CAPS = {
|
||||
binaryChunkingV1: "binary_chunking_v1",
|
||||
helloRejection: "hello_rejection",
|
||||
// COMPAT(ownedSubscriptions): added in v0.8.0, remove legacy ownership after 2027-03-09.
|
||||
ownedSubscriptions: "owned_subscriptions",
|
||||
|
||||
@@ -3834,6 +3834,7 @@ export const ServerInfoStatusPayloadSchema = z
|
||||
selectiveAgentTimeline: z.boolean().optional(),
|
||||
explicitEventSubscriptions: z.boolean().optional(),
|
||||
ownedSubscriptions: z.boolean().optional(),
|
||||
binaryChunkingV1: z.boolean().optional(),
|
||||
workspaceSetupObservation: z.boolean().optional(),
|
||||
workspaceSortTimes: z.boolean().optional(),
|
||||
// COMPAT(canonicalSubmittedPrompts): added in v0.2.6, remove gate after 2027-01-30.
|
||||
|
||||
@@ -2091,3 +2091,80 @@ test.each(["removed", "disabled"] as const)(
|
||||
).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
"a 50 MiB timeline crosses a routed host without dropping content or the route",
|
||||
{ timeout: 30000 },
|
||||
async () => {
|
||||
const b = await daemon();
|
||||
const a = await daemon();
|
||||
const destination = await api(b);
|
||||
const origin = await api(a);
|
||||
const workspace = await destination.paseo.workspaces.create({
|
||||
source: { kind: "directory", path: b.staticDir },
|
||||
});
|
||||
const agent = await workspace.agents.create({
|
||||
config: { provider: "claude/claude-test", modeId: "full-access" },
|
||||
title: "Large history",
|
||||
});
|
||||
// Keep each row below the established item budget; exercise transport size across rows.
|
||||
const text = "x".repeat(240 * 1024);
|
||||
const rowCount = 220;
|
||||
for (let index = 0; index < rowCount; index++) {
|
||||
await b.daemon.agentManager.appendTimelineItem(agent.id, { type: "user_message", text });
|
||||
}
|
||||
await origin.paseo.hosts.add({ connection: `127.0.0.1:${b.port}` });
|
||||
const serverId = (await online(origin.paseo)).hosts[0]!.serverId!;
|
||||
const remote = await origin.paseo.hosts.client(serverId);
|
||||
const response = await remote.agents.ref(agent.id).timeline.refetch({ limit: 0 });
|
||||
const messages = response.entries.flatMap((row) =>
|
||||
row.item.type === "user_message" ? [row.item.text] : [],
|
||||
);
|
||||
expect(messages).toHaveLength(rowCount);
|
||||
expect(messages.every((message) => message === text)).toBe(true);
|
||||
expect(
|
||||
messages.reduce((bytes, message) => bytes + Buffer.byteLength(message), 0),
|
||||
).toBeGreaterThan(50 * 1024 * 1024);
|
||||
expect((await remote.workspaces.list()).entries).toHaveLength(1);
|
||||
await remote.dispose();
|
||||
},
|
||||
);
|
||||
|
||||
test("a pre-write host budget rejection leaves sibling routes connected", async () => {
|
||||
const { HostTransportBudget } = await import("@getpaseo/client/node");
|
||||
const b = await daemon();
|
||||
const origin = createHostClient(
|
||||
`127.0.0.1:${b.port}`,
|
||||
{ clientId: "budget-origin", reconnect: { enabled: false } },
|
||||
new HostTransportBudget(1024 * 1024),
|
||||
);
|
||||
cleanup.push(() => origin.close());
|
||||
await origin.connect();
|
||||
const serverId = origin.getLastServerInfoMessage()!.serverId;
|
||||
const routeIds = [
|
||||
await origin.openHostLink("budget-first"),
|
||||
await origin.openHostLink("budget-sibling"),
|
||||
];
|
||||
const peers = routeIds.map(
|
||||
(id, index) =>
|
||||
new DaemonClient({
|
||||
url: "ws://host-link/ws",
|
||||
clientId: `budget-child-${index}`,
|
||||
reconnect: { enabled: false },
|
||||
transportFactory: createHostChannelTransport(origin, serverId, `budget-${index}`, id),
|
||||
}),
|
||||
);
|
||||
for (const peer of peers) {
|
||||
cleanup.push(() => peer.close());
|
||||
await peer.connect();
|
||||
}
|
||||
await expect(async () =>
|
||||
origin.sendHostFrame(routeIds[0]!, new Uint8Array(2 * 1024 * 1024)),
|
||||
).rejects.toMatchObject({ code: "resource_limit" });
|
||||
await peers[1]!.ping({ timeoutMs: 2000 });
|
||||
expect([origin.isConnected, ...peers.map((peer) => peer.isConnected)]).toEqual([
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
]);
|
||||
}, 30000);
|
||||
|
||||
@@ -510,10 +510,13 @@ export class ConnectedHosts {
|
||||
return;
|
||||
}
|
||||
if (!verified) throw new HostError("host_offline", "Host identity has not been verified");
|
||||
try {
|
||||
transport.send(data);
|
||||
} catch {
|
||||
const rejectWrite = () =>
|
||||
route.close("resource_limit", "Host buffer limit reached or transport unavailable");
|
||||
try {
|
||||
const write = transport.send(data);
|
||||
if (write) void write.catch(rejectWrite);
|
||||
} catch {
|
||||
rejectWrite();
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
import type { SessionDelivery } from "../session/owned-subscriptions/index.js";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { createHash } from "node:crypto";
|
||||
import { encodeHostRouteFrame, type HostRouteFrame } from "@getpaseo/protocol/binary-frames/index";
|
||||
import {
|
||||
MAX_HOST_CHANNEL_FRAME_BYTES,
|
||||
encodeHostRouteFrame,
|
||||
type HostRouteFrame,
|
||||
} from "@getpaseo/protocol/binary-frames/index";
|
||||
import type {
|
||||
DaemonPermission,
|
||||
SessionInboundMessage,
|
||||
@@ -10,6 +14,7 @@ import type {
|
||||
|
||||
/** A logical socket has no network resources or authority of its own. */
|
||||
export class VirtualHostSocket extends EventEmitter {
|
||||
readonly maxFrameBytes = MAX_HOST_CHANNEL_FRAME_BYTES;
|
||||
readyState = 1;
|
||||
constructor(
|
||||
private readonly output: (data: string | Uint8Array) => Promise<void>,
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
} from "@getpaseo/protocol/binary-frames/index";
|
||||
import {
|
||||
getDaemonAuthFailureReason,
|
||||
type DaemonClient as HostClient,
|
||||
DaemonClient as HostClient,
|
||||
} from "@getpaseo/client/internal/daemon-client";
|
||||
import type { SessionOutboundMessage } from "@getpaseo/protocol/messages";
|
||||
import { describe, expect, test } from "vitest";
|
||||
@@ -593,72 +593,112 @@ async function waitForCapturedLog(
|
||||
}
|
||||
}, 90000);
|
||||
|
||||
test("oversized attachments fail without disconnecting or resubmitting over the relay", async () => {
|
||||
await startRelay({ useLocalRelay: true });
|
||||
const daemon = await createTestPaseoDaemon({
|
||||
relayEnabled: true,
|
||||
relayEndpoint: `127.0.0.1:${relayPort}`,
|
||||
relayUseTls: false,
|
||||
relayPublicUseTls: false,
|
||||
});
|
||||
let client: DaemonClient | null = null;
|
||||
try {
|
||||
const offerUrl = await getPairingOfferUrl({
|
||||
paseoHome: daemon.paseoHome,
|
||||
test.each([true, false])(
|
||||
"large messages cross the relay with unpaced upload admission preserved (supportsDrain=%s)",
|
||||
async (supportsDrain) => {
|
||||
await startRelay({ useLocalRelay: true });
|
||||
const daemon = await createTestPaseoDaemon({
|
||||
relayEnabled: true,
|
||||
relayEndpoint: daemon.config.relayEndpoint,
|
||||
relayEndpoint: `127.0.0.1:${relayPort}`,
|
||||
relayUseTls: false,
|
||||
relayPublicUseTls: false,
|
||||
});
|
||||
const { serverId, daemonPublicKeyB64 } = decodeOfferFromFragmentUrl(offerUrl);
|
||||
client = new DaemonClient({
|
||||
url: buildRelayWebSocketUrl({
|
||||
endpoint: `127.0.0.1:${relayPort}`,
|
||||
useTls: false,
|
||||
serverId,
|
||||
role: "client",
|
||||
}),
|
||||
e2ee: { enabled: true, daemonPublicKeyB64 },
|
||||
reconnect: { baseDelayMs: 100, maxDelayMs: 100 },
|
||||
});
|
||||
await client.connect();
|
||||
const statuses: string[] = [];
|
||||
client.subscribeConnectionStatus((state) => statuses.push(state.status));
|
||||
for (const imageMiB of [30, 61]) {
|
||||
// Attachments are base64 in the JSON, then E2EE base64 expands the frame again.
|
||||
const images = [
|
||||
{ data: "A".repeat(4 * Math.ceil((imageMiB * 1024 * 1024) / 3)), mimeType: "image/png" },
|
||||
];
|
||||
let client: HostClient | null = null;
|
||||
try {
|
||||
const offerUrl = await getPairingOfferUrl({
|
||||
paseoHome: daemon.paseoHome,
|
||||
relayEnabled: true,
|
||||
relayEndpoint: daemon.config.relayEndpoint,
|
||||
});
|
||||
const { serverId, daemonPublicKeyB64 } = decodeOfferFromFragmentUrl(offerUrl);
|
||||
client = new HostClient({
|
||||
clientId: "paced-relay-regression",
|
||||
webSocketFactory: Object.assign((url: string) => new WebSocket(url), { supportsDrain }),
|
||||
url: buildRelayWebSocketUrl({
|
||||
endpoint: `127.0.0.1:${relayPort}`,
|
||||
useTls: false,
|
||||
serverId,
|
||||
role: "client",
|
||||
}),
|
||||
e2ee: { enabled: true, daemonPublicKeyB64 },
|
||||
reconnect: { baseDelayMs: 100, maxDelayMs: 100 },
|
||||
});
|
||||
await client.connect();
|
||||
const statuses: string[] = [];
|
||||
client.subscribeConnectionStatus((state) => statuses.push(state.status));
|
||||
for (const imageMiB of [30, 61]) {
|
||||
const images = [
|
||||
{
|
||||
data: "A".repeat(4 * Math.ceil((imageMiB * 1024 * 1024) / 3)),
|
||||
mimeType: "image/png",
|
||||
},
|
||||
];
|
||||
const creation = client.createWorkspace({
|
||||
source: { kind: "directory", path: daemon.staticDir },
|
||||
agent: {
|
||||
provider: "claude",
|
||||
cwd: daemon.staticDir,
|
||||
initialPrompt: "Describe these screens.",
|
||||
images,
|
||||
},
|
||||
});
|
||||
if (supportsDrain)
|
||||
await withTimeout(creation, 30000, "Chunked workspace creation did not settle");
|
||||
else await expect(creation).rejects.toThrow(/too large.*32\.0 MiB/);
|
||||
}
|
||||
expect((await client.fetchAgents()).entries).toHaveLength(supportsDrain ? 2 : 0);
|
||||
await expect(
|
||||
withTimeout(
|
||||
client.createWorkspace({
|
||||
source: { kind: "directory", path: daemon.staticDir },
|
||||
agent: {
|
||||
provider: "claude",
|
||||
cwd: daemon.staticDir,
|
||||
initialPrompt: "Describe these screens.",
|
||||
images,
|
||||
},
|
||||
}),
|
||||
10000,
|
||||
"Workspace creation did not settle",
|
||||
),
|
||||
).rejects.toThrow(/too large.*32\.0 MiB/);
|
||||
expect((await client.fetchAgents()).entries).toEqual([]);
|
||||
}
|
||||
const agent = await client.createAgent({ provider: "claude", cwd: daemon.staticDir });
|
||||
await expect(
|
||||
client.sendMessage(agent.id, "Describe these screens.", {
|
||||
client.createWorkspace({
|
||||
source: { kind: "directory", path: daemon.staticDir },
|
||||
agent: {
|
||||
provider: "claude",
|
||||
cwd: daemon.staticDir,
|
||||
initialPrompt: "Describe these screens.",
|
||||
images: [{ data: "A".repeat(101 * 1024 * 1024), mimeType: "image/png" }],
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow(/too large.*100\.0 MiB/);
|
||||
expect((await client.fetchAgents()).entries).toHaveLength(supportsDrain ? 2 : 0);
|
||||
const agent = await client.createAgent({ provider: "claude", cwd: daemon.staticDir });
|
||||
const attachment = client.sendMessage(agent.id, "Describe these screens.", {
|
||||
images: [{ data: "A".repeat(40 * 1024 * 1024), mimeType: "image/png" }],
|
||||
}),
|
||||
).rejects.toThrow(/too large.*32\.0 MiB/);
|
||||
await client.sendMessage(agent.id, "A small follow-up still works.");
|
||||
expect((await client.fetchAgents()).entries).toHaveLength(1);
|
||||
expect(statuses).toEqual(["connected"]);
|
||||
} finally {
|
||||
await client?.close();
|
||||
await daemon.close();
|
||||
await stopRelay();
|
||||
}
|
||||
}, 60000);
|
||||
});
|
||||
if (supportsDrain) await attachment;
|
||||
else await expect(attachment).rejects.toThrow(/too large.*32\.0 MiB/);
|
||||
await expect(
|
||||
client.sendMessage(agent.id, "Describe these screens.", {
|
||||
images: [{ data: "A".repeat(101 * 1024 * 1024), mimeType: "image/png" }],
|
||||
}),
|
||||
).rejects.toThrow(/too large.*100\.0 MiB/);
|
||||
await client.sendMessage(agent.id, "A small follow-up still works.");
|
||||
expect((await client.fetchAgents()).entries).toHaveLength(supportsDrain ? 3 : 1);
|
||||
// Respect the per-item content budget while exceeding the relay frame limit.
|
||||
const text = "x".repeat(240 * 1024);
|
||||
for (let row = 0; row < 220; row++)
|
||||
await daemon.daemon.agentManager.appendTimelineItem(agent.id, {
|
||||
type: "user_message",
|
||||
text,
|
||||
});
|
||||
const timeline = await client.fetchAgentTimeline(agent.id, {
|
||||
projection: "canonical",
|
||||
limit: 0,
|
||||
timeout: 30000,
|
||||
});
|
||||
const delivered = timeline.entries.filter(
|
||||
(row) => row.item.type === "user_message" && row.item.text === text,
|
||||
);
|
||||
expect(delivered).toHaveLength(220);
|
||||
expect(delivered.length * Buffer.byteLength(text)).toBeGreaterThan(50 * 1024 * 1024);
|
||||
await client.ping();
|
||||
expect(statuses).toEqual(["connected"]);
|
||||
} finally {
|
||||
await client?.close();
|
||||
await daemon.close();
|
||||
await stopRelay();
|
||||
}
|
||||
},
|
||||
120000,
|
||||
);
|
||||
|
||||
test("daemon connects to relay and client ping/pong works through relay", async () => {
|
||||
process.env.PASEO_PRIMARY_LAN_IP = "192.168.1.12";
|
||||
|
||||
@@ -1836,3 +1836,58 @@ describe("list terminals across directories", () => {
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test("a paused terminal consumer catches up through a snapshot instead of retaining every output frame", async () => {
|
||||
const { existsSync } = await import("node:fs");
|
||||
const cwd = tmpCwd();
|
||||
tempDirs.push(cwd);
|
||||
const marker = path.join(cwd, "output-complete");
|
||||
const created = await createTerminalInWorkspace(ctx.client, {
|
||||
cwd,
|
||||
options: {
|
||||
command: process.execPath,
|
||||
args: [
|
||||
"-e",
|
||||
`
|
||||
const fs = require('node:fs');
|
||||
process.stdin.setRawMode?.(true);
|
||||
process.stdin.resume();
|
||||
process.stdin.once('data', () => {
|
||||
const chunk = Buffer.alloc(256 * 1024, 120);
|
||||
let remaining = 128;
|
||||
const write = () => {
|
||||
if (remaining-- === 0) { fs.writeFileSync(${JSON.stringify(marker)}, 'done'); return; }
|
||||
process.stdout.write(chunk, write);
|
||||
};
|
||||
write();
|
||||
});
|
||||
`,
|
||||
],
|
||||
},
|
||||
});
|
||||
const ws = await connectRawWebSocket(ctx.daemon.port);
|
||||
try {
|
||||
const slot = await subscribeRawTerminal(ws, created.terminal!.id, "slow-terminal");
|
||||
await waitForRawBinaryFrame(
|
||||
ws,
|
||||
(frame) => frame.slot === slot && frame.opcode === TerminalStreamOpcode.Snapshot,
|
||||
);
|
||||
const capture = startTerminalOutputCapture(ws, slot);
|
||||
try {
|
||||
ws.pause();
|
||||
sendRawTerminalInput(ws, slot, "go\n");
|
||||
await waitForCondition(() => existsSync(marker), 20000);
|
||||
ws.resume();
|
||||
await waitForCondition(() => capture.snapshotCount > 0, 10000);
|
||||
expect(ws.readyState).toBe(WebSocket.OPEN);
|
||||
expect(capture.frames.reduce((total, frame) => total + frame.payloadBytes, 0)).toBeLessThan(
|
||||
32 * 1024 * 1024,
|
||||
);
|
||||
} finally {
|
||||
ws.resume();
|
||||
capture.stop();
|
||||
}
|
||||
} finally {
|
||||
await closeWebSocket(ws);
|
||||
}
|
||||
}, 40000);
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
import type pino from "pino";
|
||||
import { createPersistedProjectRecord } from "../workspace-registry.js";
|
||||
import type {
|
||||
PersistedProjectRecord,
|
||||
PersistedWorkspaceRecord,
|
||||
ProjectRegistry,
|
||||
WorkspaceRegistry,
|
||||
} from "../workspace-registry.js";
|
||||
|
||||
export function createTestRegistries() {
|
||||
let archiveGate: {
|
||||
started: ReturnType<typeof deferred>;
|
||||
release: ReturnType<typeof deferred>;
|
||||
} | null = null;
|
||||
const holdNextArchive = () => {
|
||||
archiveGate = { started: deferred(), release: deferred() };
|
||||
return archiveGate;
|
||||
};
|
||||
const projects = new Map<string, PersistedProjectRecord>();
|
||||
const workspaces = new Map<string, PersistedWorkspaceRecord>();
|
||||
|
||||
const projectRegistry: ProjectRegistry = {
|
||||
initialize: async () => {},
|
||||
existsOnDisk: async () => true,
|
||||
list: async () => Array.from(projects.values()),
|
||||
get: async (id: string) => projects.get(id) ?? null,
|
||||
getOrCreateActiveByRoot: async (input) => {
|
||||
const existing = Array.from(projects.values()).find(
|
||||
(project) => !project.archivedAt && project.rootPath === input.rootPath,
|
||||
);
|
||||
if (existing) return existing;
|
||||
const record = createPersistedProjectRecord({
|
||||
projectId: `prj_${projects.size}`,
|
||||
rootPath: input.rootPath,
|
||||
kind: input.kind,
|
||||
displayName: input.displayName,
|
||||
createdAt: input.timestamp,
|
||||
updatedAt: input.timestamp,
|
||||
});
|
||||
projects.set(record.projectId, record);
|
||||
return record;
|
||||
},
|
||||
update: async (id, updater) => {
|
||||
const current = projects.get(id);
|
||||
if (!current) return null;
|
||||
const updated = updater(current);
|
||||
projects.set(id, updated);
|
||||
return updated;
|
||||
},
|
||||
upsert: async (record: PersistedProjectRecord) => {
|
||||
projects.set(record.projectId, record);
|
||||
},
|
||||
archive: async (id: string, archivedAt: string) => {
|
||||
const existing = projects.get(id);
|
||||
if (existing) {
|
||||
projects.set(id, { ...existing, archivedAt, updatedAt: archivedAt });
|
||||
}
|
||||
},
|
||||
remove: async (id: string) => {
|
||||
projects.delete(id);
|
||||
},
|
||||
};
|
||||
|
||||
const workspaceRegistry: WorkspaceRegistry = {
|
||||
initialize: async () => {},
|
||||
existsOnDisk: async () => true,
|
||||
list: async () => Array.from(workspaces.values()),
|
||||
get: async (id: string) => workspaces.get(id) ?? null,
|
||||
update: async (id, updater) => {
|
||||
const existing = workspaces.get(id);
|
||||
if (!existing) return null;
|
||||
const updated = updater(existing);
|
||||
workspaces.set(id, updated);
|
||||
return updated;
|
||||
},
|
||||
upsert: async (record: PersistedWorkspaceRecord) => {
|
||||
workspaces.set(record.workspaceId, record);
|
||||
},
|
||||
archive: async (id: string, archivedAt: string) => {
|
||||
const gate = archiveGate;
|
||||
archiveGate = null;
|
||||
if (gate) {
|
||||
gate.started.resolve();
|
||||
await gate.release.promise;
|
||||
}
|
||||
const existing = workspaces.get(id);
|
||||
if (!existing || existing.archivedAt) return false;
|
||||
workspaces.set(id, { ...existing, archivedAt, updatedAt: archivedAt });
|
||||
return true;
|
||||
},
|
||||
remove: async (id: string) => {
|
||||
workspaces.delete(id);
|
||||
},
|
||||
};
|
||||
|
||||
return { projects, workspaces, projectRegistry, workspaceRegistry, holdNextArchive };
|
||||
}
|
||||
|
||||
export function createTestLogger() {
|
||||
const logger = {
|
||||
child: () => logger,
|
||||
trace: () => undefined,
|
||||
debug: () => undefined,
|
||||
info: () => undefined,
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
};
|
||||
return logger as unknown as pino.Logger;
|
||||
}
|
||||
|
||||
export function deferred(): { promise: Promise<void>; resolve(): void } {
|
||||
let resolve!: () => void;
|
||||
const promise = new Promise<void>((accept) => {
|
||||
resolve = accept;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import path from "node:path";
|
||||
import { stat } from "node:fs/promises";
|
||||
import { setImmediate as nextTurn } from "node:timers/promises";
|
||||
import { measureMainThreadWork } from "../../test-utils/main-thread-work.js";
|
||||
import { createTestRegistries, createTestLogger } from "./reconciliation-registries.js";
|
||||
import {
|
||||
createPersistedProjectRecord,
|
||||
createPersistedWorkspaceRecord,
|
||||
} from "../workspace-registry.js";
|
||||
import { WorkspaceReconciliationService } from "../workspace-reconciliation-service.js";
|
||||
import { deriveProjectKey } from "../project-key.js";
|
||||
|
||||
const rootPath = process.argv[2];
|
||||
if (!rootPath) throw new Error("Project root is required");
|
||||
const timestamp = "2025-01-01T00:00:00.000Z";
|
||||
const { projects, workspaces, projectRegistry, workspaceRegistry } = createTestRegistries();
|
||||
projects.set(
|
||||
"p1",
|
||||
createPersistedProjectRecord({
|
||||
projectId: "p1",
|
||||
rootPath,
|
||||
projectKey: deriveProjectKey({
|
||||
rootPath,
|
||||
remoteUrl: null,
|
||||
worktreeRoot: null,
|
||||
mainRepoRoot: null,
|
||||
}),
|
||||
kind: "non_git",
|
||||
displayName: "project",
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
}),
|
||||
);
|
||||
for (let i = 0; i < 20000; i++) {
|
||||
const workspaceId = `w${i}`;
|
||||
workspaces.set(
|
||||
workspaceId,
|
||||
createPersistedWorkspaceRecord({
|
||||
workspaceId,
|
||||
projectId: "p1",
|
||||
cwd: path.join(rootPath, workspaceId),
|
||||
kind: "directory",
|
||||
displayName: workspaceId,
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const service = new WorkspaceReconciliationService({
|
||||
projectRegistry,
|
||||
workspaceRegistry,
|
||||
logger: createTestLogger(),
|
||||
});
|
||||
|
||||
// Compile before launch and measure in plain Node, outside the test runner's
|
||||
// thread and TypeScript loader. Keep the first real reconciliation cold.
|
||||
// Initialize async filesystem support using the bundle, leaving every project
|
||||
// and workspace path untouched until the first reconciliation.
|
||||
await stat(process.argv[1]);
|
||||
await nextTurn();
|
||||
try {
|
||||
const measured = await measureMainThreadWork("workspace reconciliation main-thread work", () =>
|
||||
service.runOnce(),
|
||||
);
|
||||
process.stdout.write(
|
||||
JSON.stringify({
|
||||
blockingMs: measured.blockingMs,
|
||||
filesystemRequests: measured.filesystemRequests,
|
||||
changesApplied: measured.result.changesApplied.length,
|
||||
allArchived: [...workspaces.values()].every((workspace) => workspace.archivedAt !== null),
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
service.dispose();
|
||||
}
|
||||
@@ -90,7 +90,11 @@ test(
|
||||
TEST_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
async function connectSocket(port: number, clientId: string): Promise<WebSocket> {
|
||||
async function connectSocket(
|
||||
port: number,
|
||||
clientId: string,
|
||||
capabilities?: Record<string, unknown>,
|
||||
): Promise<WebSocket> {
|
||||
const socket = new WebSocket(`ws://127.0.0.1:${port}/ws`);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.once("open", resolve);
|
||||
@@ -98,7 +102,7 @@ async function connectSocket(port: number, clientId: string): Promise<WebSocket>
|
||||
});
|
||||
await sendAndWait(
|
||||
socket,
|
||||
{ type: "hello", clientId, clientType: "browser", protocolVersion: 1 },
|
||||
{ type: "hello", clientId, clientType: "browser", protocolVersion: 1, capabilities },
|
||||
(message) =>
|
||||
message.type === "session" &&
|
||||
message.message.type === "status" &&
|
||||
@@ -166,3 +170,97 @@ function sendAndWait(
|
||||
socket.send(JSON.stringify(message));
|
||||
return response;
|
||||
}
|
||||
|
||||
test("an awaited timeline framing overflow closes the actual daemon socket", async () => {
|
||||
const { createHostClient } = await import("@getpaseo/client/node");
|
||||
const { createPaseoApi } = await import("@getpaseo/client");
|
||||
const { default: pino } = await import("pino");
|
||||
const { Writable } = await import("node:stream");
|
||||
const messages: string[] = [];
|
||||
const logger = pino(
|
||||
{ level: "warn" },
|
||||
new Writable({
|
||||
write(chunk, _encoding, done) {
|
||||
messages.push(JSON.parse(chunk.toString()).msg);
|
||||
done();
|
||||
},
|
||||
}),
|
||||
);
|
||||
daemon = await createTestPaseoDaemon({ logger });
|
||||
const client = createHostClient(`127.0.0.1:${daemon.port}`, {
|
||||
clientId: "framing-seed",
|
||||
reconnect: { enabled: false },
|
||||
});
|
||||
await client.connect();
|
||||
const api = createPaseoApi(client);
|
||||
try {
|
||||
const workspace = await api.workspaces.create({
|
||||
source: { kind: "directory", path: daemon.staticDir },
|
||||
});
|
||||
const agents = [];
|
||||
const text = "x".repeat(240 * 1024);
|
||||
for (const [index, rows] of [350, 90].entries()) {
|
||||
const agent = await workspace.agents.create({
|
||||
config: { provider: "claude/claude-test", modeId: "full-access" },
|
||||
title: `Framing overflow ${index}`,
|
||||
});
|
||||
agents.push(agent.id);
|
||||
for (let row = 0; row < rows; row++)
|
||||
await daemon.daemon.agentManager.appendTimelineItem(agent.id, {
|
||||
type: "user_message",
|
||||
text,
|
||||
});
|
||||
}
|
||||
const source = await connectSocket(daemon.port, "framing-overflow", {
|
||||
binary_chunking_v1: true,
|
||||
});
|
||||
sockets.push(source);
|
||||
const closed = new Promise<void>((resolve) => source.once("close", () => resolve()));
|
||||
const firstChunk = new Promise<void>((resolve) => {
|
||||
const listener = (data: RawData, binary: boolean) => {
|
||||
if (!binary || (data as Buffer)[0] !== 0x31) return;
|
||||
source.off("message", listener);
|
||||
source.pause();
|
||||
resolve();
|
||||
};
|
||||
source.on("message", listener);
|
||||
});
|
||||
const fetch = (agentId: string, requestId: string) =>
|
||||
source.send(
|
||||
JSON.stringify({
|
||||
type: "session",
|
||||
message: {
|
||||
type: "fetch_agent_timeline_request",
|
||||
requestId,
|
||||
agentId,
|
||||
direction: "tail",
|
||||
projection: "canonical",
|
||||
limit: 0,
|
||||
},
|
||||
}),
|
||||
);
|
||||
fetch(agents[0]!, "first");
|
||||
await firstChunk;
|
||||
fetch(agents[1]!, "overflow");
|
||||
await expect
|
||||
.poll(() => messages.includes("Closing physical WebSocket after outbound framing failure"), {
|
||||
timeout: 15000,
|
||||
})
|
||||
.toBe(true);
|
||||
expect(messages).not.toContain("Closing physical WebSocket at outbound high-water mark");
|
||||
source.resume();
|
||||
await closed;
|
||||
} finally {
|
||||
await api.dispose();
|
||||
await client.close();
|
||||
}
|
||||
}, 60000);
|
||||
|
||||
test("malformed chunk metadata closes the daemon socket with a protocol error", async () => {
|
||||
daemon = await createTestPaseoDaemon();
|
||||
const source = await connectSocket(daemon.port, "invalid-framing", { binary_chunking_v1: true });
|
||||
sockets.push(source);
|
||||
const closed = new Promise<number>((resolve) => source.once("close", (code) => resolve(code)));
|
||||
source.send(new Uint8Array([0x31, 2, 0, 3, 65]));
|
||||
expect(await closed).toBe(1002);
|
||||
});
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import {
|
||||
ChunkedTransport,
|
||||
ChunkedTransportProtocolError,
|
||||
} from "@getpaseo/protocol/chunked-transport";
|
||||
import { type WorkspaceRuntimeStore } from "./workspace-runtime-store/index.js";
|
||||
import {
|
||||
createAgentMessageDelivery,
|
||||
@@ -124,8 +128,8 @@ import {
|
||||
MAX_PHYSICAL_SOCKET_BUFFERED_BYTES,
|
||||
outboundFrameByteLength,
|
||||
physicalSocketHasCapacity,
|
||||
sendBoundedPhysicalFrame,
|
||||
sendBoundedPhysicalFrameAndWait,
|
||||
sendBoundedPhysicalFrame,
|
||||
} from "./websocket/physical-socket.js";
|
||||
|
||||
const WS_CLOSE_DAEMON_AUTH_FAILED = 4401;
|
||||
@@ -440,6 +444,7 @@ function bufferFromWsData(data: Buffer | ArrayBuffer | Buffer[] | string): Buffe
|
||||
}
|
||||
|
||||
export interface WebSocketLike {
|
||||
maxFrameBytes?: number;
|
||||
readyState: number;
|
||||
bufferedAmount?: number;
|
||||
send: (
|
||||
@@ -1273,6 +1278,67 @@ export class VoiceAssistantWebSocketServer {
|
||||
return { sockets: writableSockets, payload, bytes: outboundFrameByteLength(payload) };
|
||||
}
|
||||
|
||||
private sendHelloResponse(ws: WebSocketLike, session: Session, hello: WSHelloMessage): void {
|
||||
// The peer learns our support from this ordinary response before receiving chunks.
|
||||
this.sendToClient(ws, this.createServerInfoMessage(session));
|
||||
this.chunkedSocket(ws).enabled = hello.capabilities?.[CLIENT_CAPS.binaryChunkingV1] === true;
|
||||
}
|
||||
|
||||
private readonly chunkedSockets = new WeakMap<WebSocketLike, ChunkedTransport>();
|
||||
|
||||
private chunkedSocket(ws: WebSocketLike): ChunkedTransport {
|
||||
let transport = this.chunkedSockets.get(ws);
|
||||
if (!transport) {
|
||||
transport = new ChunkedTransport({
|
||||
frameBytes: ws.maxFrameBytes ?? 32 * 1024 * 1024,
|
||||
onFailure: (err) =>
|
||||
this.closePhysicalSocket({
|
||||
ws,
|
||||
logMessage: "Closing physical WebSocket after outbound framing failure",
|
||||
logFields: { err },
|
||||
}),
|
||||
send: (frame, waitForDrain) => {
|
||||
const params = {
|
||||
socket: ws,
|
||||
frame,
|
||||
onHighWater: () => this.closeAtOutboundHighWater(ws),
|
||||
};
|
||||
if (waitForDrain)
|
||||
return sendBoundedPhysicalFrameAndWait(params).then((sent) => {
|
||||
if (!sent) throw new Error("Source is closed");
|
||||
return undefined;
|
||||
});
|
||||
if (!sendBoundedPhysicalFrame(params)) throw new Error("Source is closed");
|
||||
},
|
||||
deliver: (frame) =>
|
||||
this.handleRawMessage(ws, typeof frame === "string" ? frame : Buffer.from(frame)),
|
||||
});
|
||||
this.chunkedSockets.set(ws, transport);
|
||||
}
|
||||
return transport;
|
||||
}
|
||||
|
||||
private socketBufferedAmount(ws: WebSocketLike): number | null {
|
||||
const queued = this.chunkedSockets.get(ws)?.queuedByteLength ?? 0;
|
||||
if (typeof ws.bufferedAmount === "number") return ws.bufferedAmount + queued;
|
||||
return queued > 0 ? queued : null;
|
||||
}
|
||||
|
||||
private sendSocketFrame(
|
||||
ws: WebSocketLike,
|
||||
frame: string | Uint8Array,
|
||||
bytes: number,
|
||||
waitForDrain = false,
|
||||
): Promise<void> {
|
||||
const transport = this.chunkedSocket(ws);
|
||||
const admittedBytes = transport.isWriting
|
||||
? bytes
|
||||
: Math.min(bytes, ws.maxFrameBytes ?? 32 * 1024 * 1024);
|
||||
if (!this.ensureOutboundCapacity(ws, admittedBytes))
|
||||
return Promise.reject(new Error("Source is closed"));
|
||||
return transport.send(frame, { waitForDrain, byteLength: bytes });
|
||||
}
|
||||
|
||||
private async sendMessageToClientAndWait(
|
||||
ws: WebSocketLike,
|
||||
message: WSOutboundMessage,
|
||||
@@ -1280,13 +1346,7 @@ export class VoiceAssistantWebSocketServer {
|
||||
const prepared = this.prepareJsonDelivery([ws], message, true);
|
||||
if (!prepared) throw new Error("Source message delivery unavailable");
|
||||
try {
|
||||
const sent = await sendBoundedPhysicalFrameAndWait({
|
||||
socket: ws,
|
||||
frame: prepared.payload,
|
||||
frameBytes: prepared.bytes,
|
||||
onHighWater: () => this.closeAtOutboundHighWater(ws),
|
||||
});
|
||||
if (!sent) throw new Error("Source is closed");
|
||||
await this.sendSocketFrame(ws, prepared.payload, prepared.bytes, true);
|
||||
this.runtimeMetrics.recordOutboundMessage(message, ws.bufferedAmount);
|
||||
} catch (err) {
|
||||
this.logger.warn({ err }, "ws_send_failed");
|
||||
@@ -1313,14 +1373,7 @@ export class VoiceAssistantWebSocketServer {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const sent = await sendBoundedPhysicalFrameAndWait({
|
||||
socket: ws,
|
||||
frame,
|
||||
onHighWater: () => this.closeAtOutboundHighWater(ws),
|
||||
});
|
||||
if (!sent) {
|
||||
throw new Error("Physical WebSocket is not open");
|
||||
}
|
||||
await this.sendSocketFrame(ws, frame, frame.byteLength, true);
|
||||
this.runtimeMetrics.recordOutboundBinaryFrame(ws.bufferedAmount);
|
||||
} catch (err) {
|
||||
this.logger.warn({ err }, "ws_send_failed");
|
||||
@@ -1335,13 +1388,9 @@ export class VoiceAssistantWebSocketServer {
|
||||
recordSent: () => void,
|
||||
): void {
|
||||
try {
|
||||
const sent = sendBoundedPhysicalFrame({
|
||||
socket: ws,
|
||||
frame,
|
||||
frameBytes,
|
||||
onHighWater: () => this.closeAtOutboundHighWater(ws),
|
||||
void this.sendSocketFrame(ws, frame, frameBytes).then(recordSent, (err) => {
|
||||
this.logger.warn({ err }, "ws_send_failed");
|
||||
});
|
||||
if (sent) recordSent();
|
||||
} catch (err) {
|
||||
this.logger.warn({ err }, "ws_send_failed");
|
||||
}
|
||||
@@ -1349,7 +1398,13 @@ export class VoiceAssistantWebSocketServer {
|
||||
|
||||
private ensureOutboundCapacity(ws: WebSocketLike, frameBytes: number): boolean {
|
||||
if (ws.readyState !== 1) return false;
|
||||
if (physicalSocketHasCapacity(ws, frameBytes)) return true;
|
||||
if (
|
||||
physicalSocketHasCapacity(
|
||||
{ bufferedAmount: this.socketBufferedAmount(ws) ?? undefined },
|
||||
frameBytes,
|
||||
)
|
||||
)
|
||||
return true;
|
||||
|
||||
this.closeAtOutboundHighWater(ws);
|
||||
return false;
|
||||
@@ -1522,16 +1577,15 @@ export class VoiceAssistantWebSocketServer {
|
||||
if (!connection) {
|
||||
return null;
|
||||
}
|
||||
if (source) return (source as WebSocketLike).bufferedAmount ?? null;
|
||||
if (source) return this.socketBufferedAmount(source as WebSocketLike);
|
||||
// Relay-attached sockets are a WebSocketLike that doesn't expose
|
||||
// bufferedAmount. Return null when no socket gives a signal so the
|
||||
// terminal fallback can't mistake "no signal" for "client keeping up";
|
||||
// a direct ws reports its real buffered bytes (0 when drained).
|
||||
let maxBuffered: number | null = null;
|
||||
for (const socket of connection.sockets) {
|
||||
if (typeof socket.bufferedAmount === "number") {
|
||||
maxBuffered = Math.max(maxBuffered ?? 0, socket.bufferedAmount);
|
||||
}
|
||||
const buffered = this.socketBufferedAmount(socket);
|
||||
if (buffered !== null) maxBuffered = Math.max(maxBuffered ?? 0, buffered);
|
||||
}
|
||||
return maxBuffered;
|
||||
},
|
||||
@@ -1568,8 +1622,9 @@ export class VoiceAssistantWebSocketServer {
|
||||
let total = 0;
|
||||
for (const socket of this.sessions.keys()) {
|
||||
if (socket instanceof VirtualHostSocket) continue;
|
||||
if (typeof socket.bufferedAmount !== "number") return null;
|
||||
total += socket.bufferedAmount;
|
||||
const buffered = this.socketBufferedAmount(socket);
|
||||
if (buffered === null) return null;
|
||||
total += buffered;
|
||||
}
|
||||
return total;
|
||||
},
|
||||
@@ -1770,7 +1825,7 @@ export class VoiceAssistantWebSocketServer {
|
||||
this.externalSessionsByKey.set(sessionKey, connection);
|
||||
}
|
||||
pending.identity.sessionId = connection.session.getSessionId();
|
||||
this.sendToClient(ws, this.createServerInfoMessage(connection.session));
|
||||
this.sendHelloResponse(ws, connection.session, message);
|
||||
connection.connectionLogger.info(
|
||||
{
|
||||
...toConnectionLogFields(pending.identity),
|
||||
@@ -1906,7 +1961,7 @@ export class VoiceAssistantWebSocketServer {
|
||||
existing.sockets.add(ws);
|
||||
this.sessions.set(ws, existing);
|
||||
pending.identity.sessionId = existing.session.getSessionId();
|
||||
this.sendToClient(ws, this.createServerInfoMessage(existing.session));
|
||||
this.sendHelloResponse(ws, existing.session, message);
|
||||
pending.connectionLogger.info(
|
||||
{
|
||||
...toConnectionLogFields(pending.identity),
|
||||
@@ -1952,6 +2007,7 @@ export class VoiceAssistantWebSocketServer {
|
||||
usageSetting: true,
|
||||
usageSources: true,
|
||||
ownedSubscriptions: true,
|
||||
binaryChunkingV1: true,
|
||||
agentRequestReceipts: true,
|
||||
workspaceRequestReceipts: true,
|
||||
creationLifecycle: true,
|
||||
@@ -2165,10 +2221,16 @@ export class VoiceAssistantWebSocketServer {
|
||||
private bindSocketHandlers(ws: WebSocketLike): void {
|
||||
ws.on("message", (...args: unknown[]) => {
|
||||
const data = args[0] as Buffer | ArrayBuffer | Buffer[] | string;
|
||||
this.handleRawMessage(ws, data);
|
||||
try {
|
||||
this.chunkedSocket(ws).receive(typeof data === "string" ? data : bufferFromWsData(data));
|
||||
} catch (error) {
|
||||
if (!(error instanceof ChunkedTransportProtocolError)) throw error;
|
||||
ws.close(1002, "Invalid chunked transport");
|
||||
}
|
||||
});
|
||||
|
||||
ws.on("close", async (...args: unknown[]) => {
|
||||
this.chunkedSockets.get(ws)?.close();
|
||||
const code = args[0];
|
||||
const reason = args[1];
|
||||
await this.detachSocket(ws, {
|
||||
@@ -2568,7 +2630,7 @@ export class VoiceAssistantWebSocketServer {
|
||||
identity: JSON.stringify([connection.principalId, connection.clientId]),
|
||||
permissions: () => connection.session.getPermissions(),
|
||||
routed: () => connection.session.isRouted(),
|
||||
bufferedAmount: () => ws.bufferedAmount ?? null,
|
||||
bufferedAmount: () => this.socketBufferedAmount(ws),
|
||||
delivery: connection.session.delivery,
|
||||
admit: (socket, permissions) =>
|
||||
this.attachSocket(socket, undefined, undefined, false, {
|
||||
|
||||
@@ -140,3 +140,74 @@ test("encrypted sockets do not double-count bytes already buffered by the transp
|
||||
|
||||
expect(socket.bufferedAmount).toBe(payload.byteLength + 40);
|
||||
});
|
||||
|
||||
test("50 MiB logical text and routed binary cross encrypted relay frames in both directions", async () => {
|
||||
const { ChunkedTransport } = await import("@getpaseo/protocol/chunked-transport");
|
||||
const { openClientSecureChannel, openDaemonSecureChannel } = await import("@getpaseo/relay/e2ee");
|
||||
const { generateKeyPair, exportPublicKey, maxSecuredPlaintextByteLength } =
|
||||
await import("@getpaseo/relay");
|
||||
const { encodeHostRouteFrame, decodeHostRouteFrame } =
|
||||
await import("@getpaseo/protocol/binary-frames/index");
|
||||
type Transport = import("@getpaseo/relay/e2ee").Transport;
|
||||
const limit = 32 * 1024 * 1024;
|
||||
const wireSizes: number[] = [];
|
||||
const endpoint = (): Transport => ({
|
||||
send: () => {},
|
||||
close: () => {},
|
||||
onmessage: null,
|
||||
onclose: null,
|
||||
onerror: null,
|
||||
});
|
||||
const left = endpoint();
|
||||
const right = endpoint();
|
||||
const wire = (destination: Transport) => (data: string | ArrayBuffer) => {
|
||||
const size = typeof data === "string" ? Buffer.byteLength(data) : data.byteLength;
|
||||
expect(size).toBeLessThanOrEqual(limit);
|
||||
wireSizes.push(size);
|
||||
queueMicrotask(() => destination.onmessage?.({ data, isBinary: typeof data !== "string" }));
|
||||
};
|
||||
left.send = wire(right);
|
||||
right.send = wire(left);
|
||||
const key = generateKeyPair();
|
||||
const delivered: (string | Uint8Array)[][] = [[], []];
|
||||
let receivers: InstanceType<typeof ChunkedTransport>[] = [];
|
||||
const incoming = (index: number) => (data: string | ArrayBuffer) =>
|
||||
receivers[index]!.receive(typeof data === "string" ? data : new Uint8Array(data));
|
||||
const daemonPromise = openDaemonSecureChannel(right, {
|
||||
daemonKeyPair: key,
|
||||
events: { onmessage: incoming(1) },
|
||||
});
|
||||
const client = await openClientSecureChannel(left, {
|
||||
daemonPublicKeyB64: exportPublicKey(key.publicKey),
|
||||
events: { onmessage: incoming(0) },
|
||||
});
|
||||
const daemon = await daemonPromise;
|
||||
receivers = [client, daemon].map(
|
||||
(channel, index) =>
|
||||
new ChunkedTransport({
|
||||
frameBytes: maxSecuredPlaintextByteLength(limit),
|
||||
send: (frame) => channel.send(typeof frame === "string" ? frame : frame.slice().buffer),
|
||||
deliver: (frame) => {
|
||||
delivered[index]!.push(frame);
|
||||
},
|
||||
}),
|
||||
);
|
||||
for (const receiver of receivers) receiver.enabled = true;
|
||||
const text = "x".repeat(50 * 1024 * 1024);
|
||||
wireSizes.length = 0;
|
||||
await receivers[1]!.send(text);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(delivered[0]).toEqual([text]);
|
||||
expect(wireSizes.length).toBeGreaterThan(1);
|
||||
const routed = encodeHostRouteFrame("route", new Uint8Array(50 * 1024 * 1024).fill(7));
|
||||
await receivers[0]!.send(routed);
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(delivered[1]).toHaveLength(1);
|
||||
const received = decodeHostRouteFrame(delivered[1]![0] as Uint8Array)!;
|
||||
expect(received.routeId).toBe("route");
|
||||
expect(
|
||||
Buffer.from(received.payload).equals(Buffer.from(new Uint8Array(50 * 1024 * 1024).fill(7))),
|
||||
).toBe(true);
|
||||
client.close();
|
||||
daemon.close();
|
||||
}, 30_000);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { maxSecuredPlaintextByteLength } from "@getpaseo/relay";
|
||||
import { MAX_RELAY_WEBSOCKET_MESSAGE_BYTES } from "@getpaseo/protocol/websocket-message-limit";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { MAX_PHYSICAL_SOCKET_BUFFERED_BYTES } from "./physical-socket.js";
|
||||
|
||||
@@ -8,6 +10,7 @@ export interface EncryptedRelayChannel {
|
||||
}
|
||||
|
||||
export interface EncryptedRelaySocket {
|
||||
readonly maxFrameBytes: number;
|
||||
readonly readyState: number;
|
||||
readonly bufferedAmount: number;
|
||||
send: (data: string | Uint8Array | ArrayBuffer) => void | Promise<void>;
|
||||
@@ -43,6 +46,7 @@ export function createEncryptedRelaySocket(params: {
|
||||
});
|
||||
|
||||
return {
|
||||
maxFrameBytes: maxSecuredPlaintextByteLength(MAX_RELAY_WEBSOCKET_MESSAGE_BYTES),
|
||||
get readyState() {
|
||||
return readyState;
|
||||
},
|
||||
|
||||
@@ -120,3 +120,96 @@ test("the awaitable physical send rejects callback errors", async () => {
|
||||
}),
|
||||
).rejects.toThrow("send failed");
|
||||
});
|
||||
|
||||
test("ordinary frames expose a paused real socket's backlog and terminate at the physical backstop", async () => {
|
||||
const { once } = await import("node:events");
|
||||
const { WebSocket, WebSocketServer } = await import("ws");
|
||||
const { ChunkedTransport } = await import("@getpaseo/protocol/chunked-transport");
|
||||
const server = new WebSocketServer({ host: "127.0.0.1", port: 0 });
|
||||
await once(server, "listening");
|
||||
const connected = once(server, "connection");
|
||||
const address = server.address();
|
||||
if (typeof address === "string" || !address) throw new Error("Missing server address");
|
||||
const client = new WebSocket(`ws://127.0.0.1:${address.port}`);
|
||||
await once(client, "open");
|
||||
const [socket] = (await connected) as [InstanceType<typeof WebSocket>];
|
||||
const tcp = (socket as unknown as { _socket: import("node:net").Socket })._socket;
|
||||
let terminated = false;
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 32 * 1024 * 1024,
|
||||
deliver: () => {},
|
||||
send: (frame, wait) => {
|
||||
const options = {
|
||||
socket,
|
||||
frame,
|
||||
onHighWater: () => {
|
||||
terminated = true;
|
||||
socket.terminate();
|
||||
},
|
||||
};
|
||||
if (wait) return sendBoundedPhysicalFrameAndWait(options).then(() => undefined);
|
||||
sendBoundedPhysicalFrame(options);
|
||||
},
|
||||
});
|
||||
try {
|
||||
client.pause();
|
||||
tcp.cork();
|
||||
const frame = new Uint8Array(256 * 1024);
|
||||
for (let count = 0; count < 200; count++) await transport.send(frame);
|
||||
expect(socket.bufferedAmount).toBeGreaterThan(4 * 1024 * 1024);
|
||||
expect(transport.queuedByteLength).toBe(0);
|
||||
for (let count = 0; count < 100; count++) {
|
||||
if (terminated) break;
|
||||
await transport.send(frame);
|
||||
}
|
||||
expect(terminated).toBe(true);
|
||||
} finally {
|
||||
transport.close();
|
||||
client.terminate();
|
||||
socket.terminate();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("an awaited framing failure terminates its real socket immediately", async () => {
|
||||
const { once } = await import("node:events");
|
||||
const { WebSocket, WebSocketServer } = await import("ws");
|
||||
const { ChunkedTransport } = await import("@getpaseo/protocol/chunked-transport");
|
||||
const server = new WebSocketServer({ host: "127.0.0.1", port: 0 });
|
||||
await once(server, "listening");
|
||||
const connected = once(server, "connection");
|
||||
const address = server.address();
|
||||
if (typeof address === "string" || !address) throw new Error("Missing server address");
|
||||
const client = new WebSocket(`ws://127.0.0.1:${address.port}`);
|
||||
await once(client, "open");
|
||||
const [socket] = (await connected) as [InstanceType<typeof WebSocket>];
|
||||
const tcp = (socket as unknown as { _socket: import("node:net").Socket })._socket;
|
||||
const transport = new ChunkedTransport({
|
||||
frameBytes: 32 * 1024 * 1024,
|
||||
deliver: () => {},
|
||||
onFailure: () => socket.terminate(),
|
||||
send: (frame) =>
|
||||
sendBoundedPhysicalFrameAndWait({
|
||||
socket,
|
||||
frame,
|
||||
onHighWater: () => socket.terminate(),
|
||||
}).then(() => undefined),
|
||||
});
|
||||
transport.enabled = true;
|
||||
try {
|
||||
client.pause();
|
||||
tcp.cork();
|
||||
const closed = once(socket, "close");
|
||||
const first = transport.send(new Uint8Array(100 * 1024 * 1024), { waitForDrain: true });
|
||||
const overflow = transport.send(new Uint8Array([1]), { waitForDrain: true });
|
||||
const settled = Promise.allSettled([first, overflow]);
|
||||
await closed;
|
||||
expect(socket.readyState).toBe(WebSocket.CLOSED);
|
||||
expect((await settled).map((result) => result.status)).toEqual(["rejected", "rejected"]);
|
||||
} finally {
|
||||
transport.close();
|
||||
client.terminate();
|
||||
socket.terminate();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,45 +1,27 @@
|
||||
import { execFile, execFileSync } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { build } from "esbuild";
|
||||
import {
|
||||
createTestRegistries,
|
||||
createTestLogger,
|
||||
deferred,
|
||||
} from "./test-fixtures/reconciliation-registries.js";
|
||||
import { measureMainThreadWork } from "../test-utils/main-thread-work.js";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import type { ProjectCheckoutLitePayload } from "@getpaseo/protocol/messages";
|
||||
import type pino from "pino";
|
||||
import { afterAll, beforeAll, afterEach, describe, expect, test } from "vitest";
|
||||
import { afterEach, describe, expect, test } from "vitest";
|
||||
import {
|
||||
createPersistedProjectRecord,
|
||||
createPersistedWorkspaceRecord,
|
||||
} from "./workspace-registry.js";
|
||||
import type {
|
||||
PersistedProjectRecord,
|
||||
PersistedWorkspaceRecord,
|
||||
ProjectRegistry,
|
||||
WorkspaceRegistry,
|
||||
} from "./workspace-registry.js";
|
||||
import {
|
||||
type ReconciliationChange,
|
||||
WorkspaceReconciliationService,
|
||||
} from "./workspace-reconciliation-service.js";
|
||||
import { deriveProjectKey } from "./project-key.js";
|
||||
|
||||
let probeDirectory: string;
|
||||
let probePath: string;
|
||||
beforeAll(async () => {
|
||||
probeDirectory = mkdtempSync(path.join(tmpdir(), "reconciliation-probe-"));
|
||||
probePath = path.join(probeDirectory, "reconciliation.mjs");
|
||||
await build({
|
||||
entryPoints: [
|
||||
fileURLToPath(new URL("./test-fixtures/reconciliation-responsiveness.ts", import.meta.url)),
|
||||
],
|
||||
outfile: probePath,
|
||||
bundle: true,
|
||||
platform: "node",
|
||||
format: "esm",
|
||||
});
|
||||
});
|
||||
afterAll(() => rmSync(probeDirectory, { recursive: true, force: true }));
|
||||
|
||||
function canonicalLocalProjectKey(rootPath: string): string {
|
||||
return deriveProjectKey({
|
||||
rootPath,
|
||||
@@ -49,6 +31,107 @@ function canonicalLocalProjectKey(rootPath: string): string {
|
||||
});
|
||||
}
|
||||
|
||||
function createTestRegistries() {
|
||||
let archiveGate: {
|
||||
started: ReturnType<typeof deferred>;
|
||||
release: ReturnType<typeof deferred>;
|
||||
} | null = null;
|
||||
const holdNextArchive = () => {
|
||||
archiveGate = { started: deferred(), release: deferred() };
|
||||
return archiveGate;
|
||||
};
|
||||
const projects = new Map<string, PersistedProjectRecord>();
|
||||
const workspaces = new Map<string, PersistedWorkspaceRecord>();
|
||||
|
||||
const projectRegistry: ProjectRegistry = {
|
||||
initialize: async () => {},
|
||||
existsOnDisk: async () => true,
|
||||
list: async () => Array.from(projects.values()),
|
||||
get: async (id: string) => projects.get(id) ?? null,
|
||||
getOrCreateActiveByRoot: async (input) => {
|
||||
const existing = Array.from(projects.values()).find(
|
||||
(project) => !project.archivedAt && project.rootPath === input.rootPath,
|
||||
);
|
||||
if (existing) return existing;
|
||||
const record = createPersistedProjectRecord({
|
||||
projectId: `prj_${projects.size}`,
|
||||
rootPath: input.rootPath,
|
||||
kind: input.kind,
|
||||
displayName: input.displayName,
|
||||
createdAt: input.timestamp,
|
||||
updatedAt: input.timestamp,
|
||||
});
|
||||
projects.set(record.projectId, record);
|
||||
return record;
|
||||
},
|
||||
update: async (id, updater) => {
|
||||
const current = projects.get(id);
|
||||
if (!current) return null;
|
||||
const updated = updater(current);
|
||||
projects.set(id, updated);
|
||||
return updated;
|
||||
},
|
||||
upsert: async (record: PersistedProjectRecord) => {
|
||||
projects.set(record.projectId, record);
|
||||
},
|
||||
archive: async (id: string, archivedAt: string) => {
|
||||
const existing = projects.get(id);
|
||||
if (existing) {
|
||||
projects.set(id, { ...existing, archivedAt, updatedAt: archivedAt });
|
||||
}
|
||||
},
|
||||
remove: async (id: string) => {
|
||||
projects.delete(id);
|
||||
},
|
||||
};
|
||||
|
||||
const workspaceRegistry: WorkspaceRegistry = {
|
||||
initialize: async () => {},
|
||||
existsOnDisk: async () => true,
|
||||
list: async () => Array.from(workspaces.values()),
|
||||
get: async (id: string) => workspaces.get(id) ?? null,
|
||||
update: async (id, updater) => {
|
||||
const existing = workspaces.get(id);
|
||||
if (!existing) return null;
|
||||
const updated = updater(existing);
|
||||
workspaces.set(id, updated);
|
||||
return updated;
|
||||
},
|
||||
upsert: async (record: PersistedWorkspaceRecord) => {
|
||||
workspaces.set(record.workspaceId, record);
|
||||
},
|
||||
archive: async (id: string, archivedAt: string) => {
|
||||
const gate = archiveGate;
|
||||
archiveGate = null;
|
||||
if (gate) {
|
||||
gate.started.resolve();
|
||||
await gate.release.promise;
|
||||
}
|
||||
const existing = workspaces.get(id);
|
||||
if (!existing || existing.archivedAt) return false;
|
||||
workspaces.set(id, { ...existing, archivedAt, updatedAt: archivedAt });
|
||||
return true;
|
||||
},
|
||||
remove: async (id: string) => {
|
||||
workspaces.delete(id);
|
||||
},
|
||||
};
|
||||
|
||||
return { projects, workspaces, projectRegistry, workspaceRegistry, holdNextArchive };
|
||||
}
|
||||
|
||||
function createTestLogger() {
|
||||
const logger = {
|
||||
child: () => logger,
|
||||
trace: () => undefined,
|
||||
debug: () => undefined,
|
||||
info: () => undefined,
|
||||
warn: () => undefined,
|
||||
error: () => undefined,
|
||||
};
|
||||
return logger as unknown as pino.Logger;
|
||||
}
|
||||
|
||||
interface CapturedLogRecord {
|
||||
message: string;
|
||||
payload: unknown;
|
||||
@@ -127,6 +210,14 @@ function createCheckout(
|
||||
};
|
||||
}
|
||||
|
||||
function deferred(): { promise: Promise<void>; resolve(): void } {
|
||||
let resolve!: () => void;
|
||||
const promise = new Promise<void>((accept) => {
|
||||
resolve = accept;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
class TestCheckouts {
|
||||
readonly reads: string[] = [];
|
||||
private readonly checkouts = new Map<string, ProjectCheckoutLitePayload>();
|
||||
@@ -535,30 +626,53 @@ describe("WorkspaceReconciliationService", () => {
|
||||
test("keeps the event loop responsive while inspecting 20000 real workspace paths", async () => {
|
||||
const rootPath = realpathSync(mkdtempSync(path.join(tmpdir(), "reconcile-many-")));
|
||||
tempDirs.push(rootPath);
|
||||
// Unrelated runner work must not consume the reconciliation budget.
|
||||
const unrelatedWork = new Promise<void>((resolve) => {
|
||||
setImmediate(() => {
|
||||
const cpu = Reflect.get(process, "threadCpuUsage") as typeof process.cpuUsage;
|
||||
const start = cpu();
|
||||
while (cpu(start).user + cpu(start).system < 80_000) {}
|
||||
resolve();
|
||||
});
|
||||
const { projects, workspaces, projectRegistry, workspaceRegistry } = createTestRegistries();
|
||||
projects.set(
|
||||
"p1",
|
||||
createPersistedProjectRecord({
|
||||
projectId: "p1",
|
||||
rootPath,
|
||||
projectKey: canonicalLocalProjectKey(rootPath),
|
||||
kind: "non_git",
|
||||
displayName: "project",
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
}),
|
||||
);
|
||||
for (let i = 0; i < 20000; i++) {
|
||||
const workspaceId = `w${i}`;
|
||||
workspaces.set(
|
||||
workspaceId,
|
||||
createPersistedWorkspaceRecord({
|
||||
workspaceId,
|
||||
projectId: "p1",
|
||||
cwd: path.join(rootPath, workspaceId),
|
||||
kind: "directory",
|
||||
displayName: workspaceId,
|
||||
createdAt: timestamp,
|
||||
updatedAt: timestamp,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const service = new WorkspaceReconciliationService({
|
||||
projectRegistry,
|
||||
workspaceRegistry,
|
||||
logger: createTestLogger(),
|
||||
});
|
||||
const { stdout, stderr } = await promisify(execFile)(process.execPath, [probePath, rootPath], {
|
||||
timeout: 25_000,
|
||||
});
|
||||
process.stderr.write(stderr);
|
||||
const { changesApplied, blockingMs, filesystemRequests, allArchived } = JSON.parse(stdout) as {
|
||||
changesApplied: number;
|
||||
blockingMs: number;
|
||||
filesystemRequests: number;
|
||||
allArchived: boolean;
|
||||
};
|
||||
await unrelatedWork;
|
||||
expect(blockingMs).toBeLessThan(20);
|
||||
expect(filesystemRequests).toBe(20001);
|
||||
expect(changesApplied).toBe(20000);
|
||||
expect(allArchived).toBe(true);
|
||||
try {
|
||||
const { result, blockingMs, filesystemRequests } = await measureMainThreadWork(
|
||||
"workspace reconciliation main-thread work",
|
||||
() => service.runOnce(),
|
||||
);
|
||||
expect(blockingMs).toBeLessThan(20);
|
||||
expect(filesystemRequests).toBe(20001);
|
||||
expect(result.changesApplied).toHaveLength(20000);
|
||||
expect([...workspaces.values()].every((workspace) => workspace.archivedAt !== null)).toBe(
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
service.dispose();
|
||||
}
|
||||
}, 30000);
|
||||
|
||||
test("preserves workspace archival that lands during boot reconciliation", async () => {
|
||||
|
||||
Reference in New Issue
Block a user