mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-20 23:35:27 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d18d43787c | ||
|
|
a60ceb0a72 | ||
|
|
4910508d00 | ||
|
|
c89e083260 | ||
|
|
273af797f2 | ||
|
|
e0009dfbad |
@@ -709,6 +709,13 @@ class FakeDaemon {
|
||||
);
|
||||
}
|
||||
|
||||
socketFailsOnHello(): void {
|
||||
this.transport.send = () => {
|
||||
throw new Error("WebSocket not open");
|
||||
};
|
||||
this.onOpen();
|
||||
}
|
||||
|
||||
daemonAnswersPingsAfter(delay: "fast" | `${number}s`): void {
|
||||
this.pongMode = {
|
||||
kind: "answer",
|
||||
@@ -2683,6 +2690,211 @@ test("foreground tears down a silently dead socket after two fresh five-second w
|
||||
expect(second.closesFromClient()).toEqual([]);
|
||||
});
|
||||
|
||||
test("foreground replaces a stale connect attempt and retries at the base delay if it fails", async () => {
|
||||
useHeartbeatClock();
|
||||
const stalled = new FakeDaemon();
|
||||
const replacement = new FakeDaemon();
|
||||
const healthy = new FakeDaemon();
|
||||
const { client, attempts } = connectThroughReplacements("foreground-stalled", [
|
||||
stalled,
|
||||
replacement,
|
||||
healthy,
|
||||
]);
|
||||
const connection = client.connect();
|
||||
void connection.catch(() => undefined);
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
expect(client.getConnectionState().status).toBe("connecting");
|
||||
|
||||
client.ensureConnected({ verify: true });
|
||||
expect(attempts()).toBe(2);
|
||||
expect(stalled.closesFromClient()).toHaveLength(1);
|
||||
|
||||
replacement.daemonClosesWith("network still waking");
|
||||
await vi.advanceTimersByTimeAsync(1_499);
|
||||
expect(attempts()).toBe(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(attempts()).toBe(3);
|
||||
healthy.openConnection();
|
||||
await connection;
|
||||
expect(client.getConnectionState().status).toBe("connected");
|
||||
});
|
||||
|
||||
test("foreground preserves RPCs waiting for a pending handshake", async () => {
|
||||
useHeartbeatClock();
|
||||
const stalled = new FakeDaemon();
|
||||
const replacement = new FakeDaemon();
|
||||
const { client } = connectThroughReplacements("foreground-queued", [stalled, replacement]);
|
||||
const connection = client.connect();
|
||||
void connection.catch(() => undefined);
|
||||
const ping = client.ping({ requestId: "queued-ping", timeoutMs: 10_000 });
|
||||
void ping.catch(() => undefined);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
client.ensureConnected({ verify: true });
|
||||
replacement.openConnection();
|
||||
|
||||
await connection;
|
||||
await expect(ping).resolves.toMatchObject({ requestId: "queued-ping" });
|
||||
});
|
||||
|
||||
test("foreground shares a pending credential lookup with the current connection attempt", async () => {
|
||||
useHeartbeatClock();
|
||||
const stalled = new FakeDaemon();
|
||||
const replacement = new FakeDaemon();
|
||||
const daemons = [stalled, replacement];
|
||||
const credentials: Array<(value: string) => void> = [];
|
||||
let attempts = 0;
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "foreground-credential-lookup",
|
||||
localCredential: () => new Promise((resolve) => credentials.push(resolve)),
|
||||
transportFactory: () => daemons[attempts++]!.transport,
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
const connection = client.connect();
|
||||
void connection.catch(() => undefined);
|
||||
client.ensureConnected({ verify: true });
|
||||
expect(credentials).toHaveLength(1);
|
||||
credentials[0]!("initial-credential");
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
expect(attempts).toBe(1);
|
||||
|
||||
client.ensureConnected({ verify: true });
|
||||
client.ensureConnected({ verify: true });
|
||||
expect(credentials).toHaveLength(2);
|
||||
expect(stalled.closesFromClient()).toHaveLength(1);
|
||||
credentials[1]!("replacement-credential");
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(attempts).toBe(2);
|
||||
replacement.openConnection();
|
||||
|
||||
await connection;
|
||||
expect(client.getConnectionState()).toEqual({ status: "connected" });
|
||||
});
|
||||
|
||||
test("foreground replaces a stalled credential lookup and ignores its late result", async () => {
|
||||
useHeartbeatClock();
|
||||
const daemon = new FakeDaemon();
|
||||
const credentials: Array<(value: string) => void> = [];
|
||||
let attempts = 0;
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "foreground-stalled-credentials",
|
||||
connectTimeoutMs: 4_000,
|
||||
localCredential: () => new Promise((resolve) => credentials.push(resolve)),
|
||||
transportFactory: () => {
|
||||
attempts += 1;
|
||||
return daemon.transport;
|
||||
},
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
const connection = client.connect();
|
||||
void connection.catch(() => undefined);
|
||||
await vi.advanceTimersByTimeAsync(3_100);
|
||||
client.ensureConnected({ verify: true });
|
||||
expect(credentials).toHaveLength(2);
|
||||
credentials[1]!("replacement-credential");
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
daemon.openConnection();
|
||||
await connection;
|
||||
|
||||
credentials[0]!("obsolete-credential");
|
||||
await vi.advanceTimersByTimeAsync(2_000);
|
||||
expect(attempts).toBe(1);
|
||||
expect(client.getConnectionState()).toEqual({ status: "connected" });
|
||||
});
|
||||
|
||||
test("connection timeout includes credential lookup", async () => {
|
||||
useHeartbeatClock();
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId: "credential-lookup-timeout",
|
||||
connectTimeoutMs: 4_000,
|
||||
reconnect: { enabled: false },
|
||||
localCredential: () => new Promise(() => {}),
|
||||
});
|
||||
clients.push(client);
|
||||
|
||||
const connection = client.connect();
|
||||
const outcome = connection.catch((error: unknown) => error);
|
||||
await vi.advanceTimersByTimeAsync(3_999);
|
||||
expect(client.getConnectionState().status).toBe("connecting");
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(client.getConnectionState().status).toBe("disconnected");
|
||||
expect(await outcome).toMatchObject({ message: "Connection timed out" });
|
||||
});
|
||||
|
||||
test("a failed hello retires the deadline of the foreground replacement", async () => {
|
||||
useHeartbeatClock();
|
||||
const stalled = new FakeDaemon();
|
||||
const closing = new FakeDaemon();
|
||||
const healthy = new FakeDaemon();
|
||||
const { client, attempts } = connectThroughReplacements(
|
||||
"foreground-failed-hello",
|
||||
[stalled, closing, healthy],
|
||||
{ connectTimeoutMs: 4_000 },
|
||||
);
|
||||
const connection = client.connect();
|
||||
void connection.catch(() => undefined);
|
||||
await vi.advanceTimersByTimeAsync(3_100);
|
||||
client.ensureConnected({ verify: true });
|
||||
expect(attempts()).toBe(2);
|
||||
await vi.advanceTimersByTimeAsync(3_900);
|
||||
closing.socketFailsOnHello();
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_499);
|
||||
expect(client.lastError).toBe("WebSocket not open");
|
||||
expect(attempts()).toBe(2);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(attempts()).toBe(3);
|
||||
healthy.openConnection();
|
||||
await connection;
|
||||
});
|
||||
|
||||
test("foreground keeps a connect attempt that just started", async () => {
|
||||
useHeartbeatClock();
|
||||
const pending = new FakeDaemon();
|
||||
const { client, attempts } = connectThroughReplacements("foreground-fresh", [pending]);
|
||||
const connection = client.connect();
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
|
||||
client.ensureConnected({ verify: true });
|
||||
expect(attempts()).toBe(1);
|
||||
expect(pending.closesFromClient()).toEqual([]);
|
||||
|
||||
pending.openConnection();
|
||||
await connection;
|
||||
expect(client.getConnectionState().status).toBe("connected");
|
||||
});
|
||||
|
||||
test("foreground resets the retry delay after failed connections in the background", async () => {
|
||||
useHeartbeatClock();
|
||||
const daemons = Array.from({ length: 7 }, () => new FakeDaemon());
|
||||
const { client, attempts } = connectThroughReplacements("foreground-backoff", daemons);
|
||||
const connection = client.connect();
|
||||
daemons[0]!.openConnection();
|
||||
await connection;
|
||||
|
||||
daemons[0]!.daemonClosesWith("network suspended");
|
||||
for (let retry = 0; retry < 4; retry += 1) {
|
||||
await vi.advanceTimersByTimeAsync(1_500 * 2 ** retry);
|
||||
expect(attempts()).toBe(retry + 2);
|
||||
daemons[retry + 1]!.daemonClosesWith("network suspended");
|
||||
}
|
||||
|
||||
client.ensureConnected({ verify: true });
|
||||
expect(attempts()).toBe(6);
|
||||
daemons[5]!.daemonClosesWith("network still waking");
|
||||
await vi.advanceTimersByTimeAsync(1_499);
|
||||
expect(attempts()).toBe(6);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(attempts()).toBe(7);
|
||||
daemons[6]!.openConnection();
|
||||
expect(client.getConnectionState().status).toBe("connected");
|
||||
});
|
||||
|
||||
test("foreground verification preserves a healthy socket and deduplicates simultaneous checks", async () => {
|
||||
useHeartbeatClock();
|
||||
const daemon = new FakeDaemon();
|
||||
@@ -2817,14 +3029,14 @@ test("chunk progress beyond the daemon lease keeps sending heartbeat pings", asy
|
||||
function connectThroughReplacements(
|
||||
clientId: string,
|
||||
daemons: FakeDaemon[],
|
||||
options?: { reconnect?: { enabled: boolean } },
|
||||
options?: { reconnect?: { enabled: boolean }; connectTimeoutMs?: number },
|
||||
): { client: DaemonClient; attempts: () => number } {
|
||||
let attempts = 0;
|
||||
const client = new DaemonClient({
|
||||
url: "ws://test",
|
||||
clientId,
|
||||
logger: noopLogger,
|
||||
...(options?.reconnect ? { reconnect: options.reconnect } : {}),
|
||||
...options,
|
||||
transportFactory: () => {
|
||||
const daemon = daemons[Math.min(attempts, daemons.length - 1)]!;
|
||||
attempts += 1;
|
||||
|
||||
@@ -1159,8 +1159,14 @@ function toTimeoutError(error: unknown, label: string, timeoutMs: number): Error
|
||||
return error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
|
||||
interface ConnectionAttempt {
|
||||
startedAt: number;
|
||||
timeout: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
const DEFAULT_RECONNECT_BASE_DELAY_MS = 1500;
|
||||
const DEFAULT_RECONNECT_MAX_DELAY_MS = 30000;
|
||||
const FOREGROUND_STALE_CONNECT_MS = 3_000;
|
||||
const DEFAULT_SESSION_RPC_TIMEOUT_MS = 60_000;
|
||||
const PUSH_TOKEN_REVOCATION_TIMEOUT_MS = 2_000;
|
||||
const DEFAULT_CONNECT_TIMEOUT_MS = 15_000;
|
||||
@@ -1455,7 +1461,7 @@ export class DaemonClient {
|
||||
private checkoutStatusInFlight: Map<string, Promise<CheckoutStatusPayload>> = new Map();
|
||||
private connectionListeners: Set<(status: ConnectionState) => void> = new Set();
|
||||
private reconnectTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
private connectTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
private connectionAttempt: ConnectionAttempt | null = null;
|
||||
private pendingGenericTransportErrorTimeout: ReturnType<typeof setTimeout> | null = null;
|
||||
private reconnectAttempt = 0;
|
||||
private traceConnectionAttempt = 0;
|
||||
@@ -1579,16 +1585,16 @@ export class DaemonClient {
|
||||
}
|
||||
|
||||
this.shouldReconnect = true;
|
||||
this.connectPromise = new Promise((resolve, reject) => {
|
||||
const connection = new Promise<void>((resolve, reject) => {
|
||||
this.connectResolve = resolve;
|
||||
this.connectReject = reject;
|
||||
this.attemptConnect();
|
||||
});
|
||||
|
||||
return this.connectPromise;
|
||||
this.connectPromise = connection;
|
||||
this.attemptConnect();
|
||||
return connection;
|
||||
}
|
||||
|
||||
private async attemptConnect(): Promise<void> {
|
||||
private attemptConnect(): void {
|
||||
if (this.connectionState.status === "disposed") {
|
||||
this.rejectConnect(new Error("Daemon client is disposed"));
|
||||
return;
|
||||
@@ -1601,6 +1607,12 @@ export class DaemonClient {
|
||||
if (this.connectionState.status === "connecting") {
|
||||
return;
|
||||
}
|
||||
void this.openConnectionAttempt("CONNECT_REQUEST");
|
||||
}
|
||||
|
||||
private async openConnectionAttempt(
|
||||
trigger: "CONNECT_REQUEST" | "STALE_CONNECT_ON_FOREGROUND",
|
||||
): Promise<void> {
|
||||
// This attempt supersedes any retry the last disconnect scheduled. Left
|
||||
// armed, that retry would tear down the connection this attempt opens.
|
||||
if (this.reconnectTimeout) {
|
||||
@@ -1608,14 +1620,24 @@ export class DaemonClient {
|
||||
this.reconnectTimeout = null;
|
||||
}
|
||||
|
||||
// Retire the old transport before credential lookup can yield.
|
||||
this.disposeTransport(
|
||||
1001,
|
||||
trigger === "STALE_CONNECT_ON_FOREGROUND" ? "Stale connection attempt" : "Reconnecting",
|
||||
);
|
||||
this.lastServerInfoMessage = null;
|
||||
this.authFailureReasonValue = null;
|
||||
const attempt = this.startConnectionAttempt();
|
||||
this.updateConnectionState(
|
||||
{ status: "connecting", attempt: this.reconnectAttempt },
|
||||
{ event: trigger },
|
||||
);
|
||||
|
||||
try {
|
||||
const resolution = resolveConnectionAuth(this.config);
|
||||
const selected = resolution instanceof Promise ? await resolution : resolution;
|
||||
if (!this.shouldReconnect) return;
|
||||
if (this.connectionAttempt !== attempt) return;
|
||||
this.helloAuth = selected.helloAuth;
|
||||
// Reconnect can overlap with browser close/error delivery ordering.
|
||||
// Always dispose previous transport before constructing the next one.
|
||||
this.disposeTransport();
|
||||
const baseTransportFactory =
|
||||
this.config.transportFactory ??
|
||||
createWebSocketTransportFactory(this.config.webSocketFactory ?? defaultWebSocketFactory);
|
||||
@@ -1663,31 +1685,6 @@ export class DaemonClient {
|
||||
deliver: (frame) => this.handleTransportMessage(frame),
|
||||
onActivity: () => this.recordTransportActivity(),
|
||||
});
|
||||
this.lastServerInfoMessage = null;
|
||||
this.authFailureReasonValue = null;
|
||||
|
||||
this.updateConnectionState(
|
||||
{
|
||||
status: "connecting",
|
||||
attempt: this.reconnectAttempt,
|
||||
},
|
||||
{ event: "CONNECT_REQUEST" },
|
||||
);
|
||||
this.resetConnectTimeout();
|
||||
const timeoutMs = Math.max(1, this.config.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS);
|
||||
this.connectTimeout = setTimeout(() => {
|
||||
if (this.connectionState.status !== "connecting") {
|
||||
return;
|
||||
}
|
||||
this.lastErrorValue = "Connection timed out";
|
||||
this.disposeTransport(1001, "Connection timed out");
|
||||
this.scheduleReconnect({
|
||||
reason: "Connection timed out",
|
||||
event: "CONNECT_TIMEOUT",
|
||||
reasonCode: "connect_timeout",
|
||||
});
|
||||
}, timeoutMs);
|
||||
|
||||
this.transportCleanup = [
|
||||
transport.onOpen(() => {
|
||||
this.traceInstant("paseo.connection.open");
|
||||
@@ -1704,7 +1701,6 @@ export class DaemonClient {
|
||||
event && typeof event === "object" && "code" in event ? event.code : "unknown",
|
||||
),
|
||||
});
|
||||
this.resetConnectTimeout();
|
||||
if (this.pendingGenericTransportErrorTimeout) {
|
||||
clearTimeout(this.pendingGenericTransportErrorTimeout);
|
||||
this.pendingGenericTransportErrorTimeout = null;
|
||||
@@ -1728,7 +1724,7 @@ export class DaemonClient {
|
||||
}),
|
||||
transport.onError((event) => {
|
||||
this.traceInstant("paseo.connection.error");
|
||||
this.resetConnectTimeout();
|
||||
this.clearConnectionAttempt();
|
||||
const reason = describeTransportError(event);
|
||||
const isGeneric = reason === "Transport error";
|
||||
// Browser WebSocket.onerror often provides no useful details and is followed
|
||||
@@ -1782,7 +1778,7 @@ export class DaemonClient {
|
||||
}),
|
||||
];
|
||||
} catch (error) {
|
||||
this.resetConnectTimeout();
|
||||
if (this.connectionAttempt !== attempt) return;
|
||||
const message = error instanceof Error ? error.message : "Failed to connect";
|
||||
this.lastErrorValue = message;
|
||||
this.scheduleReconnect({
|
||||
@@ -1828,7 +1824,7 @@ export class DaemonClient {
|
||||
clearTimeout(this.reconnectTimeout);
|
||||
this.reconnectTimeout = null;
|
||||
}
|
||||
this.resetConnectTimeout();
|
||||
this.clearConnectionAttempt();
|
||||
this.disposeTransport(1000, "Client closed");
|
||||
this.providerSnapshotUpdates.clear();
|
||||
this.clearWaiters(new Error("Daemon client closed"));
|
||||
@@ -1863,7 +1859,19 @@ export class DaemonClient {
|
||||
if (options?.verify) this.verifyConnectionOnForeground();
|
||||
return;
|
||||
}
|
||||
if (this.connectionState.status === "connecting") return;
|
||||
if (options?.verify) this.reconnectAttempt = 0;
|
||||
if (this.connectionState.status === "connecting") {
|
||||
if (
|
||||
options?.verify &&
|
||||
this.config.reconnect?.enabled !== false &&
|
||||
this.connectionAttempt &&
|
||||
Date.now() - this.connectionAttempt.startedAt >= FOREGROUND_STALE_CONNECT_MS
|
||||
) {
|
||||
// Keep work queued while the client opens a fresh transport.
|
||||
void this.openConnectionAttempt("STALE_CONNECT_ON_FOREGROUND");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (this.connectPromise) {
|
||||
this.attemptConnect();
|
||||
return;
|
||||
@@ -7475,7 +7483,7 @@ export class DaemonClient {
|
||||
}
|
||||
|
||||
private cleanupTransport(): void {
|
||||
this.resetConnectTimeout();
|
||||
this.clearConnectionAttempt();
|
||||
if (this.pendingGenericTransportErrorTimeout) {
|
||||
clearTimeout(this.pendingGenericTransportErrorTimeout);
|
||||
this.pendingGenericTransportErrorTimeout = null;
|
||||
@@ -7490,12 +7498,28 @@ export class DaemonClient {
|
||||
this.transportCleanup = [];
|
||||
}
|
||||
|
||||
private resetConnectTimeout(): void {
|
||||
if (!this.connectTimeout) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(this.connectTimeout);
|
||||
this.connectTimeout = null;
|
||||
private startConnectionAttempt(): ConnectionAttempt {
|
||||
const timeoutMs = Math.max(1, this.config.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS);
|
||||
const attempt: ConnectionAttempt = {
|
||||
startedAt: Date.now(),
|
||||
timeout: setTimeout(() => {
|
||||
if (this.connectionAttempt !== attempt) return;
|
||||
this.disposeTransport(1001, "Connection timed out");
|
||||
this.scheduleReconnect({
|
||||
reason: "Connection timed out",
|
||||
event: "CONNECT_TIMEOUT",
|
||||
reasonCode: "connect_timeout",
|
||||
});
|
||||
}, timeoutMs),
|
||||
};
|
||||
this.connectionAttempt = attempt;
|
||||
return attempt;
|
||||
}
|
||||
|
||||
private clearConnectionAttempt(): void {
|
||||
if (!this.connectionAttempt) return;
|
||||
clearTimeout(this.connectionAttempt.timeout);
|
||||
this.connectionAttempt = null;
|
||||
}
|
||||
|
||||
private handleTransportMessage(data: unknown): void {
|
||||
@@ -7846,6 +7870,7 @@ export class DaemonClient {
|
||||
// Attempt now instead of after the backoff delay. Failures of that attempt back off as usual.
|
||||
immediate?: boolean;
|
||||
}): void {
|
||||
this.clearConnectionAttempt();
|
||||
if (this.reconnectTimeout) {
|
||||
clearTimeout(this.reconnectTimeout);
|
||||
this.reconnectTimeout = null;
|
||||
@@ -8033,7 +8058,7 @@ export class DaemonClient {
|
||||
this.destinationServerId = serverInfo.serverId;
|
||||
this.connectionFailure = undefined;
|
||||
if (this.connectionState.status === "connecting") {
|
||||
this.resetConnectTimeout();
|
||||
this.clearConnectionAttempt();
|
||||
this.reconnectAttempt = 0;
|
||||
this.updateConnectionState({ status: "connected" }, { event: "HELLO_SERVER_INFO" });
|
||||
this.startLivenessHeartbeat();
|
||||
|
||||
Reference in New Issue
Block a user