mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-22 00:05:15 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
79503ba5a1 | ||
|
|
65fadb620a |
@@ -21,6 +21,7 @@ import { useSessionStore, type Agent } from "@/stores/session-store";
|
||||
import { normalizeAgentSnapshot } from "@/utils/agent-snapshots";
|
||||
import { isAgentArchiving, setAgentArchiving } from "@/hooks/use-archive-agent";
|
||||
import { queryClient } from "@/data/query-client";
|
||||
import { checkoutStatusQueryKey } from "@/git/query-keys";
|
||||
import {
|
||||
HostRuntimeController,
|
||||
HostRuntimeStore,
|
||||
@@ -1748,7 +1749,7 @@ describe("HostRuntimeController", () => {
|
||||
});
|
||||
|
||||
latencies["relay:relay.paseo.sh:443"] = 30;
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await vi.advanceTimersByTimeAsync(240_000);
|
||||
await controller.runProbeCycleNow();
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe("direct:lan:6767");
|
||||
|
||||
@@ -1807,6 +1808,429 @@ describe("HostRuntimeController", () => {
|
||||
await controller.stop();
|
||||
});
|
||||
|
||||
it("ignores busy active heartbeat spikes when comparing against an idle alternate probe", async () => {
|
||||
// Replays the recorded 10:04-10:08 relay sequence (#9149): the active relay has a ~250-300 ms
|
||||
// baseline over its recent heartbeats, spikes to 383/503/491 ms right before each alternate
|
||||
// probe, and the idle alternate measures 236/322/291 ms.
|
||||
useHostRuntimeClock();
|
||||
const host = makeHost({
|
||||
connections: [relayPrimary, relayAlternate],
|
||||
preferredConnectionId: relayPrimary.id,
|
||||
});
|
||||
const clients: FakeDaemonClient[] = [];
|
||||
let alternateProbes = 0;
|
||||
const latencies: Record<string, number | Error> = {
|
||||
[relayPrimary.id]: 260,
|
||||
[relayAlternate.id]: 280,
|
||||
};
|
||||
const baseDeps = makeDeps(latencies, clients);
|
||||
const controller = new HostRuntimeController({
|
||||
host,
|
||||
deps: {
|
||||
...baseDeps,
|
||||
connectToDaemon: async (input) => {
|
||||
if (input.connection.id === relayAlternate.id) {
|
||||
alternateProbes += 1;
|
||||
}
|
||||
return baseDeps.connectToDaemon(input);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await controller.start({ autoProbe: false });
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
|
||||
const activeClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
|
||||
|
||||
const rounds = [
|
||||
{ intervalMs: 120_000, baseline: 295, spike: 383, alternate: 236 },
|
||||
{ intervalMs: 240_000, baseline: 251, spike: 503, alternate: 322 },
|
||||
{ intervalMs: 480_000, baseline: 341, spike: 491, alternate: 291 },
|
||||
];
|
||||
for (const round of rounds) {
|
||||
await vi.advanceTimersByTimeAsync(round.intervalMs - 10_000);
|
||||
activeClient.heartbeatReportsRtt(round.baseline);
|
||||
await controller.runProbeCycleNow();
|
||||
await vi.advanceTimersByTimeAsync(10_000);
|
||||
activeClient.heartbeatReportsRtt(round.spike);
|
||||
latencies[relayAlternate.id] = round.alternate;
|
||||
await controller.runProbeCycleNow();
|
||||
}
|
||||
|
||||
expect(alternateProbes).toBe(4);
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
|
||||
expect(controller.getSnapshot().client).toBe(activeClient);
|
||||
expect(activeClient.isDisposed()).toBe(false);
|
||||
await controller.stop();
|
||||
});
|
||||
|
||||
it("requires a proportional RTT improvement before switching away from a high-latency link", async () => {
|
||||
useHostRuntimeClock();
|
||||
const host = makeHost({
|
||||
connections: [relayPrimary, relayAlternate],
|
||||
preferredConnectionId: relayPrimary.id,
|
||||
});
|
||||
const clients: FakeDaemonClient[] = [];
|
||||
const latencies: Record<string, number | Error> = {
|
||||
[relayPrimary.id]: 300,
|
||||
[relayAlternate.id]: 320,
|
||||
};
|
||||
const controller = new HostRuntimeController({
|
||||
host,
|
||||
deps: makeDeps(latencies, clients),
|
||||
});
|
||||
|
||||
await controller.start({ autoProbe: false });
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
|
||||
const activeClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
|
||||
|
||||
// 60 ms faster on a 300 ms link (20%) is noise, not enough to justify a switch.
|
||||
activeClient.heartbeatReportsRtt(300);
|
||||
latencies[relayAlternate.id] = 240;
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
await vi.advanceTimersByTimeAsync(30 * 60_000);
|
||||
await controller.runProbeCycleNow();
|
||||
}
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
|
||||
|
||||
// 150 ms faster on a 300 ms link (50%) is a real win and switches after 3 consecutive probes.
|
||||
latencies[relayAlternate.id] = 150;
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
await vi.advanceTimersByTimeAsync(index === 0 ? 30 * 60_000 : 120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
}
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayAlternate.id);
|
||||
await controller.stop();
|
||||
});
|
||||
|
||||
it("exponentially backs off inactive connection probes while the active connection keeps winning", async () => {
|
||||
useHostRuntimeClock();
|
||||
const host = makeHost({
|
||||
connections: [relayPrimary, relayAlternate],
|
||||
preferredConnectionId: relayPrimary.id,
|
||||
});
|
||||
const clients: FakeDaemonClient[] = [];
|
||||
let alternateProbes = 0;
|
||||
const baseDeps = makeDeps(
|
||||
{
|
||||
[relayPrimary.id]: 250,
|
||||
[relayAlternate.id]: 260,
|
||||
},
|
||||
clients,
|
||||
);
|
||||
const controller = new HostRuntimeController({
|
||||
host,
|
||||
deps: {
|
||||
...baseDeps,
|
||||
connectToDaemon: async (input) => {
|
||||
if (input.connection.id === relayAlternate.id) {
|
||||
alternateProbes += 1;
|
||||
}
|
||||
return baseDeps.connectToDaemon(input);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await controller.start({ autoProbe: false });
|
||||
const activeClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
|
||||
activeClient.heartbeatReportsRtt(250);
|
||||
expect(alternateProbes).toBe(1);
|
||||
|
||||
// First inactive online probe runs at 2m -> active wins, next interval backs off to 4m.
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
expect(alternateProbes).toBe(2);
|
||||
|
||||
// At +2m (still < 4m), no inactive probe runs.
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
expect(alternateProbes).toBe(2);
|
||||
|
||||
// At +4m total, second inactive probe runs -> backs off to 8m.
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
expect(alternateProbes).toBe(3);
|
||||
|
||||
// At +6m (< 8m), still quiet; at +8m, third inactive probe runs -> backs off to 16m.
|
||||
await vi.advanceTimersByTimeAsync(6 * 60_000);
|
||||
await controller.runProbeCycleNow();
|
||||
expect(alternateProbes).toBe(3);
|
||||
await vi.advanceTimersByTimeAsync(2 * 60_000);
|
||||
await controller.runProbeCycleNow();
|
||||
expect(alternateProbes).toBe(4);
|
||||
|
||||
// Foreground verification resets backoff back to 2m without forcing an immediate probe.
|
||||
await vi.advanceTimersByTimeAsync(3 * 60_000);
|
||||
controller.ensureConnected({ verify: true });
|
||||
await controller.runProbeCycleNow();
|
||||
expect(alternateProbes).toBe(5);
|
||||
await controller.stop();
|
||||
});
|
||||
|
||||
it("keeps the active connection online while connecting the replacement during an adaptive switch", async () => {
|
||||
useHostRuntimeClock();
|
||||
const host = makeHost({
|
||||
connections: [relayPrimary, relayAlternate],
|
||||
preferredConnectionId: relayPrimary.id,
|
||||
});
|
||||
const clients: FakeDaemonClient[] = [];
|
||||
const latencies: Record<string, number | Error> = {
|
||||
[relayPrimary.id]: 300,
|
||||
[relayAlternate.id]: 320,
|
||||
};
|
||||
let replacementConnectGate: ReturnType<typeof createDeferred<void>> | null = null;
|
||||
let shouldFailReplacement = false;
|
||||
const baseDeps = makeDeps(latencies, clients);
|
||||
const controller = new HostRuntimeController({
|
||||
host,
|
||||
deps: {
|
||||
...baseDeps,
|
||||
createClient: (input) => {
|
||||
const client = baseDeps.createClient(input) as unknown as FakeDaemonClient;
|
||||
if (input.connection.id === relayAlternate.id) {
|
||||
const origConnect = client.connect.bind(client);
|
||||
client.connect = async () => {
|
||||
if (replacementConnectGate) {
|
||||
await replacementConnectGate.promise;
|
||||
}
|
||||
if (shouldFailReplacement) {
|
||||
throw new Error("replacement handshake failed");
|
||||
}
|
||||
return origConnect();
|
||||
};
|
||||
}
|
||||
return client as unknown as DaemonClient;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await controller.start({ autoProbe: false });
|
||||
const initialClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
|
||||
initialClient.heartbeatReportsRtt(300);
|
||||
latencies[relayAlternate.id] = 100;
|
||||
|
||||
const observedSnapshots: Array<{
|
||||
status: string;
|
||||
connectionId: string | null;
|
||||
hasClient: boolean;
|
||||
}> = [];
|
||||
const unsubscribe = controller.subscribe(() => {
|
||||
const snap = controller.getSnapshot();
|
||||
observedSnapshots.push({
|
||||
status: snap.connectionStatus,
|
||||
connectionId: snap.activeConnectionId,
|
||||
hasClient: snap.client !== null,
|
||||
});
|
||||
});
|
||||
|
||||
// Two wins first.
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
|
||||
// Third win triggers a switch, but replacement connect fails: active client must stay online.
|
||||
shouldFailReplacement = true;
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
|
||||
expect(controller.getSnapshot().connectionStatus).toBe("online");
|
||||
expect(controller.getSnapshot().client).toBe(initialClient);
|
||||
expect(initialClient.isDisposed()).toBe(false);
|
||||
|
||||
// Now let replacement succeed after a delay: host must stay online on initialClient while
|
||||
// replacement connects, and never emit connecting or client=null.
|
||||
shouldFailReplacement = false;
|
||||
observedSnapshots.length = 0;
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
}
|
||||
replacementConnectGate = createDeferred<void>();
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
const switchingCycle = controller.runProbeCycleNow();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
|
||||
expect(controller.getSnapshot().connectionStatus).toBe("online");
|
||||
expect(controller.getSnapshot().client).toBe(initialClient);
|
||||
expect(initialClient.isDisposed()).toBe(false);
|
||||
|
||||
replacementConnectGate.resolve();
|
||||
await switchingCycle;
|
||||
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayAlternate.id);
|
||||
expect(controller.getSnapshot().connectionStatus).toBe("online");
|
||||
expect(controller.getSnapshot().client).not.toBeNull();
|
||||
expect(controller.getSnapshot().client).not.toBe(initialClient);
|
||||
expect(initialClient.isDisposed()).toBe(true);
|
||||
expect(observedSnapshots.some((snap) => snap.status !== "online" || !snap.hasClient)).toBe(
|
||||
false,
|
||||
);
|
||||
|
||||
unsubscribe();
|
||||
await controller.stop();
|
||||
});
|
||||
|
||||
it("keeps active client status updates working after a failed make-before-break upgrade", async () => {
|
||||
useHostRuntimeClock();
|
||||
const host = makeHost({
|
||||
connections: [relayPrimary, relayAlternate],
|
||||
preferredConnectionId: relayPrimary.id,
|
||||
});
|
||||
const clients: FakeDaemonClient[] = [];
|
||||
const latencies: Record<string, number | Error> = {
|
||||
[relayPrimary.id]: 300,
|
||||
[relayAlternate.id]: 320,
|
||||
};
|
||||
const baseDeps = makeDeps(latencies, clients);
|
||||
const controller = new HostRuntimeController({
|
||||
host,
|
||||
deps: {
|
||||
...baseDeps,
|
||||
createClient: (input) => {
|
||||
const client = baseDeps.createClient(input) as unknown as FakeDaemonClient;
|
||||
if (input.connection.id === relayAlternate.id) {
|
||||
client.connect = async () => {
|
||||
throw new Error("replacement handshake failed");
|
||||
};
|
||||
}
|
||||
return client as unknown as DaemonClient;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await controller.start({ autoProbe: false });
|
||||
const initialClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
|
||||
initialClient.heartbeatReportsRtt(300);
|
||||
latencies[relayAlternate.id] = 100;
|
||||
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
}
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
|
||||
expect(controller.getSnapshot().connectionStatus).toBe("online");
|
||||
|
||||
initialClient.setConnectionState({ status: "disconnected", reason: "transport closed" });
|
||||
expect(controller.getSnapshot().connectionStatus).toBe("error");
|
||||
expect(controller.getSnapshot().lastError).toBe("transport closed");
|
||||
await controller.stop();
|
||||
});
|
||||
|
||||
it("aborts a pending make-before-break upgrade when the candidate connection is removed", async () => {
|
||||
useHostRuntimeClock();
|
||||
const host = makeHost({
|
||||
connections: [relayPrimary, relayAlternate],
|
||||
preferredConnectionId: relayPrimary.id,
|
||||
});
|
||||
const clients: FakeDaemonClient[] = [];
|
||||
const latencies: Record<string, number | Error> = {
|
||||
[relayPrimary.id]: 300,
|
||||
[relayAlternate.id]: 320,
|
||||
};
|
||||
const replacementGate = createDeferred<void>();
|
||||
const replacementClients: FakeDaemonClient[] = [];
|
||||
const baseDeps = makeDeps(latencies, clients);
|
||||
const controller = new HostRuntimeController({
|
||||
host,
|
||||
deps: {
|
||||
...baseDeps,
|
||||
createClient: (input) => {
|
||||
const client = baseDeps.createClient(input) as unknown as FakeDaemonClient;
|
||||
if (input.connection.id === relayAlternate.id) {
|
||||
replacementClients.push(client);
|
||||
const origConnect = client.connect.bind(client);
|
||||
client.connect = async () => {
|
||||
await replacementGate.promise;
|
||||
return origConnect();
|
||||
};
|
||||
}
|
||||
return client as unknown as DaemonClient;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await controller.start({ autoProbe: false });
|
||||
const initialClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
|
||||
initialClient.heartbeatReportsRtt(300);
|
||||
latencies[relayAlternate.id] = 100;
|
||||
|
||||
for (let index = 0; index < 2; index += 1) {
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await controller.runProbeCycleNow();
|
||||
}
|
||||
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
const switchingCycle = controller.runProbeCycleNow();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(replacementClients).toHaveLength(1);
|
||||
|
||||
const updatePromise = controller.updateHost({
|
||||
...host,
|
||||
connections: [relayPrimary],
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
replacementGate.resolve();
|
||||
await Promise.all([switchingCycle, updatePromise]);
|
||||
|
||||
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
|
||||
expect(controller.getSnapshot().connectionStatus).toBe("online");
|
||||
expect(controller.getSnapshot().client).toBe(initialClient);
|
||||
expect(initialClient.isDisposed()).toBe(false);
|
||||
expect(replacementClients[0]?.isDisposed()).toBe(true);
|
||||
await controller.stop();
|
||||
});
|
||||
|
||||
it("runs store cache recovery when a make-before-break adaptive switch advances the online epoch", async () => {
|
||||
useHostRuntimeClock();
|
||||
const host = makeHost({
|
||||
serverId: "srv_upgrade_cache_recovery",
|
||||
connections: [relayPrimary, relayAlternate],
|
||||
preferredConnectionId: relayPrimary.id,
|
||||
});
|
||||
const clients: FakeDaemonClient[] = [];
|
||||
const latencies: Record<string, number | Error> = {
|
||||
[relayPrimary.id]: 300,
|
||||
[relayAlternate.id]: 320,
|
||||
};
|
||||
const store = new HostRuntimeStore({
|
||||
storage: createMemoryHostRuntimeStorage(),
|
||||
deps: makeDeps(latencies, clients),
|
||||
});
|
||||
const queryKey = checkoutStatusQueryKey(host.serverId, "/repo");
|
||||
|
||||
try {
|
||||
store.syncHosts([host]);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(store.getSnapshot(host.serverId)?.connectionStatus).toBe("online");
|
||||
const initialEpoch = store.getSnapshot(host.serverId)?.connectionEpoch ?? 0;
|
||||
const activeClient = store.getSnapshot(host.serverId)?.client as unknown as FakeDaemonClient;
|
||||
const initialHistoryGen =
|
||||
useSessionStore.getState().sessions[host.serverId]?.historySyncGeneration ?? 0;
|
||||
|
||||
queryClient.setQueryData(queryKey, { branch: "main" });
|
||||
expect(queryClient.getQueryState(queryKey)?.isInvalidated).toBe(false);
|
||||
|
||||
activeClient.heartbeatReportsRtt(300);
|
||||
latencies[relayAlternate.id] = 100;
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
await store.runProbeCycleNow(host.serverId);
|
||||
}
|
||||
|
||||
expect(store.getSnapshot(host.serverId)?.activeConnectionId).toBe(relayAlternate.id);
|
||||
expect(store.getSnapshot(host.serverId)?.connectionStatus).toBe("online");
|
||||
expect(store.getSnapshot(host.serverId)?.connectionEpoch).toBeGreaterThan(initialEpoch);
|
||||
expect(queryClient.getQueryState(queryKey)?.isInvalidated).toBe(true);
|
||||
expect(
|
||||
useSessionStore.getState().sessions[host.serverId]?.historySyncGeneration ?? 0,
|
||||
).toBeGreaterThan(initialHistoryGen);
|
||||
} finally {
|
||||
queryClient.removeQueries({ queryKey });
|
||||
store.syncHosts([]);
|
||||
}
|
||||
});
|
||||
|
||||
it("exposes one snapshot with active connection and status from same source", async () => {
|
||||
const host = makeHost();
|
||||
const clients: FakeDaemonClient[] = [];
|
||||
|
||||
@@ -245,8 +245,11 @@ export interface HostRuntimeStartOptions {
|
||||
const PROBE_TICK_MS = 2_000;
|
||||
const PROBE_STEADY_MS = 10_000;
|
||||
const PROBE_MAX_BACKOFF_MS = 30_000;
|
||||
const PROBE_INACTIVE_WHILE_ONLINE_MS = 120_000;
|
||||
const PROBE_INACTIVE_WHILE_ONLINE_INITIAL_MS = 120_000;
|
||||
const PROBE_INACTIVE_WHILE_ONLINE_MAX_MS = 30 * 60_000;
|
||||
const ACTIVE_RTT_WINDOW_MS = 60_000;
|
||||
const ADAPTIVE_SWITCH_THRESHOLD_MS = 40;
|
||||
const ADAPTIVE_SWITCH_MIN_IMPROVEMENT_RATIO = 0.35;
|
||||
const ADAPTIVE_SWITCH_CONSECUTIVE_PROBES = 3;
|
||||
const CONFIGURED_OVERRIDE_BOOTSTRAP_RETRY_MS = 1_000;
|
||||
|
||||
@@ -549,13 +552,14 @@ function probeIntervalForConnection(
|
||||
firstSeenAt: number,
|
||||
isActiveOnline: boolean,
|
||||
hasActiveOnlineConnection: boolean,
|
||||
inactiveOnlineIntervalMs: number,
|
||||
now: number,
|
||||
): number {
|
||||
if (isActiveOnline) {
|
||||
return PROBE_STEADY_MS;
|
||||
}
|
||||
if (hasActiveOnlineConnection) {
|
||||
return PROBE_INACTIVE_WHILE_ONLINE_MS;
|
||||
return inactiveOnlineIntervalMs;
|
||||
}
|
||||
const age = now - firstSeenAt;
|
||||
if (age < 10_000) return 2_000;
|
||||
@@ -728,6 +732,8 @@ export class HostRuntimeController {
|
||||
private connectionLastProbedAt = new Map<string, number>();
|
||||
private switchCandidateConnectionId: string | null = null;
|
||||
private switchCandidateHitCount = 0;
|
||||
private inactiveOnlineProbeBackoffStep = 0;
|
||||
private activeRttSamples: Array<{ at: number; rttMs: number }> = [];
|
||||
private clientIdPromise: Promise<string> | null = null;
|
||||
private clientIdHash: string | null = null;
|
||||
private switchRequestVersion = 0;
|
||||
@@ -823,6 +829,9 @@ export class HostRuntimeController {
|
||||
this.activeClient = null;
|
||||
await prev.close().catch(() => undefined);
|
||||
}
|
||||
this.resetSwitchCandidate();
|
||||
this.clearActiveRttSamples();
|
||||
this.resetInactiveOnlineProbeBackoff();
|
||||
this.applyConnectionEvent({ type: "stopped" });
|
||||
this.updateSnapshot({
|
||||
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
|
||||
@@ -832,11 +841,15 @@ export class HostRuntimeController {
|
||||
|
||||
async updateHost(host: HostProfile): Promise<void> {
|
||||
const passwordChanged = this.host.password !== host.password;
|
||||
const connectionsChanged = !equal(this.host.connections, host.connections);
|
||||
if (passwordChanged) {
|
||||
this.probeRequestVersion += 1;
|
||||
this.authRejectedConnectionIds.clear();
|
||||
this.updateSnapshot({ authFailureReason: null });
|
||||
}
|
||||
if (passwordChanged || connectionsChanged) {
|
||||
this.resetInactiveOnlineProbeBackoff();
|
||||
}
|
||||
const activeConnectionId = this.snapshot.activeConnectionId;
|
||||
const previousActiveConnection = findConnectionById(this.host, activeConnectionId);
|
||||
this.host = host;
|
||||
@@ -862,6 +875,9 @@ export class HostRuntimeController {
|
||||
}
|
||||
|
||||
ensureConnected(options?: { verify?: boolean }): void {
|
||||
if (options?.verify) {
|
||||
this.resetInactiveOnlineProbeBackoff();
|
||||
}
|
||||
this.activeClient?.ensureConnected(options);
|
||||
}
|
||||
|
||||
@@ -962,6 +978,7 @@ export class HostRuntimeController {
|
||||
const activeConnectionId = this.snapshot.activeConnectionId;
|
||||
const hasActiveOnlineConnection = isOnline && activeConnectionId !== null;
|
||||
|
||||
const inactiveOnlineIntervalMs = this.inactiveOnlineProbeIntervalMs();
|
||||
const connectionsToProbe = this.host.connections.filter((connection) => {
|
||||
if (this.authRejectedConnectionIds.has(connection.id)) return false;
|
||||
const lastProbed = this.connectionLastProbedAt.get(connection.id);
|
||||
@@ -974,6 +991,7 @@ export class HostRuntimeController {
|
||||
firstSeen,
|
||||
isActiveOnline,
|
||||
hasActiveOnlineConnection,
|
||||
inactiveOnlineIntervalMs,
|
||||
now,
|
||||
);
|
||||
return now - lastProbed >= interval;
|
||||
@@ -1081,10 +1099,10 @@ export class HostRuntimeController {
|
||||
|
||||
// Adaptive switching counts fresh comparisons only. The probe map keeps the last known
|
||||
// sample per connection for display and failover; a streak hit needs a candidate measured
|
||||
// in this cycle and a heartbeat RTT from the socket that is active right now. Re-reading
|
||||
// the cached map at every ten-second active tick turned one idle-probe sample into three
|
||||
// "consecutive" wins, and a reconnected socket inherited its predecessor's RTT, so a
|
||||
// working relay connection was replaced within seconds of coming online.
|
||||
// in this cycle and a baseline RTT from the socket that is active right now. Using the
|
||||
// rolling minimum of recent active heartbeats filters out single busy/stalled ticks so a
|
||||
// loaded working socket does not lose to an idle probe, and a proportional threshold avoids
|
||||
// flapping on normal jitter on high-RTT links.
|
||||
const sampledCandidates = connectionsToProbe.filter(
|
||||
(connection) => connection.id !== currentActiveConnectionId,
|
||||
);
|
||||
@@ -1102,22 +1120,29 @@ export class HostRuntimeController {
|
||||
})
|
||||
.sort((left, right) => left.latencyMs - right.latencyMs)[0] ?? null;
|
||||
if (!fastest) {
|
||||
// Every candidate sampled this cycle failed: that breaks the streak.
|
||||
// Every candidate sampled this cycle failed: that breaks the streak and backs off.
|
||||
this.resetSwitchCandidate();
|
||||
this.advanceInactiveOnlineProbeBackoff();
|
||||
return;
|
||||
}
|
||||
|
||||
const activeRttMs = this.currentActiveSocketRttMs();
|
||||
const activeRttMs = this.currentActiveSocketBaselineRttMs(performance.now());
|
||||
if (activeRttMs === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const requiredImprovementMs = Math.max(
|
||||
ADAPTIVE_SWITCH_THRESHOLD_MS,
|
||||
activeRttMs * ADAPTIVE_SWITCH_MIN_IMPROVEMENT_RATIO,
|
||||
);
|
||||
const improvement = activeRttMs - fastest.latencyMs;
|
||||
if (improvement < ADAPTIVE_SWITCH_THRESHOLD_MS) {
|
||||
if (improvement < requiredImprovementMs) {
|
||||
this.resetSwitchCandidate();
|
||||
this.advanceInactiveOnlineProbeBackoff();
|
||||
return;
|
||||
}
|
||||
|
||||
this.resetInactiveOnlineProbeBackoff();
|
||||
if (this.switchCandidateConnectionId === fastest.connectionId) {
|
||||
this.switchCandidateHitCount += 1;
|
||||
} else {
|
||||
@@ -1130,6 +1155,7 @@ export class HostRuntimeController {
|
||||
await this.switchToConnection({
|
||||
connectionId: fastest.connectionId,
|
||||
expectedProbeVersion: requestVersion,
|
||||
keepActiveUntilConnected: true,
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -1194,6 +1220,7 @@ export class HostRuntimeController {
|
||||
return;
|
||||
}
|
||||
if (rttMs !== null) {
|
||||
this.recordActiveRttSample(rttMs, performance.now());
|
||||
probeByConnectionId.set(connection.id, {
|
||||
status: "available",
|
||||
latencyMs: rttMs,
|
||||
@@ -1245,6 +1272,45 @@ export class HostRuntimeController {
|
||||
return client.getLastLivenessRttMs();
|
||||
}
|
||||
|
||||
private recordActiveRttSample(rttMs: number, now: number): void {
|
||||
this.activeRttSamples = this.activeRttSamples.filter(
|
||||
(sample) => now - sample.at <= ACTIVE_RTT_WINDOW_MS,
|
||||
);
|
||||
this.activeRttSamples.push({ at: now, rttMs });
|
||||
}
|
||||
|
||||
private clearActiveRttSamples(): void {
|
||||
this.activeRttSamples = [];
|
||||
}
|
||||
|
||||
private currentActiveSocketBaselineRttMs(now: number): number | null {
|
||||
const currentRttMs = this.currentActiveSocketRttMs();
|
||||
if (currentRttMs === null) return null;
|
||||
this.recordActiveRttSample(currentRttMs, now);
|
||||
let baseline = this.activeRttSamples[0]!.rttMs;
|
||||
for (const sample of this.activeRttSamples) {
|
||||
if (sample.rttMs < baseline) baseline = sample.rttMs;
|
||||
}
|
||||
return baseline;
|
||||
}
|
||||
|
||||
private inactiveOnlineProbeIntervalMs(): number {
|
||||
return Math.min(
|
||||
PROBE_INACTIVE_WHILE_ONLINE_INITIAL_MS * 2 ** this.inactiveOnlineProbeBackoffStep,
|
||||
PROBE_INACTIVE_WHILE_ONLINE_MAX_MS,
|
||||
);
|
||||
}
|
||||
|
||||
private advanceInactiveOnlineProbeBackoff(): void {
|
||||
if (this.inactiveOnlineProbeIntervalMs() < PROBE_INACTIVE_WHILE_ONLINE_MAX_MS) {
|
||||
this.inactiveOnlineProbeBackoffStep += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private resetInactiveOnlineProbeBackoff(): void {
|
||||
this.inactiveOnlineProbeBackoffStep = 0;
|
||||
}
|
||||
|
||||
private resetSwitchCandidate(): void {
|
||||
this.switchCandidateConnectionId = null;
|
||||
this.switchCandidateHitCount = 0;
|
||||
@@ -1434,6 +1500,7 @@ export class HostRuntimeController {
|
||||
connectionId: string;
|
||||
expectedProbeVersion?: number;
|
||||
existingClient?: DaemonClient;
|
||||
keepActiveUntilConnected?: boolean;
|
||||
}): Promise<void> {
|
||||
const { connectionId, expectedProbeVersion, existingClient } = input;
|
||||
if (!this.canProceedForProbe(expectedProbeVersion)) {
|
||||
@@ -1455,6 +1522,24 @@ export class HostRuntimeController {
|
||||
return;
|
||||
}
|
||||
|
||||
const makeBeforeBreak =
|
||||
Boolean(input.keepActiveUntilConnected) &&
|
||||
!existingClient &&
|
||||
this.activeClient !== null &&
|
||||
this.connectionMachineState.tag === "online";
|
||||
|
||||
const nextGeneration = this.snapshot.clientGeneration + 1;
|
||||
if (makeBeforeBreak) {
|
||||
await this.switchToConnectionMakeBeforeBreak({
|
||||
connection,
|
||||
clientId,
|
||||
nextGeneration,
|
||||
requestVersion,
|
||||
expectedProbeVersion,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
await this.disposePreviousActiveClient();
|
||||
|
||||
if (!this.isSwitchStillValid(requestVersion, expectedProbeVersion)) {
|
||||
@@ -1462,7 +1547,6 @@ export class HostRuntimeController {
|
||||
return;
|
||||
}
|
||||
|
||||
const nextGeneration = this.snapshot.clientGeneration + 1;
|
||||
const client =
|
||||
existingClient ??
|
||||
this.deps.createClient({
|
||||
@@ -1479,6 +1563,8 @@ export class HostRuntimeController {
|
||||
}
|
||||
|
||||
this.activeClient = client;
|
||||
this.clearActiveRttSamples();
|
||||
this.resetInactiveOnlineProbeBackoff();
|
||||
this.applyConnectionEvent({
|
||||
type: "select_connection",
|
||||
connectionId: connection.id,
|
||||
@@ -1490,67 +1576,148 @@ export class HostRuntimeController {
|
||||
client: null,
|
||||
});
|
||||
|
||||
const failConnection = async (error: unknown) => {
|
||||
if (!this.isCurrentSwitchRequest(requestVersion) || this.activeClient !== client) return;
|
||||
const authFailure = getConnectionAuthFailureReason(error);
|
||||
if (authFailure) this.authRejectedConnectionIds.add(connection.id);
|
||||
this.unsubscribeClientStatus?.();
|
||||
this.unsubscribeClientStatus = null;
|
||||
this.unsubscribeClientHandlers?.();
|
||||
this.unsubscribeClientHandlers = null;
|
||||
client.setReconnectEnabled(false);
|
||||
this.applyConnectionEvent({ type: "connect_failed", message: toErrorMessage(error) });
|
||||
this.updateSnapshot({
|
||||
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
|
||||
client: null,
|
||||
authFailureReason: authFailure,
|
||||
});
|
||||
if (authFailure) {
|
||||
const probes = new Map(this.snapshot.probeByConnectionId);
|
||||
probes.set(connection.id, { status: "unavailable", latencyMs: null });
|
||||
this.updateSnapshot({ probeByConnectionId: probes });
|
||||
for (const candidate of this.host.connections) {
|
||||
if (!this.authRejectedConnectionIds.has(candidate.id)) {
|
||||
this.connectionLastProbedAt.delete(candidate.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
await client.close();
|
||||
} catch {
|
||||
/* Preserve the compatibility/connection error. */
|
||||
}
|
||||
if (
|
||||
authFailure &&
|
||||
this.host.connections.some((candidate) => !this.authRejectedConnectionIds.has(candidate.id))
|
||||
) {
|
||||
void this.runProbeCycleNow();
|
||||
}
|
||||
};
|
||||
try {
|
||||
if (!existingClient) await client.connect();
|
||||
if (!this.isCurrentSwitchRequest(requestVersion) || this.activeClient !== client) return;
|
||||
this.unsubscribeClientHandlers =
|
||||
this.deps.mountClientHandlers?.({ client, host: this.host, connection }) ?? null;
|
||||
this.unsubscribeClientStatus = client.subscribeConnectionStatus((state) => {
|
||||
if (!this.isCurrentSwitchRequest(requestVersion) || this.activeClient !== client) return;
|
||||
if (client.authFailureReason) {
|
||||
void failConnection(new DaemonAuthenticationError(client.authFailureReason));
|
||||
return;
|
||||
}
|
||||
if (state.status === "connected") this.learnRelaySecurity(client, connection.id);
|
||||
// Switch evidence belongs to the socket it was measured against.
|
||||
if (state.status !== "connected") this.resetSwitchCandidate();
|
||||
this.applyConnectionEvent({ type: "client_state", state, lastError: client.lastError });
|
||||
this.updateSnapshot({
|
||||
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
|
||||
...this.buildAgentDirectoryStatusPatch(),
|
||||
client,
|
||||
authFailureReason: null,
|
||||
});
|
||||
});
|
||||
this.attachActiveClient({ client, connection });
|
||||
} catch (error) {
|
||||
await failConnection(error);
|
||||
await this.failActiveConnection({ client, connection, error });
|
||||
}
|
||||
}
|
||||
|
||||
private async switchToConnectionMakeBeforeBreak(input: {
|
||||
connection: HostConnection;
|
||||
clientId: string;
|
||||
nextGeneration: number;
|
||||
requestVersion: number;
|
||||
expectedProbeVersion?: number;
|
||||
}): Promise<void> {
|
||||
const { connection, clientId, nextGeneration, requestVersion, expectedProbeVersion } = input;
|
||||
const replacement = this.deps.createClient({
|
||||
host: this.host,
|
||||
connection,
|
||||
clientId,
|
||||
runtimeGeneration: nextGeneration,
|
||||
});
|
||||
replacement.setReconnectEnabled(false);
|
||||
try {
|
||||
await replacement.connect();
|
||||
} catch (error) {
|
||||
const authFailure = getConnectionAuthFailureReason(error);
|
||||
if (authFailure) {
|
||||
this.authRejectedConnectionIds.add(connection.id);
|
||||
const probes = new Map(this.snapshot.probeByConnectionId);
|
||||
probes.set(connection.id, { status: "unavailable", latencyMs: null });
|
||||
this.updateSnapshot({ probeByConnectionId: probes });
|
||||
}
|
||||
await replacement.close().catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
const resolveValidTarget = (): HostConnection | null => {
|
||||
if (!this.isSwitchStillValid(requestVersion, expectedProbeVersion)) return null;
|
||||
const saved = findConnectionById(this.host, connection.id);
|
||||
if (!saved || connectionNeedsNewClient(connection, saved, replacement)) return null;
|
||||
return saved;
|
||||
};
|
||||
|
||||
if (!resolveValidTarget()) {
|
||||
await replacement.close().catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
replacement.setReconnectEnabled(true);
|
||||
await this.disposePreviousActiveClient();
|
||||
|
||||
const target = resolveValidTarget();
|
||||
if (!target) {
|
||||
await replacement.close().catch(() => undefined);
|
||||
return;
|
||||
}
|
||||
|
||||
this.activeClient = replacement;
|
||||
this.clearActiveRttSamples();
|
||||
this.resetInactiveOnlineProbeBackoff();
|
||||
this.applyConnectionEvent({
|
||||
type: "select_connection",
|
||||
connectionId: target.id,
|
||||
connection: toActiveConnection(target),
|
||||
});
|
||||
this.snapshot = { ...this.snapshot, clientGeneration: nextGeneration };
|
||||
this.attachActiveClient({ client: replacement, connection: target });
|
||||
}
|
||||
|
||||
private attachActiveClient(input: { client: DaemonClient; connection: HostConnection }): void {
|
||||
const { client, connection } = input;
|
||||
this.unsubscribeClientHandlers =
|
||||
this.deps.mountClientHandlers?.({ client, host: this.host, connection }) ?? null;
|
||||
this.unsubscribeClientStatus = client.subscribeConnectionStatus((state) => {
|
||||
if (this.activeClient !== client) return;
|
||||
if (client.authFailureReason) {
|
||||
void this.failActiveConnection({
|
||||
client,
|
||||
connection,
|
||||
error: new DaemonAuthenticationError(client.authFailureReason),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (state.status === "connected") this.learnRelaySecurity(client, connection.id);
|
||||
// Switch evidence belongs to the socket it was measured against.
|
||||
if (state.status !== "connected") {
|
||||
this.resetSwitchCandidate();
|
||||
this.clearActiveRttSamples();
|
||||
this.resetInactiveOnlineProbeBackoff();
|
||||
}
|
||||
this.applyConnectionEvent({ type: "client_state", state, lastError: client.lastError });
|
||||
this.updateSnapshot({
|
||||
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
|
||||
...this.buildAgentDirectoryStatusPatch(),
|
||||
client,
|
||||
authFailureReason: null,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private async failActiveConnection(input: {
|
||||
client: DaemonClient;
|
||||
connection: HostConnection;
|
||||
error: unknown;
|
||||
}): Promise<void> {
|
||||
const { client, connection, error } = input;
|
||||
if (this.activeClient !== client) return;
|
||||
const authFailure = getConnectionAuthFailureReason(error);
|
||||
if (authFailure) this.authRejectedConnectionIds.add(connection.id);
|
||||
this.unsubscribeClientStatus?.();
|
||||
this.unsubscribeClientStatus = null;
|
||||
this.unsubscribeClientHandlers?.();
|
||||
this.unsubscribeClientHandlers = null;
|
||||
client.setReconnectEnabled(false);
|
||||
this.applyConnectionEvent({ type: "connect_failed", message: toErrorMessage(error) });
|
||||
this.updateSnapshot({
|
||||
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
|
||||
client: null,
|
||||
authFailureReason: authFailure,
|
||||
});
|
||||
if (authFailure) {
|
||||
const probes = new Map(this.snapshot.probeByConnectionId);
|
||||
probes.set(connection.id, { status: "unavailable", latencyMs: null });
|
||||
this.updateSnapshot({ probeByConnectionId: probes });
|
||||
for (const candidate of this.host.connections) {
|
||||
if (!this.authRejectedConnectionIds.has(candidate.id)) {
|
||||
this.connectionLastProbedAt.delete(candidate.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
await client.close();
|
||||
} catch {
|
||||
/* Preserve the compatibility/connection error. */
|
||||
}
|
||||
if (
|
||||
authFailure &&
|
||||
this.host.connections.some((candidate) => !this.authRejectedConnectionIds.has(candidate.id))
|
||||
) {
|
||||
void this.runProbeCycleNow();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1693,6 +1860,7 @@ export class HostRuntimeStore {
|
||||
private hostRegistryStatus: HostRegistryStatus = "loading";
|
||||
private deps: HostRuntimeControllerDeps;
|
||||
private lastConnectionStatusByServer = new Map<string, HostRuntimeConnectionStatus>();
|
||||
private lastOnlineEpochByServer = new Map<string, number>();
|
||||
private connectionStatusStartedAtByServer = new Map<string, number>();
|
||||
private queuedAgentDrainInFlight = new Set<string>();
|
||||
private directorySyncByServer = new Map<string, DirectorySync>();
|
||||
@@ -1950,6 +2118,7 @@ export class HostRuntimeStore {
|
||||
|
||||
rekeyMap(this.controllers, oldServerId, newServerId);
|
||||
rekeyMap(this.lastConnectionStatusByServer, oldServerId, newServerId);
|
||||
rekeyMap(this.lastOnlineEpochByServer, oldServerId, newServerId);
|
||||
rekeyMap(this.connectionStatusStartedAtByServer, oldServerId, newServerId);
|
||||
this.replicaCache.reconcileServerId(oldServerId, newServerId);
|
||||
projectIconCache.reconcileServerId(oldServerId, newServerId);
|
||||
@@ -2442,6 +2611,7 @@ export class HostRuntimeStore {
|
||||
this.fleetDirectoryDemand.delete(serverId);
|
||||
this.controllers.delete(serverId);
|
||||
this.lastConnectionStatusByServer.delete(serverId);
|
||||
this.lastOnlineEpochByServer.delete(serverId);
|
||||
this.connectionStatusStartedAtByServer.delete(serverId);
|
||||
this.directorySyncByServer.get(serverId)?.dispose();
|
||||
this.directorySyncByServer.delete(serverId);
|
||||
@@ -2492,6 +2662,9 @@ export class HostRuntimeStore {
|
||||
);
|
||||
const initialSnapshot = controller.getSnapshot();
|
||||
this.lastConnectionStatusByServer.set(host.serverId, initialSnapshot.connectionStatus);
|
||||
if (initialSnapshot.connectionStatus === "online") {
|
||||
this.lastOnlineEpochByServer.set(host.serverId, initialSnapshot.connectionEpoch);
|
||||
}
|
||||
this.connectionStatusStartedAtByServer.set(host.serverId, Date.now());
|
||||
controller.subscribe(() => {
|
||||
const snapshot = controller.getSnapshot();
|
||||
@@ -2591,6 +2764,7 @@ export class HostRuntimeStore {
|
||||
const controller = this.controllers.get(serverId);
|
||||
if (!controller) {
|
||||
this.lastConnectionStatusByServer.delete(serverId);
|
||||
this.lastOnlineEpochByServer.delete(serverId);
|
||||
this.connectionStatusStartedAtByServer.delete(serverId);
|
||||
return;
|
||||
}
|
||||
@@ -2605,6 +2779,7 @@ export class HostRuntimeStore {
|
||||
},
|
||||
});
|
||||
const previousStatus = this.lastConnectionStatusByServer.get(serverId);
|
||||
const previousOnlineEpoch = this.lastOnlineEpochByServer.get(serverId);
|
||||
const statusChanged = previousStatus !== snapshot.connectionStatus;
|
||||
const isUnavailable =
|
||||
snapshot.connectionStatus !== "online" && snapshot.connectionStatus !== "idle";
|
||||
@@ -2615,7 +2790,11 @@ export class HostRuntimeStore {
|
||||
}
|
||||
this.lastConnectionStatusByServer.set(serverId, snapshot.connectionStatus);
|
||||
const didTransitionOnline =
|
||||
snapshot.connectionStatus === "online" && previousStatus !== "online";
|
||||
snapshot.connectionStatus === "online" &&
|
||||
(previousStatus !== "online" || previousOnlineEpoch !== snapshot.connectionEpoch);
|
||||
if (snapshot.connectionStatus === "online") {
|
||||
this.lastOnlineEpochByServer.set(serverId, snapshot.connectionEpoch);
|
||||
}
|
||||
if (didTransitionOnline) {
|
||||
useSessionStore.getState().bumpHistorySyncGeneration(serverId);
|
||||
for (const child of this.controllers.values()) void child.reconnectVia(serverId);
|
||||
|
||||
Reference in New Issue
Block a user