Compare commits

...
Author SHA1 Message Date
Mohamed Boudra 79503ba5a1 fix(app): preserve active listener, recheck candidate, and refresh caches on make-before-break upgrade
Keep installed active-client status listeners bound to `this.activeClient`
rather than `switchRequestVersion` so a pending or failed speculative
upgrade never mutes disconnect events on the active socket. Recheck that
the target connection still exists and matches before and after disposing
the old client so removing or editing an endpoint during handshake aborts
the upgrade. Track the last online `connectionEpoch` in `HostRuntimeStore`
so make-before-break upgrades that stay `online` still trigger reconnect
cache invalidation and history sync generation bumps.
2026-11-21 11:48:48 +01:00
Mohamed Boudra 65fadb620a fix(app): stop busy-heartbeat adaptive switches and back off online probes
Hosts with multiple saved connections still replaced healthy relay links
during active use: every 120 seconds the runtime compared an idle alternate
probe against the single latest heartbeat RTT on the busy active socket
using a fixed 40 ms threshold, then tore down the live socket before
connecting the replacement (leaving the host in connecting for ~2 s).

Use the 60-second rolling minimum of active heartbeats as the baseline,
require at least max(40 ms, 35% of baseline RTT) improvement to switch,
exponentially back off inactive online probes from 2m to 30m while the
active connection keeps winning (resetting on candidate win, reconnect,
connection list changes, or foreground verify), and connect the replacement
before disposing the live client on adaptive switches.

Closes #9149
2026-11-21 11:37:41 +01:00
2 changed files with 672 additions and 69 deletions
+425 -1
View File
@@ -21,6 +21,7 @@ import { useSessionStore, type Agent } from "@/stores/session-store";
import { normalizeAgentSnapshot } from "@/utils/agent-snapshots";
import { isAgentArchiving, setAgentArchiving } from "@/hooks/use-archive-agent";
import { queryClient } from "@/data/query-client";
import { checkoutStatusQueryKey } from "@/git/query-keys";
import {
HostRuntimeController,
HostRuntimeStore,
@@ -1748,7 +1749,7 @@ describe("HostRuntimeController", () => {
});
latencies["relay:relay.paseo.sh:443"] = 30;
await vi.advanceTimersByTimeAsync(120_000);
await vi.advanceTimersByTimeAsync(240_000);
await controller.runProbeCycleNow();
expect(controller.getSnapshot().activeConnectionId).toBe("direct:lan:6767");
@@ -1807,6 +1808,429 @@ describe("HostRuntimeController", () => {
await controller.stop();
});
it("ignores busy active heartbeat spikes when comparing against an idle alternate probe", async () => {
// Replays the recorded 10:04-10:08 relay sequence (#9149): the active relay has a ~250-300 ms
// baseline over its recent heartbeats, spikes to 383/503/491 ms right before each alternate
// probe, and the idle alternate measures 236/322/291 ms.
useHostRuntimeClock();
const host = makeHost({
connections: [relayPrimary, relayAlternate],
preferredConnectionId: relayPrimary.id,
});
const clients: FakeDaemonClient[] = [];
let alternateProbes = 0;
const latencies: Record<string, number | Error> = {
[relayPrimary.id]: 260,
[relayAlternate.id]: 280,
};
const baseDeps = makeDeps(latencies, clients);
const controller = new HostRuntimeController({
host,
deps: {
...baseDeps,
connectToDaemon: async (input) => {
if (input.connection.id === relayAlternate.id) {
alternateProbes += 1;
}
return baseDeps.connectToDaemon(input);
},
},
});
await controller.start({ autoProbe: false });
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
const activeClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
const rounds = [
{ intervalMs: 120_000, baseline: 295, spike: 383, alternate: 236 },
{ intervalMs: 240_000, baseline: 251, spike: 503, alternate: 322 },
{ intervalMs: 480_000, baseline: 341, spike: 491, alternate: 291 },
];
for (const round of rounds) {
await vi.advanceTimersByTimeAsync(round.intervalMs - 10_000);
activeClient.heartbeatReportsRtt(round.baseline);
await controller.runProbeCycleNow();
await vi.advanceTimersByTimeAsync(10_000);
activeClient.heartbeatReportsRtt(round.spike);
latencies[relayAlternate.id] = round.alternate;
await controller.runProbeCycleNow();
}
expect(alternateProbes).toBe(4);
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
expect(controller.getSnapshot().client).toBe(activeClient);
expect(activeClient.isDisposed()).toBe(false);
await controller.stop();
});
it("requires a proportional RTT improvement before switching away from a high-latency link", async () => {
useHostRuntimeClock();
const host = makeHost({
connections: [relayPrimary, relayAlternate],
preferredConnectionId: relayPrimary.id,
});
const clients: FakeDaemonClient[] = [];
const latencies: Record<string, number | Error> = {
[relayPrimary.id]: 300,
[relayAlternate.id]: 320,
};
const controller = new HostRuntimeController({
host,
deps: makeDeps(latencies, clients),
});
await controller.start({ autoProbe: false });
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
const activeClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
// 60 ms faster on a 300 ms link (20%) is noise, not enough to justify a switch.
activeClient.heartbeatReportsRtt(300);
latencies[relayAlternate.id] = 240;
for (let index = 0; index < 3; index += 1) {
await vi.advanceTimersByTimeAsync(30 * 60_000);
await controller.runProbeCycleNow();
}
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
// 150 ms faster on a 300 ms link (50%) is a real win and switches after 3 consecutive probes.
latencies[relayAlternate.id] = 150;
for (let index = 0; index < 3; index += 1) {
await vi.advanceTimersByTimeAsync(index === 0 ? 30 * 60_000 : 120_000);
await controller.runProbeCycleNow();
}
expect(controller.getSnapshot().activeConnectionId).toBe(relayAlternate.id);
await controller.stop();
});
it("exponentially backs off inactive connection probes while the active connection keeps winning", async () => {
useHostRuntimeClock();
const host = makeHost({
connections: [relayPrimary, relayAlternate],
preferredConnectionId: relayPrimary.id,
});
const clients: FakeDaemonClient[] = [];
let alternateProbes = 0;
const baseDeps = makeDeps(
{
[relayPrimary.id]: 250,
[relayAlternate.id]: 260,
},
clients,
);
const controller = new HostRuntimeController({
host,
deps: {
...baseDeps,
connectToDaemon: async (input) => {
if (input.connection.id === relayAlternate.id) {
alternateProbes += 1;
}
return baseDeps.connectToDaemon(input);
},
},
});
await controller.start({ autoProbe: false });
const activeClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
activeClient.heartbeatReportsRtt(250);
expect(alternateProbes).toBe(1);
// First inactive online probe runs at 2m -> active wins, next interval backs off to 4m.
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
expect(alternateProbes).toBe(2);
// At +2m (still < 4m), no inactive probe runs.
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
expect(alternateProbes).toBe(2);
// At +4m total, second inactive probe runs -> backs off to 8m.
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
expect(alternateProbes).toBe(3);
// At +6m (< 8m), still quiet; at +8m, third inactive probe runs -> backs off to 16m.
await vi.advanceTimersByTimeAsync(6 * 60_000);
await controller.runProbeCycleNow();
expect(alternateProbes).toBe(3);
await vi.advanceTimersByTimeAsync(2 * 60_000);
await controller.runProbeCycleNow();
expect(alternateProbes).toBe(4);
// Foreground verification resets backoff back to 2m without forcing an immediate probe.
await vi.advanceTimersByTimeAsync(3 * 60_000);
controller.ensureConnected({ verify: true });
await controller.runProbeCycleNow();
expect(alternateProbes).toBe(5);
await controller.stop();
});
it("keeps the active connection online while connecting the replacement during an adaptive switch", async () => {
useHostRuntimeClock();
const host = makeHost({
connections: [relayPrimary, relayAlternate],
preferredConnectionId: relayPrimary.id,
});
const clients: FakeDaemonClient[] = [];
const latencies: Record<string, number | Error> = {
[relayPrimary.id]: 300,
[relayAlternate.id]: 320,
};
let replacementConnectGate: ReturnType<typeof createDeferred<void>> | null = null;
let shouldFailReplacement = false;
const baseDeps = makeDeps(latencies, clients);
const controller = new HostRuntimeController({
host,
deps: {
...baseDeps,
createClient: (input) => {
const client = baseDeps.createClient(input) as unknown as FakeDaemonClient;
if (input.connection.id === relayAlternate.id) {
const origConnect = client.connect.bind(client);
client.connect = async () => {
if (replacementConnectGate) {
await replacementConnectGate.promise;
}
if (shouldFailReplacement) {
throw new Error("replacement handshake failed");
}
return origConnect();
};
}
return client as unknown as DaemonClient;
},
},
});
await controller.start({ autoProbe: false });
const initialClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
initialClient.heartbeatReportsRtt(300);
latencies[relayAlternate.id] = 100;
const observedSnapshots: Array<{
status: string;
connectionId: string | null;
hasClient: boolean;
}> = [];
const unsubscribe = controller.subscribe(() => {
const snap = controller.getSnapshot();
observedSnapshots.push({
status: snap.connectionStatus,
connectionId: snap.activeConnectionId,
hasClient: snap.client !== null,
});
});
// Two wins first.
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
// Third win triggers a switch, but replacement connect fails: active client must stay online.
shouldFailReplacement = true;
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
expect(controller.getSnapshot().connectionStatus).toBe("online");
expect(controller.getSnapshot().client).toBe(initialClient);
expect(initialClient.isDisposed()).toBe(false);
// Now let replacement succeed after a delay: host must stay online on initialClient while
// replacement connects, and never emit connecting or client=null.
shouldFailReplacement = false;
observedSnapshots.length = 0;
for (let index = 0; index < 2; index += 1) {
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
}
replacementConnectGate = createDeferred<void>();
await vi.advanceTimersByTimeAsync(120_000);
const switchingCycle = controller.runProbeCycleNow();
await vi.advanceTimersByTimeAsync(0);
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
expect(controller.getSnapshot().connectionStatus).toBe("online");
expect(controller.getSnapshot().client).toBe(initialClient);
expect(initialClient.isDisposed()).toBe(false);
replacementConnectGate.resolve();
await switchingCycle;
expect(controller.getSnapshot().activeConnectionId).toBe(relayAlternate.id);
expect(controller.getSnapshot().connectionStatus).toBe("online");
expect(controller.getSnapshot().client).not.toBeNull();
expect(controller.getSnapshot().client).not.toBe(initialClient);
expect(initialClient.isDisposed()).toBe(true);
expect(observedSnapshots.some((snap) => snap.status !== "online" || !snap.hasClient)).toBe(
false,
);
unsubscribe();
await controller.stop();
});
it("keeps active client status updates working after a failed make-before-break upgrade", async () => {
useHostRuntimeClock();
const host = makeHost({
connections: [relayPrimary, relayAlternate],
preferredConnectionId: relayPrimary.id,
});
const clients: FakeDaemonClient[] = [];
const latencies: Record<string, number | Error> = {
[relayPrimary.id]: 300,
[relayAlternate.id]: 320,
};
const baseDeps = makeDeps(latencies, clients);
const controller = new HostRuntimeController({
host,
deps: {
...baseDeps,
createClient: (input) => {
const client = baseDeps.createClient(input) as unknown as FakeDaemonClient;
if (input.connection.id === relayAlternate.id) {
client.connect = async () => {
throw new Error("replacement handshake failed");
};
}
return client as unknown as DaemonClient;
},
},
});
await controller.start({ autoProbe: false });
const initialClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
initialClient.heartbeatReportsRtt(300);
latencies[relayAlternate.id] = 100;
for (let index = 0; index < 3; index += 1) {
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
}
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
expect(controller.getSnapshot().connectionStatus).toBe("online");
initialClient.setConnectionState({ status: "disconnected", reason: "transport closed" });
expect(controller.getSnapshot().connectionStatus).toBe("error");
expect(controller.getSnapshot().lastError).toBe("transport closed");
await controller.stop();
});
it("aborts a pending make-before-break upgrade when the candidate connection is removed", async () => {
useHostRuntimeClock();
const host = makeHost({
connections: [relayPrimary, relayAlternate],
preferredConnectionId: relayPrimary.id,
});
const clients: FakeDaemonClient[] = [];
const latencies: Record<string, number | Error> = {
[relayPrimary.id]: 300,
[relayAlternate.id]: 320,
};
const replacementGate = createDeferred<void>();
const replacementClients: FakeDaemonClient[] = [];
const baseDeps = makeDeps(latencies, clients);
const controller = new HostRuntimeController({
host,
deps: {
...baseDeps,
createClient: (input) => {
const client = baseDeps.createClient(input) as unknown as FakeDaemonClient;
if (input.connection.id === relayAlternate.id) {
replacementClients.push(client);
const origConnect = client.connect.bind(client);
client.connect = async () => {
await replacementGate.promise;
return origConnect();
};
}
return client as unknown as DaemonClient;
},
},
});
await controller.start({ autoProbe: false });
const initialClient = controller.getSnapshot().client as unknown as FakeDaemonClient;
initialClient.heartbeatReportsRtt(300);
latencies[relayAlternate.id] = 100;
for (let index = 0; index < 2; index += 1) {
await vi.advanceTimersByTimeAsync(120_000);
await controller.runProbeCycleNow();
}
await vi.advanceTimersByTimeAsync(120_000);
const switchingCycle = controller.runProbeCycleNow();
await vi.advanceTimersByTimeAsync(0);
expect(replacementClients).toHaveLength(1);
const updatePromise = controller.updateHost({
...host,
connections: [relayPrimary],
});
await vi.advanceTimersByTimeAsync(0);
replacementGate.resolve();
await Promise.all([switchingCycle, updatePromise]);
expect(controller.getSnapshot().activeConnectionId).toBe(relayPrimary.id);
expect(controller.getSnapshot().connectionStatus).toBe("online");
expect(controller.getSnapshot().client).toBe(initialClient);
expect(initialClient.isDisposed()).toBe(false);
expect(replacementClients[0]?.isDisposed()).toBe(true);
await controller.stop();
});
it("runs store cache recovery when a make-before-break adaptive switch advances the online epoch", async () => {
useHostRuntimeClock();
const host = makeHost({
serverId: "srv_upgrade_cache_recovery",
connections: [relayPrimary, relayAlternate],
preferredConnectionId: relayPrimary.id,
});
const clients: FakeDaemonClient[] = [];
const latencies: Record<string, number | Error> = {
[relayPrimary.id]: 300,
[relayAlternate.id]: 320,
};
const store = new HostRuntimeStore({
storage: createMemoryHostRuntimeStorage(),
deps: makeDeps(latencies, clients),
});
const queryKey = checkoutStatusQueryKey(host.serverId, "/repo");
try {
store.syncHosts([host]);
await vi.advanceTimersByTimeAsync(0);
expect(store.getSnapshot(host.serverId)?.connectionStatus).toBe("online");
const initialEpoch = store.getSnapshot(host.serverId)?.connectionEpoch ?? 0;
const activeClient = store.getSnapshot(host.serverId)?.client as unknown as FakeDaemonClient;
const initialHistoryGen =
useSessionStore.getState().sessions[host.serverId]?.historySyncGeneration ?? 0;
queryClient.setQueryData(queryKey, { branch: "main" });
expect(queryClient.getQueryState(queryKey)?.isInvalidated).toBe(false);
activeClient.heartbeatReportsRtt(300);
latencies[relayAlternate.id] = 100;
for (let index = 0; index < 3; index += 1) {
await vi.advanceTimersByTimeAsync(120_000);
await store.runProbeCycleNow(host.serverId);
}
expect(store.getSnapshot(host.serverId)?.activeConnectionId).toBe(relayAlternate.id);
expect(store.getSnapshot(host.serverId)?.connectionStatus).toBe("online");
expect(store.getSnapshot(host.serverId)?.connectionEpoch).toBeGreaterThan(initialEpoch);
expect(queryClient.getQueryState(queryKey)?.isInvalidated).toBe(true);
expect(
useSessionStore.getState().sessions[host.serverId]?.historySyncGeneration ?? 0,
).toBeGreaterThan(initialHistoryGen);
} finally {
queryClient.removeQueries({ queryKey });
store.syncHosts([]);
}
});
it("exposes one snapshot with active connection and status from same source", async () => {
const host = makeHost();
const clients: FakeDaemonClient[] = [];
+247 -68
View File
@@ -245,8 +245,11 @@ export interface HostRuntimeStartOptions {
const PROBE_TICK_MS = 2_000;
const PROBE_STEADY_MS = 10_000;
const PROBE_MAX_BACKOFF_MS = 30_000;
const PROBE_INACTIVE_WHILE_ONLINE_MS = 120_000;
const PROBE_INACTIVE_WHILE_ONLINE_INITIAL_MS = 120_000;
const PROBE_INACTIVE_WHILE_ONLINE_MAX_MS = 30 * 60_000;
const ACTIVE_RTT_WINDOW_MS = 60_000;
const ADAPTIVE_SWITCH_THRESHOLD_MS = 40;
const ADAPTIVE_SWITCH_MIN_IMPROVEMENT_RATIO = 0.35;
const ADAPTIVE_SWITCH_CONSECUTIVE_PROBES = 3;
const CONFIGURED_OVERRIDE_BOOTSTRAP_RETRY_MS = 1_000;
@@ -549,13 +552,14 @@ function probeIntervalForConnection(
firstSeenAt: number,
isActiveOnline: boolean,
hasActiveOnlineConnection: boolean,
inactiveOnlineIntervalMs: number,
now: number,
): number {
if (isActiveOnline) {
return PROBE_STEADY_MS;
}
if (hasActiveOnlineConnection) {
return PROBE_INACTIVE_WHILE_ONLINE_MS;
return inactiveOnlineIntervalMs;
}
const age = now - firstSeenAt;
if (age < 10_000) return 2_000;
@@ -728,6 +732,8 @@ export class HostRuntimeController {
private connectionLastProbedAt = new Map<string, number>();
private switchCandidateConnectionId: string | null = null;
private switchCandidateHitCount = 0;
private inactiveOnlineProbeBackoffStep = 0;
private activeRttSamples: Array<{ at: number; rttMs: number }> = [];
private clientIdPromise: Promise<string> | null = null;
private clientIdHash: string | null = null;
private switchRequestVersion = 0;
@@ -823,6 +829,9 @@ export class HostRuntimeController {
this.activeClient = null;
await prev.close().catch(() => undefined);
}
this.resetSwitchCandidate();
this.clearActiveRttSamples();
this.resetInactiveOnlineProbeBackoff();
this.applyConnectionEvent({ type: "stopped" });
this.updateSnapshot({
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
@@ -832,11 +841,15 @@ export class HostRuntimeController {
async updateHost(host: HostProfile): Promise<void> {
const passwordChanged = this.host.password !== host.password;
const connectionsChanged = !equal(this.host.connections, host.connections);
if (passwordChanged) {
this.probeRequestVersion += 1;
this.authRejectedConnectionIds.clear();
this.updateSnapshot({ authFailureReason: null });
}
if (passwordChanged || connectionsChanged) {
this.resetInactiveOnlineProbeBackoff();
}
const activeConnectionId = this.snapshot.activeConnectionId;
const previousActiveConnection = findConnectionById(this.host, activeConnectionId);
this.host = host;
@@ -862,6 +875,9 @@ export class HostRuntimeController {
}
ensureConnected(options?: { verify?: boolean }): void {
if (options?.verify) {
this.resetInactiveOnlineProbeBackoff();
}
this.activeClient?.ensureConnected(options);
}
@@ -962,6 +978,7 @@ export class HostRuntimeController {
const activeConnectionId = this.snapshot.activeConnectionId;
const hasActiveOnlineConnection = isOnline && activeConnectionId !== null;
const inactiveOnlineIntervalMs = this.inactiveOnlineProbeIntervalMs();
const connectionsToProbe = this.host.connections.filter((connection) => {
if (this.authRejectedConnectionIds.has(connection.id)) return false;
const lastProbed = this.connectionLastProbedAt.get(connection.id);
@@ -974,6 +991,7 @@ export class HostRuntimeController {
firstSeen,
isActiveOnline,
hasActiveOnlineConnection,
inactiveOnlineIntervalMs,
now,
);
return now - lastProbed >= interval;
@@ -1081,10 +1099,10 @@ export class HostRuntimeController {
// Adaptive switching counts fresh comparisons only. The probe map keeps the last known
// sample per connection for display and failover; a streak hit needs a candidate measured
// in this cycle and a heartbeat RTT from the socket that is active right now. Re-reading
// the cached map at every ten-second active tick turned one idle-probe sample into three
// "consecutive" wins, and a reconnected socket inherited its predecessor's RTT, so a
// working relay connection was replaced within seconds of coming online.
// in this cycle and a baseline RTT from the socket that is active right now. Using the
// rolling minimum of recent active heartbeats filters out single busy/stalled ticks so a
// loaded working socket does not lose to an idle probe, and a proportional threshold avoids
// flapping on normal jitter on high-RTT links.
const sampledCandidates = connectionsToProbe.filter(
(connection) => connection.id !== currentActiveConnectionId,
);
@@ -1102,22 +1120,29 @@ export class HostRuntimeController {
})
.sort((left, right) => left.latencyMs - right.latencyMs)[0] ?? null;
if (!fastest) {
// Every candidate sampled this cycle failed: that breaks the streak.
// Every candidate sampled this cycle failed: that breaks the streak and backs off.
this.resetSwitchCandidate();
this.advanceInactiveOnlineProbeBackoff();
return;
}
const activeRttMs = this.currentActiveSocketRttMs();
const activeRttMs = this.currentActiveSocketBaselineRttMs(performance.now());
if (activeRttMs === null) {
return;
}
const requiredImprovementMs = Math.max(
ADAPTIVE_SWITCH_THRESHOLD_MS,
activeRttMs * ADAPTIVE_SWITCH_MIN_IMPROVEMENT_RATIO,
);
const improvement = activeRttMs - fastest.latencyMs;
if (improvement < ADAPTIVE_SWITCH_THRESHOLD_MS) {
if (improvement < requiredImprovementMs) {
this.resetSwitchCandidate();
this.advanceInactiveOnlineProbeBackoff();
return;
}
this.resetInactiveOnlineProbeBackoff();
if (this.switchCandidateConnectionId === fastest.connectionId) {
this.switchCandidateHitCount += 1;
} else {
@@ -1130,6 +1155,7 @@ export class HostRuntimeController {
await this.switchToConnection({
connectionId: fastest.connectionId,
expectedProbeVersion: requestVersion,
keepActiveUntilConnected: true,
});
}
};
@@ -1194,6 +1220,7 @@ export class HostRuntimeController {
return;
}
if (rttMs !== null) {
this.recordActiveRttSample(rttMs, performance.now());
probeByConnectionId.set(connection.id, {
status: "available",
latencyMs: rttMs,
@@ -1245,6 +1272,45 @@ export class HostRuntimeController {
return client.getLastLivenessRttMs();
}
private recordActiveRttSample(rttMs: number, now: number): void {
this.activeRttSamples = this.activeRttSamples.filter(
(sample) => now - sample.at <= ACTIVE_RTT_WINDOW_MS,
);
this.activeRttSamples.push({ at: now, rttMs });
}
private clearActiveRttSamples(): void {
this.activeRttSamples = [];
}
private currentActiveSocketBaselineRttMs(now: number): number | null {
const currentRttMs = this.currentActiveSocketRttMs();
if (currentRttMs === null) return null;
this.recordActiveRttSample(currentRttMs, now);
let baseline = this.activeRttSamples[0]!.rttMs;
for (const sample of this.activeRttSamples) {
if (sample.rttMs < baseline) baseline = sample.rttMs;
}
return baseline;
}
private inactiveOnlineProbeIntervalMs(): number {
return Math.min(
PROBE_INACTIVE_WHILE_ONLINE_INITIAL_MS * 2 ** this.inactiveOnlineProbeBackoffStep,
PROBE_INACTIVE_WHILE_ONLINE_MAX_MS,
);
}
private advanceInactiveOnlineProbeBackoff(): void {
if (this.inactiveOnlineProbeIntervalMs() < PROBE_INACTIVE_WHILE_ONLINE_MAX_MS) {
this.inactiveOnlineProbeBackoffStep += 1;
}
}
private resetInactiveOnlineProbeBackoff(): void {
this.inactiveOnlineProbeBackoffStep = 0;
}
private resetSwitchCandidate(): void {
this.switchCandidateConnectionId = null;
this.switchCandidateHitCount = 0;
@@ -1434,6 +1500,7 @@ export class HostRuntimeController {
connectionId: string;
expectedProbeVersion?: number;
existingClient?: DaemonClient;
keepActiveUntilConnected?: boolean;
}): Promise<void> {
const { connectionId, expectedProbeVersion, existingClient } = input;
if (!this.canProceedForProbe(expectedProbeVersion)) {
@@ -1455,6 +1522,24 @@ export class HostRuntimeController {
return;
}
const makeBeforeBreak =
Boolean(input.keepActiveUntilConnected) &&
!existingClient &&
this.activeClient !== null &&
this.connectionMachineState.tag === "online";
const nextGeneration = this.snapshot.clientGeneration + 1;
if (makeBeforeBreak) {
await this.switchToConnectionMakeBeforeBreak({
connection,
clientId,
nextGeneration,
requestVersion,
expectedProbeVersion,
});
return;
}
await this.disposePreviousActiveClient();
if (!this.isSwitchStillValid(requestVersion, expectedProbeVersion)) {
@@ -1462,7 +1547,6 @@ export class HostRuntimeController {
return;
}
const nextGeneration = this.snapshot.clientGeneration + 1;
const client =
existingClient ??
this.deps.createClient({
@@ -1479,6 +1563,8 @@ export class HostRuntimeController {
}
this.activeClient = client;
this.clearActiveRttSamples();
this.resetInactiveOnlineProbeBackoff();
this.applyConnectionEvent({
type: "select_connection",
connectionId: connection.id,
@@ -1490,67 +1576,148 @@ export class HostRuntimeController {
client: null,
});
const failConnection = async (error: unknown) => {
if (!this.isCurrentSwitchRequest(requestVersion) || this.activeClient !== client) return;
const authFailure = getConnectionAuthFailureReason(error);
if (authFailure) this.authRejectedConnectionIds.add(connection.id);
this.unsubscribeClientStatus?.();
this.unsubscribeClientStatus = null;
this.unsubscribeClientHandlers?.();
this.unsubscribeClientHandlers = null;
client.setReconnectEnabled(false);
this.applyConnectionEvent({ type: "connect_failed", message: toErrorMessage(error) });
this.updateSnapshot({
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
client: null,
authFailureReason: authFailure,
});
if (authFailure) {
const probes = new Map(this.snapshot.probeByConnectionId);
probes.set(connection.id, { status: "unavailable", latencyMs: null });
this.updateSnapshot({ probeByConnectionId: probes });
for (const candidate of this.host.connections) {
if (!this.authRejectedConnectionIds.has(candidate.id)) {
this.connectionLastProbedAt.delete(candidate.id);
}
}
}
try {
await client.close();
} catch {
/* Preserve the compatibility/connection error. */
}
if (
authFailure &&
this.host.connections.some((candidate) => !this.authRejectedConnectionIds.has(candidate.id))
) {
void this.runProbeCycleNow();
}
};
try {
if (!existingClient) await client.connect();
if (!this.isCurrentSwitchRequest(requestVersion) || this.activeClient !== client) return;
this.unsubscribeClientHandlers =
this.deps.mountClientHandlers?.({ client, host: this.host, connection }) ?? null;
this.unsubscribeClientStatus = client.subscribeConnectionStatus((state) => {
if (!this.isCurrentSwitchRequest(requestVersion) || this.activeClient !== client) return;
if (client.authFailureReason) {
void failConnection(new DaemonAuthenticationError(client.authFailureReason));
return;
}
if (state.status === "connected") this.learnRelaySecurity(client, connection.id);
// Switch evidence belongs to the socket it was measured against.
if (state.status !== "connected") this.resetSwitchCandidate();
this.applyConnectionEvent({ type: "client_state", state, lastError: client.lastError });
this.updateSnapshot({
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
...this.buildAgentDirectoryStatusPatch(),
client,
authFailureReason: null,
});
});
this.attachActiveClient({ client, connection });
} catch (error) {
await failConnection(error);
await this.failActiveConnection({ client, connection, error });
}
}
private async switchToConnectionMakeBeforeBreak(input: {
connection: HostConnection;
clientId: string;
nextGeneration: number;
requestVersion: number;
expectedProbeVersion?: number;
}): Promise<void> {
const { connection, clientId, nextGeneration, requestVersion, expectedProbeVersion } = input;
const replacement = this.deps.createClient({
host: this.host,
connection,
clientId,
runtimeGeneration: nextGeneration,
});
replacement.setReconnectEnabled(false);
try {
await replacement.connect();
} catch (error) {
const authFailure = getConnectionAuthFailureReason(error);
if (authFailure) {
this.authRejectedConnectionIds.add(connection.id);
const probes = new Map(this.snapshot.probeByConnectionId);
probes.set(connection.id, { status: "unavailable", latencyMs: null });
this.updateSnapshot({ probeByConnectionId: probes });
}
await replacement.close().catch(() => undefined);
return;
}
const resolveValidTarget = (): HostConnection | null => {
if (!this.isSwitchStillValid(requestVersion, expectedProbeVersion)) return null;
const saved = findConnectionById(this.host, connection.id);
if (!saved || connectionNeedsNewClient(connection, saved, replacement)) return null;
return saved;
};
if (!resolveValidTarget()) {
await replacement.close().catch(() => undefined);
return;
}
replacement.setReconnectEnabled(true);
await this.disposePreviousActiveClient();
const target = resolveValidTarget();
if (!target) {
await replacement.close().catch(() => undefined);
return;
}
this.activeClient = replacement;
this.clearActiveRttSamples();
this.resetInactiveOnlineProbeBackoff();
this.applyConnectionEvent({
type: "select_connection",
connectionId: target.id,
connection: toActiveConnection(target),
});
this.snapshot = { ...this.snapshot, clientGeneration: nextGeneration };
this.attachActiveClient({ client: replacement, connection: target });
}
private attachActiveClient(input: { client: DaemonClient; connection: HostConnection }): void {
const { client, connection } = input;
this.unsubscribeClientHandlers =
this.deps.mountClientHandlers?.({ client, host: this.host, connection }) ?? null;
this.unsubscribeClientStatus = client.subscribeConnectionStatus((state) => {
if (this.activeClient !== client) return;
if (client.authFailureReason) {
void this.failActiveConnection({
client,
connection,
error: new DaemonAuthenticationError(client.authFailureReason),
});
return;
}
if (state.status === "connected") this.learnRelaySecurity(client, connection.id);
// Switch evidence belongs to the socket it was measured against.
if (state.status !== "connected") {
this.resetSwitchCandidate();
this.clearActiveRttSamples();
this.resetInactiveOnlineProbeBackoff();
}
this.applyConnectionEvent({ type: "client_state", state, lastError: client.lastError });
this.updateSnapshot({
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
...this.buildAgentDirectoryStatusPatch(),
client,
authFailureReason: null,
});
});
}
private async failActiveConnection(input: {
client: DaemonClient;
connection: HostConnection;
error: unknown;
}): Promise<void> {
const { client, connection, error } = input;
if (this.activeClient !== client) return;
const authFailure = getConnectionAuthFailureReason(error);
if (authFailure) this.authRejectedConnectionIds.add(connection.id);
this.unsubscribeClientStatus?.();
this.unsubscribeClientStatus = null;
this.unsubscribeClientHandlers?.();
this.unsubscribeClientHandlers = null;
client.setReconnectEnabled(false);
this.applyConnectionEvent({ type: "connect_failed", message: toErrorMessage(error) });
this.updateSnapshot({
...toSnapshotConnectionPatch(this.connectionMachineState, this.connectionEpoch),
client: null,
authFailureReason: authFailure,
});
if (authFailure) {
const probes = new Map(this.snapshot.probeByConnectionId);
probes.set(connection.id, { status: "unavailable", latencyMs: null });
this.updateSnapshot({ probeByConnectionId: probes });
for (const candidate of this.host.connections) {
if (!this.authRejectedConnectionIds.has(candidate.id)) {
this.connectionLastProbedAt.delete(candidate.id);
}
}
}
try {
await client.close();
} catch {
/* Preserve the compatibility/connection error. */
}
if (
authFailure &&
this.host.connections.some((candidate) => !this.authRejectedConnectionIds.has(candidate.id))
) {
void this.runProbeCycleNow();
}
}
@@ -1693,6 +1860,7 @@ export class HostRuntimeStore {
private hostRegistryStatus: HostRegistryStatus = "loading";
private deps: HostRuntimeControllerDeps;
private lastConnectionStatusByServer = new Map<string, HostRuntimeConnectionStatus>();
private lastOnlineEpochByServer = new Map<string, number>();
private connectionStatusStartedAtByServer = new Map<string, number>();
private queuedAgentDrainInFlight = new Set<string>();
private directorySyncByServer = new Map<string, DirectorySync>();
@@ -1950,6 +2118,7 @@ export class HostRuntimeStore {
rekeyMap(this.controllers, oldServerId, newServerId);
rekeyMap(this.lastConnectionStatusByServer, oldServerId, newServerId);
rekeyMap(this.lastOnlineEpochByServer, oldServerId, newServerId);
rekeyMap(this.connectionStatusStartedAtByServer, oldServerId, newServerId);
this.replicaCache.reconcileServerId(oldServerId, newServerId);
projectIconCache.reconcileServerId(oldServerId, newServerId);
@@ -2442,6 +2611,7 @@ export class HostRuntimeStore {
this.fleetDirectoryDemand.delete(serverId);
this.controllers.delete(serverId);
this.lastConnectionStatusByServer.delete(serverId);
this.lastOnlineEpochByServer.delete(serverId);
this.connectionStatusStartedAtByServer.delete(serverId);
this.directorySyncByServer.get(serverId)?.dispose();
this.directorySyncByServer.delete(serverId);
@@ -2492,6 +2662,9 @@ export class HostRuntimeStore {
);
const initialSnapshot = controller.getSnapshot();
this.lastConnectionStatusByServer.set(host.serverId, initialSnapshot.connectionStatus);
if (initialSnapshot.connectionStatus === "online") {
this.lastOnlineEpochByServer.set(host.serverId, initialSnapshot.connectionEpoch);
}
this.connectionStatusStartedAtByServer.set(host.serverId, Date.now());
controller.subscribe(() => {
const snapshot = controller.getSnapshot();
@@ -2591,6 +2764,7 @@ export class HostRuntimeStore {
const controller = this.controllers.get(serverId);
if (!controller) {
this.lastConnectionStatusByServer.delete(serverId);
this.lastOnlineEpochByServer.delete(serverId);
this.connectionStatusStartedAtByServer.delete(serverId);
return;
}
@@ -2605,6 +2779,7 @@ export class HostRuntimeStore {
},
});
const previousStatus = this.lastConnectionStatusByServer.get(serverId);
const previousOnlineEpoch = this.lastOnlineEpochByServer.get(serverId);
const statusChanged = previousStatus !== snapshot.connectionStatus;
const isUnavailable =
snapshot.connectionStatus !== "online" && snapshot.connectionStatus !== "idle";
@@ -2615,7 +2790,11 @@ export class HostRuntimeStore {
}
this.lastConnectionStatusByServer.set(serverId, snapshot.connectionStatus);
const didTransitionOnline =
snapshot.connectionStatus === "online" && previousStatus !== "online";
snapshot.connectionStatus === "online" &&
(previousStatus !== "online" || previousOnlineEpoch !== snapshot.connectionEpoch);
if (snapshot.connectionStatus === "online") {
this.lastOnlineEpochByServer.set(serverId, snapshot.connectionEpoch);
}
if (didTransitionOnline) {
useSessionStore.getState().bumpHistorySyncGeneration(serverId);
for (const child of this.controllers.values()) void child.reconnectVia(serverId);