mirror of
https://github.com/getpaseo/paseo.git
synced 2026-11-19 23:05:21 +00:00
fix(client): recover routed hosts when their gateway reconnects (#9558)
* client: recover routed hosts when their gateway reconnects Assisted-By: OpenAI GPT-6.1 Sol high in omp/18.16.0 * client: wake routed clients before waiting for probes Assisted-By: OpenAI GPT-6.1 Sol high in omp/18.18.9
This commit is contained in:
@@ -2149,6 +2149,68 @@ describe("HostRuntimeController", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("wakes an active routed client before an unrelated direct probe settles", async () => {
|
||||
useHostRuntimeClock();
|
||||
const route: HostConnection = {
|
||||
id: "via:srv_parent",
|
||||
type: "routed",
|
||||
originServerId: "srv_parent",
|
||||
};
|
||||
const host = makeHost({ connections: [route] });
|
||||
const client = makeConnectedProbeClient(5);
|
||||
const directProbeStarted = createDeferred<void>();
|
||||
const directProbe = createDeferred<void>();
|
||||
let directProbeCalls = 0;
|
||||
const controller = new HostRuntimeController({
|
||||
host,
|
||||
deps: {
|
||||
getClientId: async () => "blocked-direct-probe",
|
||||
createClient: () => new FakeDaemonClient() as unknown as DaemonClient,
|
||||
connectToDaemon: async ({ connection }) => {
|
||||
if (connection.type === "directTcp") {
|
||||
directProbeCalls += 1;
|
||||
directProbeStarted.resolve();
|
||||
await directProbe.promise;
|
||||
throw new Error("Direct connection unavailable");
|
||||
}
|
||||
return {
|
||||
client: client as unknown as DaemonClient,
|
||||
serverId: host.serverId,
|
||||
hostname: host.label,
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
await controller.start({ autoProbe: false });
|
||||
client.setConnectionState({
|
||||
status: "disconnected",
|
||||
reason: "origin offline",
|
||||
nextRetryAt: new Date(Date.now() + 12000).toISOString(),
|
||||
});
|
||||
const probing = controller.updateHost({
|
||||
...host,
|
||||
connections: [
|
||||
route,
|
||||
{ id: "direct:unreachable", type: "directTcp", endpoint: "unreachable:6767" },
|
||||
],
|
||||
});
|
||||
await directProbeStarted.promise;
|
||||
const recovering = controller.reconnectVia("srv_parent");
|
||||
try {
|
||||
expect(client.ensureConnectedCalls).toBe(1);
|
||||
expect(controller.getSnapshot().connectionStatus).toBe("online");
|
||||
expect(directProbeCalls).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(client.ensureConnectedCalls).toBe(1);
|
||||
expect(directProbeCalls).toBe(1);
|
||||
} finally {
|
||||
directProbe.resolve();
|
||||
await Promise.all([probing, recovering]);
|
||||
await controller.stop();
|
||||
}
|
||||
expect(directProbeCalls).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps active client generation stable during background probe cycles", async () => {
|
||||
useHostRuntimeClock();
|
||||
const host = makeHost({
|
||||
@@ -2195,6 +2257,146 @@ describe("HostRuntimeController", () => {
|
||||
});
|
||||
|
||||
describe("HostRuntimeStore", () => {
|
||||
function gatewayRecoveryFixture() {
|
||||
const parent = new FakeDaemonClient();
|
||||
parent.connectedHosts = true;
|
||||
parent.hostDirectory = Promise.resolve({
|
||||
self: { serverId: "srv_parent", label: "parent" },
|
||||
hosts: ["srv_child", "srv_healthy"].map((serverId) => ({
|
||||
registrationId: `reg_${serverId}`,
|
||||
serverId,
|
||||
label: serverId,
|
||||
source: { id: "user", key: serverId },
|
||||
metadata: {},
|
||||
enabled: true,
|
||||
status: "online" as const,
|
||||
targetDescription: "relay",
|
||||
lastOnlineAt: null,
|
||||
nextRetryAt: null,
|
||||
error: null,
|
||||
})),
|
||||
});
|
||||
const parentReady = createDeferred<void>();
|
||||
const child = makeConnectedProbeClient(5);
|
||||
const healthy = makeConnectedProbeClient(5);
|
||||
const unrelated = makeConnectedProbeClient(5);
|
||||
const attempts = new Map<string, number>();
|
||||
const hosts = [
|
||||
makeHost({
|
||||
serverId: "srv_parent",
|
||||
connections: [
|
||||
{
|
||||
id: "relay:parent",
|
||||
type: "relay",
|
||||
relayEndpoint: "relay.paseo.sh:443",
|
||||
daemonPublicKeyB64: "pk_parent",
|
||||
},
|
||||
],
|
||||
}),
|
||||
...["srv_child", "srv_healthy"].map((serverId) =>
|
||||
makeHost({
|
||||
serverId,
|
||||
connections: [
|
||||
{
|
||||
id: "via:srv_parent",
|
||||
type: "routed" as const,
|
||||
originServerId: "srv_parent",
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
makeHost({
|
||||
serverId: "srv_unrelated",
|
||||
connections: [
|
||||
{
|
||||
id: "direct:unrelated:6767",
|
||||
type: "directTcp",
|
||||
endpoint: "unrelated:6767",
|
||||
},
|
||||
],
|
||||
}),
|
||||
];
|
||||
const store = new HostRuntimeStore({
|
||||
storage: createMemoryHostRuntimeStorage({
|
||||
"@paseo:e2e": "1",
|
||||
"@paseo:daemon-registry": JSON.stringify(hosts),
|
||||
}),
|
||||
deps: {
|
||||
getClientId: async () => "gateway-recovery",
|
||||
createClient: () => new FakeDaemonClient() as unknown as DaemonClient,
|
||||
connectToDaemon: async ({ host, connection }) => {
|
||||
attempts.set(host.serverId, (attempts.get(host.serverId) ?? 0) + 1);
|
||||
let client: FakeDaemonClient;
|
||||
if (host.serverId === "srv_parent") {
|
||||
await parentReady.promise;
|
||||
parent.setConnectionState({ status: "connected" });
|
||||
client = parent;
|
||||
} else if (connection.type === "routed") {
|
||||
if (!parent.isConnected) throw new Error("Origin host srv_parent is unavailable");
|
||||
client = host.serverId === "srv_child" ? child : healthy;
|
||||
} else {
|
||||
client = unrelated;
|
||||
}
|
||||
return {
|
||||
client: client as unknown as DaemonClient,
|
||||
serverId: host.serverId,
|
||||
hostname: host.label,
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
return { store, parent, parentReady, child, healthy, unrelated, attempts };
|
||||
}
|
||||
|
||||
it("retries saved child routes immediately when their relay gateway first connects", async () => {
|
||||
useHostRuntimeClock();
|
||||
const { store, parentReady, attempts } = gatewayRecoveryFixture();
|
||||
try {
|
||||
await store.boot();
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
expect(attempts.get("srv_child")).toBe(1);
|
||||
expect(store.getSnapshot("srv_child")?.connectionStatus).not.toBe("online");
|
||||
parentReady.resolve();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(store.getSnapshot("srv_parent")?.connectionStatus).toBe("online");
|
||||
expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online");
|
||||
expect(store.getSnapshot("srv_healthy")?.connectionStatus).toBe("online");
|
||||
expect(attempts.get("srv_child")).toBe(2);
|
||||
expect(attempts.get("srv_healthy")).toBe(2);
|
||||
expect(attempts.get("srv_unrelated")).toBe(1);
|
||||
} finally {
|
||||
store.syncHosts([]);
|
||||
}
|
||||
});
|
||||
|
||||
it("wakes disconnected routed clients on gateway recovery without disturbing healthy hosts", async () => {
|
||||
useHostRuntimeClock();
|
||||
const { store, parent, parentReady, child, healthy, unrelated } = gatewayRecoveryFixture();
|
||||
try {
|
||||
parentReady.resolve();
|
||||
await store.boot();
|
||||
await vi.advanceTimersByTimeAsync(2000);
|
||||
expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online");
|
||||
parent.setConnectionState({ status: "disconnected", reason: "relay lost" });
|
||||
child.setConnectionState({
|
||||
status: "disconnected",
|
||||
reason: "origin offline",
|
||||
nextRetryAt: new Date(Date.now() + 12000).toISOString(),
|
||||
});
|
||||
parent.setConnectionState({ status: "connected" });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(child.ensureConnectedCalls).toBe(1);
|
||||
expect(store.getSnapshot("srv_child")?.connectionStatus).toBe("online");
|
||||
expect(healthy.ensureConnectedCalls).toBe(0);
|
||||
expect(unrelated.ensureConnectedCalls).toBe(0);
|
||||
parent.setConnectionState({ status: "connected" });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(child.ensureConnectedCalls).toBe(1);
|
||||
} finally {
|
||||
store.syncHosts([]);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"keeps B and C distinct with routed and direct candidates (B direct first: %s)",
|
||||
async (directFirst) => {
|
||||
|
||||
@@ -869,6 +869,22 @@ export class HostRuntimeController {
|
||||
this.activeClient?.ensureConnected(options);
|
||||
}
|
||||
|
||||
async reconnectVia(originServerId: string): Promise<void> {
|
||||
const routes = this.host.connections.filter(
|
||||
(connection) =>
|
||||
connection.type === "routed" &&
|
||||
connection.originServerId === originServerId &&
|
||||
!this.authRejectedConnectionIds.has(connection.id),
|
||||
);
|
||||
if (routes.length === 0 || !this.started || isHostRuntimeConnected(this.snapshot)) return;
|
||||
if (routes.some((route) => route.id === this.snapshot.activeConnectionId))
|
||||
this.ensureConnected();
|
||||
await this.probeCycleInFlight;
|
||||
if (!this.started || this.snapshot.connectionStatus === "online") return;
|
||||
for (const route of routes) this.connectionLastProbedAt.delete(route.id);
|
||||
await this.runProbeCycleNow();
|
||||
}
|
||||
|
||||
markAgentDirectorySyncLoading(): void {
|
||||
const status = this.snapshot.hasEverLoadedAgentDirectory ? "revalidating" : "initial_loading";
|
||||
this.updateSnapshot({
|
||||
@@ -2601,6 +2617,7 @@ export class HostRuntimeStore {
|
||||
snapshot.connectionStatus === "online" && previousStatus !== "online";
|
||||
if (didTransitionOnline) {
|
||||
useSessionStore.getState().bumpHistorySyncGeneration(serverId);
|
||||
for (const child of this.controllers.values()) void child.reconnectVia(serverId);
|
||||
// Checkout git data is push-driven; pushes emitted while disconnected are gone for
|
||||
// good (the daemon dedupes by snapshot fingerprint). Mark the caches stale so active
|
||||
// queries refetch now and evicted ones on their next mount.
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
type PluginClientBundle,
|
||||
type PluginClientBundleStore,
|
||||
} from "./daemon-client";
|
||||
import { createHostChannelTransport } from "./hosts/index.js";
|
||||
import { CLIENT_CAPS } from "@getpaseo/protocol/client-capabilities";
|
||||
import { ChunkedTransport } from "@getpaseo/protocol/chunked-transport";
|
||||
import { BROWSER_AUTOMATION_COMMAND_NAMES } from "@getpaseo/protocol/browser-automation/rpc-schemas";
|
||||
@@ -9562,7 +9563,6 @@ test.each(["latency", "dictation start", "dictation finish"])(
|
||||
);
|
||||
|
||||
test("routed channels inherit an origin without drain support", async () => {
|
||||
const { createHostChannelTransport } = await import("./hosts/index.js");
|
||||
const mock = createMockTransport();
|
||||
mock.transport.supportsDrain = false;
|
||||
const client = new DaemonClient({
|
||||
@@ -10133,3 +10133,69 @@ describe("provider options across daemon versions", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
test.each([
|
||||
{ name: "route", initialRouteId: null, link: false },
|
||||
{ name: "link", initialRouteId: null, link: true },
|
||||
{ name: "initial route", initialRouteId: "route", link: false },
|
||||
])(
|
||||
"$name fails promptly when its origin is offline during initialization",
|
||||
async ({ initialRouteId, link }) => {
|
||||
vi.useFakeTimers();
|
||||
const origin = new DaemonClient({
|
||||
url: "ws://origin",
|
||||
clientId: "offline-origin",
|
||||
transportFactory: () => createMockTransport().transport,
|
||||
reconnect: { enabled: false },
|
||||
logger: createMockLogger(),
|
||||
});
|
||||
clients.push(origin);
|
||||
const unsubscribe = vi.fn();
|
||||
const subscribe = origin.subscribeConnectionStatus.bind(origin);
|
||||
vi.spyOn(origin, "subscribeConnectionStatus").mockImplementation((listener) => {
|
||||
const release = subscribe(listener);
|
||||
return () => {
|
||||
release();
|
||||
unsubscribe();
|
||||
};
|
||||
});
|
||||
const child = new DaemonClient({
|
||||
url: "ws://routed-host/ws",
|
||||
clientId: "offline-origin-child",
|
||||
transportFactory: createHostChannelTransport(origin, "child", "scope", initialRouteId, link),
|
||||
reconnect: { enabled: false },
|
||||
logger: createMockLogger(),
|
||||
});
|
||||
clients.push(child);
|
||||
const rejected = expect(child.connect()).rejects.toThrow();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(child.getConnectionState().status).toBe("disconnected");
|
||||
await rejected;
|
||||
expect(unsubscribe).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
test("disposing a route before its deferred close does not reopen it on origin recovery", async () => {
|
||||
vi.useFakeTimers();
|
||||
const mock = createMockTransport();
|
||||
const origin = new DaemonClient({
|
||||
url: "ws://origin",
|
||||
clientId: "disposed-route-origin",
|
||||
transportFactory: () => mock.transport,
|
||||
reconnect: { enabled: false },
|
||||
logger: createMockLogger(),
|
||||
});
|
||||
clients.push(origin);
|
||||
const route = createHostChannelTransport(origin, "child", "scope", null)({ url: "ws://route" });
|
||||
const opened = vi.fn();
|
||||
const closed = vi.fn();
|
||||
route.onOpen(opened);
|
||||
route.onClose(closed);
|
||||
route.close();
|
||||
const connecting = origin.connect();
|
||||
mock.triggerOpen({ features: { ownedSubscriptions: true, connectedHosts: true } });
|
||||
await connecting;
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(opened).not.toHaveBeenCalled();
|
||||
expect(closed).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -86,7 +86,8 @@ export function createHostChannelTransport(
|
||||
);
|
||||
cleanups.push(
|
||||
origin.subscribeConnectionStatus((connectionState) => {
|
||||
if (connectionState.status !== "connected") close();
|
||||
// The initial status arrives before the client installs transport handlers.
|
||||
if (connectionState.status !== "connected") queueMicrotask(close);
|
||||
}),
|
||||
);
|
||||
let opening: Promise<string>;
|
||||
|
||||
Reference in New Issue
Block a user