fix(server): time out stalled encrypted relay handshakes (#5648)

packages/server/src/server/relay-transport.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
-import type pino from "pino";
+import pino from "pino";
 import { createClientChannel, type Transport } from "@getpaseo/relay/e2ee";
 import { exportPublicKey, generateKeyPair } from "@getpaseo/relay";
 import { startRelayTransport } from "./relay-transport";
@@ -146,6 +146,37 @@ describe("relay-transport control lifecycle", () => {
     vi.useRealTimers();
   });
 
+  test("releases a stalled encrypted data connection without restarting control", async () => {
+    vi.useFakeTimers();
+    let attachments = 0;
+    controllers.push(
+      startRelayTransport({
+        logger: pino({ level: "silent" }),
+        attachSocket: async () => {
+          attachments += 1;
+        },
+        relayEndpoint: "relay.paseo.sh:443",
+        relayUseTls: true,
+        serverId: "srv_test",
+        daemonKeyPair: generateKeyPair(),
+        createWebSocket: relay.createWebSocket,
+      }),
+    );
+    const control = relay.sockets[0];
+    control.open();
+    const connected = JSON.stringify({ type: "connected", connectionId: "stalled" });
+    control.message(connected);
+    const stalled = relay.sockets[1];
+    stalled.open();
+    await vi.advanceTimersByTimeAsync(15_000);
+    expect(stalled.terminateCalls).toBe(1);
+    expect(attachments).toBe(0);
+    expect(control.terminateCalls).toBe(0);
+    control.message(connected);
+    expect(relay.sockets).toHaveLength(3);
+    expect(relay.sockets[2].url).toBe(stalled.url);
+  });
+
   test("logs relay_control_connected only after first valid control message", () => {
     const logger = createMockLogger();
     const controller = startRelayTransport({

packages/server/src/server/relay-transport.ts

@@ -420,6 +420,10 @@ async function attachEncryptedSocket(
   attachSocket: (ws: RelaySocketLike, metadata?: ExternalSocketMetadata) => Promise<void>,
   metadata?: ExternalSocketMetadata,
 ): Promise<void> {
+  const handshakeTimeout = setTimeout(() => {
+    logger.warn("relay_e2ee_handshake_timeout_terminating");
+    socket.terminate();
+  }, 15_000);
   try {
     const relayTransport = createRelayTransportAdapter(socket, logger);
     const emitter = new EventEmitter();
@@ -440,6 +444,7 @@ async function attachEncryptedSocket(
         emitter.emit("error", error);
       },
     });
+    clearTimeout(handshakeTimeout);
     const encryptedSocket = createEncryptedRelaySocket({
       channel,
       emitter,
@@ -459,6 +464,8 @@ async function attachEncryptedSocket(
     } catch {
       // ignore
     }
+  } finally {
+    clearTimeout(handshakeTimeout);
   }
 }