common/hermes
2
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(gateway): drain-aware hermes update + faster still-working pings (#14736)

Browse Source 
cmd_update no longer SIGKILLs in-flight agent runs, and users get
'still working' status every 3 min instead of 10. Two long-standing
sources of '@user — agent gives up mid-task' reports on Telegram and
other gateways.

Drain-aware update:
- New helper hermes_cli.gateway._graceful_restart_via_sigusr1(pid,
  drain_timeout) sends SIGUSR1 to the gateway and polls os.kill(pid,
  0) until the process exits or the budget expires.
- cmd_update's systemd loop now reads MainPID via 'systemctl show
  --property=MainPID --value' and tries the graceful path first. The
  gateway's existing SIGUSR1 handler -> request_restart(via_service=
  True) -> drain -> exit(75) is wired in gateway/run.py and is
  respawned by systemd's Restart=on-failure (and the explicit
  RestartForceExitStatus=75 on newer units).
- Falls back to 'systemctl restart' when MainPID is unknown, the
  drain budget elapses, or the unit doesn't respawn after exit (older
  units missing Restart=on-failure). Old install behavior preserved.
- Drain budget = max(restart_drain_timeout, 30s) + 15s margin so the
  drain loop in run_agent + final exit have room before fallback
  fires. Composes with #14728's tool-subprocess reaping.

Notification interval:
- agent.gateway_notify_interval default 600 -> 180.
- HERMES_AGENT_NOTIFY_INTERVAL env-var fallback in gateway/run.py
  matched.
- 9-minute weak-model spinning runs now ping at 3 min and 6 min
  instead of 27 seconds before completion, removing the 'is the bot
  dead?' reflex that drives gateway-restart cycles.

Tests:
- Two new tests in tests/hermes_cli/test_update_gateway_restart.py:
  one asserts SIGUSR1 is sent and 'systemctl restart' is NOT called
  when MainPID is known and the helper succeeds; one asserts the
  fallback fires when the helper returns False.
- E2E: spawned detached bash processes confirm the helper returns
  True on SIGUSR1-handling exit (~0.5s) and False on SIGUSR1-ignoring
  processes (timeout). Verified non-existent PID and pid=0 edge cases.
- 41/41 in test_update_gateway_restart.py (was 39, +2 new).
- 154/154 in shutdown-related suites including #14728's new tests.

Reported by @GeoffWellman and @ANT_1515 on X.
This commit is contained in:
Teknium
2026-04-23 14:01:57 -07:00
committed by GitHub
parent 165b2e481a
commit 97b9b3d6a6
5 changed files with 339 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
tests/hermes_cli/test_update_gateway_restart.py
View File

@@ -422,6 +422,152 @@ class TestCmdUpdateLaunchdRestart:
]
assert len(restart_calls) == 1
@patch("shutil.which", return_value=None)
@patch("subprocess.run")
def test_update_prefers_sigusr1_over_systemctl_restart_when_mainpid_known(
self, mock_run, _mock_which, mock_args, capsys, monkeypatch,
):
"""Drain-aware update: when systemctl show reports a MainPID, the
update path sends SIGUSR1 and waits for graceful exit + respawn,
instead of ``systemctl restart`` (which SIGKILLs in-flight agents).
"""
monkeypatch.setattr(gateway_cli, "is_macos", lambda: False)
monkeypatch.setattr(gateway_cli, "supports_systemd_services", lambda: True)
monkeypatch.setattr(gateway_cli, "is_termux", lambda: False)
# Track state: before kill → "active" (old PID),
# after kill + exit → briefly inactive, then "active" again (new PID).
state = {"killed": False}
def side_effect(cmd, **kwargs):
joined = " ".join(str(c) for c in cmd)
if "rev-parse" in joined and "--abbrev-ref" in joined:
return subprocess.CompletedProcess(cmd, 0, stdout="main\n", stderr="")
if "rev-parse" in joined and "--verify" in joined:
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
if "rev-list" in joined:
return subprocess.CompletedProcess(cmd, 0, stdout="3\n", stderr="")
# Only expose a user-scope service.
if "systemctl" in joined and "list-units" in joined:
if "--user" in joined:
return subprocess.CompletedProcess(
cmd, 0,
stdout="hermes-gateway.service loaded active running\n",
stderr="",
)
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
if "systemctl" in joined and "is-active" in joined:
# Pre-kill: active. Post-kill: active again (respawned by
# Restart=on-failure). The drain loop verifies liveness
# separately via os.kill(pid, 0).
return subprocess.CompletedProcess(cmd, 0, stdout="active\n", stderr="")
# The new code path.
if "systemctl" in joined and "show" in joined and "MainPID" in joined:
return subprocess.CompletedProcess(cmd, 0, stdout="4242\n", stderr="")
# If systemctl restart is called, this test fails its intent —
# but still let it succeed so we can assert it was NOT called.
if "systemctl" in joined and "restart" in joined:
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="")
mock_run.side_effect = side_effect
# Track SIGUSR1 delivery and simulate the gateway draining + exiting.
sigusr1_sent = {"value": False}
def fake_kill(pid, sig):
import signal as _s
if pid == 4242 and sig == _s.SIGUSR1:
sigusr1_sent["value"] = True
state["killed"] = True
return
if pid == 4242 and sig == 0:
# Liveness probe — report dead once SIGUSR1 has been sent.
if state["killed"]:
raise ProcessLookupError()
return
# For any other PID/sig combination, succeed silently.
return
monkeypatch.setattr("os.kill", fake_kill)
with patch.object(gateway_cli, "find_gateway_pids", return_value=[]):
cmd_update(mock_args)
# SIGUSR1 must have been delivered to the gateway MainPID.
assert sigusr1_sent["value"], "Expected SIGUSR1 to be sent to MainPID"
# And `systemctl restart` must NOT have been used (that's the
# non-draining kill-everything path we're moving away from).
restart_calls = [
c for c in mock_run.call_args_list
if "systemctl" in " ".join(str(a) for a in c.args[0])
and "restart" in " ".join(str(a) for a in c.args[0])
]
assert restart_calls == [], (
"Graceful SIGUSR1 succeeded; `systemctl restart` should not "
f"have been called. Got: {restart_calls}"
)
captured = capsys.readouterr().out
assert "draining" in captured.lower()
assert "Restarted hermes-gateway" in captured
@patch("shutil.which", return_value=None)
@patch("subprocess.run")
def test_update_falls_back_to_systemctl_restart_when_sigusr1_times_out(
self, mock_run, _mock_which, mock_args, capsys, monkeypatch,
):
"""If the gateway doesn't exit within the drain budget (e.g. old unit
missing ``Restart=on-failure`` or an agent ignoring SIGUSR1), the
update path falls back to ``systemctl restart``.
"""
monkeypatch.setattr(gateway_cli, "is_macos", lambda: False)
monkeypatch.setattr(gateway_cli, "supports_systemd_services", lambda: True)
monkeypatch.setattr(gateway_cli, "is_termux", lambda: False)
mock_run.side_effect = _make_run_side_effect(
commit_count="3",
systemd_active=True,
)
# Patch systemctl show to report MainPID=4242 so cmd_update attempts
# the graceful path.
orig = mock_run.side_effect
def wrapped(cmd, **kwargs):
joined = " ".join(str(c) for c in cmd)
if "systemctl" in joined and "show" in joined and "MainPID" in joined:
return subprocess.CompletedProcess(cmd, 0, stdout="4242\n", stderr="")
return orig(cmd, **kwargs)
mock_run.side_effect = wrapped
# Simulate the drain helper failing to confirm a clean exit — either
# because the gateway ignored SIGUSR1 or the drain budget was
# exceeded. cmd_update() should detect this and escalate.
monkeypatch.setattr(
"hermes_cli.gateway._graceful_restart_via_sigusr1",
lambda pid, drain_timeout: False,
)
with patch.object(gateway_cli, "find_gateway_pids", return_value=[]):
cmd_update(mock_args)
# Fallback kicked in → systemctl restart was called.
restart_calls = [
c for c in mock_run.call_args_list
if "systemctl" in " ".join(str(a) for a in c.args[0])
and "restart" in " ".join(str(a) for a in c.args[0])
]
assert len(restart_calls) >= 1, (
"Drain path failed; expected fallback `systemctl restart`."
)
@patch("shutil.which", return_value=None)
@patch("subprocess.run")
def test_update_no_gateway_running_skips_restart(