common/hermes
2
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(update): kill stale dashboard processes instead of warning (#17832)

Browse Source 
`hermes update` previously just printed a warning when it detected a
running `hermes dashboard` process from the previous version, telling
the user to kill and restart it themselves.  In practice dashboards get
started and forgotten, so the warning was routinely ignored and users
ended up with a silent frontend/backend mismatch (new JS bundle served
against the old in-memory Python backend, e.g. new auth headers the old
code doesn't recognise → every API call 401s).

The dashboard has no service manager, no PID file, and we don't record
the original launch args (--host, --port, --insecure, --tui, --no-open)
so we can't auto-restart it.  But we CAN stop it, which is what the
user wants — the failure mode when the stale process is left alive is
worse than the dashboard just being down.

- POSIX: SIGTERM, poll for ~3s, SIGKILL any survivors.
- Windows: `taskkill /PID <pid> /F`.
- Print each PID's outcome plus a one-line restart hint.
- Detection logic is unchanged (same ps / wmic scan, same guards
  against the `pgrep -f` greedy-match trap from #16872 and the
  #17049 wmic UnicodeDecodeError fix).

Also split the old monolithic `_warn_stale_dashboard_processes` into
`_find_stale_dashboard_pids` (scan) + `_kill_stale_dashboard_processes`
(kill), keeping the old name as an alias so any external callers still
work.

E2E verified: spawned a fake `hermes dashboard` cmdline via
`exec -a 'hermes dashboard …' sleep 300`, ran
`_kill_stale_dashboard_processes()`, confirmed SIGTERM exit (-15)
and that a post-scan returns an empty PID list.
This commit is contained in:
Teknium
2026-04-30 01:34:34 -07:00
committed by GitHub
parent fd0796947f
commit 5b85a7d351
2 changed files with 358 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
hermes_cli/main.py
View File

@@ -5335,8 +5335,8 @@ def _build_web_ui(web_dir: Path, *, fatal: bool = False) -> bool:
return True
def _warn_stale_dashboard_processes() -> None:
"""Warn about running dashboard processes that still hold pre-update code.
def _find_stale_dashboard_pids() -> list[int]:
"""Return PIDs of ``hermes dashboard`` processes other than ourselves.
``hermes dashboard`` is a long-lived server process commonly started and
forgotten. When ``hermes update`` replaces files on disk, the running
@@ -5344,9 +5344,13 @@ def _warn_stale_dashboard_processes() -> None:
disk is updated, causing a silent frontend/backend mismatch (e.g. new
auth headers the old backend doesn't recognise → every API call 401s).
Unlike the gateway, the dashboard has no service manager (systemd /
launchd), so we can only warn — we don't auto-kill user-managed
background processes.
The dashboard has no service manager (systemd / launchd), no PID file,
and we can't know the original launch args — so the only sane action
after an update is to kill the stale process and let the user restart
it. This helper is just the detection step; see
``_kill_stale_dashboard_processes`` for the kill.
Returns an empty list on any scan error (missing ps/wmic, timeout, etc.).
"""
patterns = [
"hermes dashboard",
@@ -5372,7 +5376,7 @@ def _warn_stale_dashboard_processes() -> None:
encoding="utf-8", errors="ignore",
)
if result.returncode != 0 or result.stdout is None:
return
return []
current_cmd = ""
for line in result.stdout.split("\n"):
line = line.strip()
@@ -5414,20 +5418,110 @@ def _warn_stale_dashboard_processes() -> None:
and pid != self_pid):
dashboard_pids.append(pid)
except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
return
return []
if not dashboard_pids:
return dashboard_pids
def _kill_stale_dashboard_processes() -> None:
"""Kill ``hermes dashboard`` processes left over from the previous version.
Called at the end of ``hermes update``. The dashboard has no service
manager, so after an update the running process is guaranteed to be
serving stale Python against a freshly-updated JS bundle. Leaving it
alive produces silent frontend/backend mismatches (new auth headers the
old backend doesn't recognise → every API call 401s).
POSIX: SIGTERM, wait up to ~3s for graceful exit, SIGKILL any survivors.
Windows: ``taskkill /PID <pid> /F`` since there's no clean SIGTERM
equivalent for background console apps.
The dashboard isn't auto-restarted because we don't know the original
launch args (--host, --port, --insecure, --tui, --no-open). The user
restarts it manually; we print a hint with the previous port(s) where
possible.
"""
pids = _find_stale_dashboard_pids()
if not pids:
return
print()
print(f"⚠ {len(dashboard_pids)} dashboard process(es) still running "
f"with the previous version:")
for pid in dashboard_pids:
print(f" PID {pid}")
print(" The running backend may not match the updated frontend,")
print(" causing silent auth failures or empty data.")
print(" Restart them to pick up the changes:")
print(" kill <pid> && hermes dashboard --port <port> ...")
print(f"⟲ Stopping {len(pids)} stale dashboard process(es) "
f"(the running backend no longer matches the updated frontend)")
killed: list[int] = []
failed: list[tuple[int, str]] = []
if sys.platform == "win32":
for pid in pids:
try:
result = subprocess.run(
["taskkill", "/PID", str(pid), "/F"],
capture_output=True, text=True, timeout=10,
)
if result.returncode == 0:
killed.append(pid)
else:
failed.append((pid, (result.stderr or result.stdout or "").strip()))
except (FileNotFoundError, subprocess.TimeoutExpired, OSError) as e:
failed.append((pid, str(e)))
else:
import signal as _signal
import time as _time
# SIGTERM first — give each process a chance to shut down cleanly
# (uvicorn closes its socket, flushes logs, etc.).
for pid in pids:
try:
os.kill(pid, _signal.SIGTERM)
except ProcessLookupError:
# Already gone — count as killed.
killed.append(pid)
except (PermissionError, OSError) as e:
failed.append((pid, str(e)))
# Poll for exit up to ~3s total.
deadline = _time.monotonic() + 3.0
pending = [p for p in pids if p not in killed
and p not in {f[0] for f in failed}]
while pending and _time.monotonic() < deadline:
_time.sleep(0.1)
still_pending = []
for pid in pending:
try:
os.kill(pid, 0) # probe
except ProcessLookupError:
killed.append(pid)
except (PermissionError, OSError):
# Can't probe — assume still there.
still_pending.append(pid)
else:
still_pending.append(pid)
pending = still_pending
# SIGKILL any survivors.
for pid in pending:
try:
os.kill(pid, _signal.SIGKILL)
killed.append(pid)
except ProcessLookupError:
killed.append(pid)
except (PermissionError, OSError) as e:
failed.append((pid, str(e)))
for pid in killed:
print(f" ✓ stopped PID {pid}")
for pid, reason in failed:
print(f" ✗ failed to stop PID {pid}: {reason}")
if killed:
print(" Restart the dashboard when you're ready:")
print(" hermes dashboard --port <port>")
# Back-compat alias: some tests and any external callers may import the old
# warn-only name. The new behaviour (kill stale processes) replaces it.
_warn_stale_dashboard_processes = _kill_stale_dashboard_processes
def _update_via_zip(args):
@@ -5564,7 +5658,7 @@ def _update_via_zip(args):
print()
print("✓ Update complete!")
_warn_stale_dashboard_processes()
_kill_stale_dashboard_processes()
def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[str]:
@@ -7381,9 +7475,12 @@ def _cmd_update_impl(args, gateway_mode: bool):
except Exception as e:
logger.debug("Legacy unit check during update failed: %s", e)
# Warn about stale dashboard processes — the dashboard has no
# service manager, so we can only tell the user to restart them.
_warn_stale_dashboard_processes()
# Kill stale dashboard processes — the dashboard has no service
# manager, so leaving it alive after a code update produces a
# silent frontend/backend mismatch. We can't auto-restart it
# (no saved launch args) but we can stop it, and a hint is
# printed for the user to re-launch.
_kill_stale_dashboard_processes()
print()
print("Tip: You can now select a provider and model:")