fix(gateway): ignore redelivered /restart after PTB offset ACK fails (#11940)

When a Telegram /restart fires and PTB's graceful-shutdown `get_updates`
ACK call times out ("When polling for updates is restarted, updates may
be received twice" in gateway.log), the new gateway receives the same
/restart again and restarts a second time — a self-perpetuating loop.

Record the triggering update_id in `.restart_last_processed.json` when
handling /restart.  On the next process, reject a /restart whose
update_id <= the recorded one as a stale redelivery.  5-minute staleness
guard so an orphaned marker can't block a legitimately new /restart.

- gateway/platforms/base.py: add `platform_update_id` to MessageEvent
- gateway/platforms/telegram.py: propagate `update.update_id` through
  _build_message_event for text/command/location/media handlers
- gateway/run.py: write dedup marker in _handle_restart_command;
  _is_stale_restart_redelivery checks it before processing /restart
- tests/gateway/test_restart_redelivery_dedup.py: 9 new tests covering
  fresh restart, redelivery, staleness window, cross-platform,
  malformed-marker resilience, and no-update_id (CLI) bypass

Only active for Telegram today (the one platform with monotonic
cross-session update ordering); other platforms return False from
_is_stale_restart_redelivery and proceed normally.
This commit is contained in:
Teknium
2026-04-17 21:17:33 -07:00
committed by GitHub
parent c5c0bb9a73
commit 45acd9beb5
4 changed files with 366 additions and 6 deletions

View File

@@ -4738,6 +4738,26 @@ class GatewayRunner:
async def _handle_restart_command(self, event: MessageEvent) -> str:
"""Handle /restart command - drain active work, then restart the gateway."""
# Defensive idempotency check: if the previous gateway process
# recorded this same /restart (same platform + update_id) and the new
# process is seeing it *again*, this is a re-delivery caused by PTB's
# graceful-shutdown `get_updates` ACK failing on the way out ("Error
# while calling `get_updates` one more time to mark all fetched
# updates. Suppressing error to ensure graceful shutdown. When
# polling for updates is restarted, updates may be received twice."
# in gateway.log). Ignoring the stale redelivery prevents a
# self-perpetuating restart loop where every fresh gateway
# re-processes the same /restart command and immediately restarts
# again.
if self._is_stale_restart_redelivery(event):
logger.info(
"Ignoring redelivered /restart (platform=%s, update_id=%s) — "
"already processed by a previous gateway instance.",
event.source.platform.value if event.source and event.source.platform else "?",
event.platform_update_id,
)
return ""
if self._restart_requested or self._draining:
count = self._running_agent_count()
if count:
@@ -4760,6 +4780,26 @@ class GatewayRunner:
except Exception as e:
logger.debug("Failed to write restart notify file: %s", e)
# Record the triggering platform + update_id in a dedicated dedup
# marker. Unlike .restart_notify.json (which gets unlinked once the
# new gateway sends the "gateway restarted" notification), this
# marker persists so the new gateway can still detect a delayed
# /restart redelivery from Telegram. Overwritten on every /restart.
try:
import json as _json
import time as _time
dedup_data = {
"platform": event.source.platform.value if event.source.platform else None,
"requested_at": _time.time(),
}
if event.platform_update_id is not None:
dedup_data["update_id"] = event.platform_update_id
(_hermes_home / ".restart_last_processed.json").write_text(
_json.dumps(dedup_data)
)
except Exception as e:
logger.debug("Failed to write restart dedup marker: %s", e)
active_agents = self._running_agent_count()
# When running under a service manager (systemd/launchd), use the
# service restart path: exit with code 75 so the service manager
@@ -4775,6 +4815,58 @@ class GatewayRunner:
return f"⏳ Draining {active_agents} active agent(s) before restart..."
return "♻ Restarting gateway. If you aren't notified within 60 seconds, restart from the console with `hermes gateway restart`."
def _is_stale_restart_redelivery(self, event: MessageEvent) -> bool:
"""Return True if this /restart is a Telegram re-delivery we already handled.
The previous gateway wrote ``.restart_last_processed.json`` with the
triggering platform + update_id when it processed the /restart. If
we now see a /restart on the same platform with an update_id <= that
recorded value AND the marker is recent (< 5 minutes), it's a
redelivery and should be ignored.
Only applies to Telegram today (the only platform that exposes a
numeric cross-session update ordering); other platforms return False.
"""
if event is None or event.source is None:
return False
if event.platform_update_id is None:
return False
if event.source.platform is None:
return False
# Only Telegram populates platform_update_id currently; be explicit
# so future platforms aren't accidentally gated by this check.
try:
platform_value = event.source.platform.value
except Exception:
return False
if platform_value != "telegram":
return False
try:
import json as _json
import time as _time
marker_path = _hermes_home / ".restart_last_processed.json"
if not marker_path.exists():
return False
data = _json.loads(marker_path.read_text())
except Exception:
return False
if data.get("platform") != platform_value:
return False
recorded_uid = data.get("update_id")
if not isinstance(recorded_uid, int):
return False
# Staleness guard: ignore markers older than 5 minutes. A legitimately
# old marker (e.g. crash recovery where notify never fired) should not
# swallow a fresh /restart from the user.
requested_at = data.get("requested_at")
if isinstance(requested_at, (int, float)):
if _time.time() - requested_at > 300:
return False
return event.platform_update_id <= recorded_uid
async def _handle_help_command(self, event: MessageEvent) -> str:
"""Handle /help command - list available commands."""
from hermes_cli.commands import gateway_help_lines