fix(gateway): ignore redelivered /restart after PTB offset ACK fails (#11940)
When a Telegram /restart fires and PTB's graceful-shutdown `get_updates`
ACK call times out ("When polling for updates is restarted, updates may
be received twice" in gateway.log), the new gateway receives the same
/restart again and restarts a second time — a self-perpetuating loop.
Record the triggering update_id in `.restart_last_processed.json` when
handling /restart. On the next process, reject a /restart whose
update_id <= the recorded one as a stale redelivery. 5-minute staleness
guard so an orphaned marker can't block a legitimately new /restart.
- gateway/platforms/base.py: add `platform_update_id` to MessageEvent
- gateway/platforms/telegram.py: propagate `update.update_id` through
_build_message_event for text/command/location/media handlers
- gateway/run.py: write dedup marker in _handle_restart_command;
_is_stale_restart_redelivery checks it before processing /restart
- tests/gateway/test_restart_redelivery_dedup.py: 9 new tests covering
fresh restart, redelivery, staleness window, cross-platform,
malformed-marker resilience, and no-update_id (CLI) bypass
Only active for Telegram today (the one platform with monotonic
cross-session update ordering); other platforms return False from
_is_stale_restart_redelivery and proceed normally.
This commit is contained in:
@@ -4738,6 +4738,26 @@ class GatewayRunner:
|
||||
|
||||
async def _handle_restart_command(self, event: MessageEvent) -> str:
|
||||
"""Handle /restart command - drain active work, then restart the gateway."""
|
||||
# Defensive idempotency check: if the previous gateway process
|
||||
# recorded this same /restart (same platform + update_id) and the new
|
||||
# process is seeing it *again*, this is a re-delivery caused by PTB's
|
||||
# graceful-shutdown `get_updates` ACK failing on the way out ("Error
|
||||
# while calling `get_updates` one more time to mark all fetched
|
||||
# updates. Suppressing error to ensure graceful shutdown. When
|
||||
# polling for updates is restarted, updates may be received twice."
|
||||
# in gateway.log). Ignoring the stale redelivery prevents a
|
||||
# self-perpetuating restart loop where every fresh gateway
|
||||
# re-processes the same /restart command and immediately restarts
|
||||
# again.
|
||||
if self._is_stale_restart_redelivery(event):
|
||||
logger.info(
|
||||
"Ignoring redelivered /restart (platform=%s, update_id=%s) — "
|
||||
"already processed by a previous gateway instance.",
|
||||
event.source.platform.value if event.source and event.source.platform else "?",
|
||||
event.platform_update_id,
|
||||
)
|
||||
return ""
|
||||
|
||||
if self._restart_requested or self._draining:
|
||||
count = self._running_agent_count()
|
||||
if count:
|
||||
@@ -4760,6 +4780,26 @@ class GatewayRunner:
|
||||
except Exception as e:
|
||||
logger.debug("Failed to write restart notify file: %s", e)
|
||||
|
||||
# Record the triggering platform + update_id in a dedicated dedup
|
||||
# marker. Unlike .restart_notify.json (which gets unlinked once the
|
||||
# new gateway sends the "gateway restarted" notification), this
|
||||
# marker persists so the new gateway can still detect a delayed
|
||||
# /restart redelivery from Telegram. Overwritten on every /restart.
|
||||
try:
|
||||
import json as _json
|
||||
import time as _time
|
||||
dedup_data = {
|
||||
"platform": event.source.platform.value if event.source.platform else None,
|
||||
"requested_at": _time.time(),
|
||||
}
|
||||
if event.platform_update_id is not None:
|
||||
dedup_data["update_id"] = event.platform_update_id
|
||||
(_hermes_home / ".restart_last_processed.json").write_text(
|
||||
_json.dumps(dedup_data)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug("Failed to write restart dedup marker: %s", e)
|
||||
|
||||
active_agents = self._running_agent_count()
|
||||
# When running under a service manager (systemd/launchd), use the
|
||||
# service restart path: exit with code 75 so the service manager
|
||||
@@ -4775,6 +4815,58 @@ class GatewayRunner:
|
||||
return f"⏳ Draining {active_agents} active agent(s) before restart..."
|
||||
return "♻ Restarting gateway. If you aren't notified within 60 seconds, restart from the console with `hermes gateway restart`."
|
||||
|
||||
def _is_stale_restart_redelivery(self, event: MessageEvent) -> bool:
|
||||
"""Return True if this /restart is a Telegram re-delivery we already handled.
|
||||
|
||||
The previous gateway wrote ``.restart_last_processed.json`` with the
|
||||
triggering platform + update_id when it processed the /restart. If
|
||||
we now see a /restart on the same platform with an update_id <= that
|
||||
recorded value AND the marker is recent (< 5 minutes), it's a
|
||||
redelivery and should be ignored.
|
||||
|
||||
Only applies to Telegram today (the only platform that exposes a
|
||||
numeric cross-session update ordering); other platforms return False.
|
||||
"""
|
||||
if event is None or event.source is None:
|
||||
return False
|
||||
if event.platform_update_id is None:
|
||||
return False
|
||||
if event.source.platform is None:
|
||||
return False
|
||||
# Only Telegram populates platform_update_id currently; be explicit
|
||||
# so future platforms aren't accidentally gated by this check.
|
||||
try:
|
||||
platform_value = event.source.platform.value
|
||||
except Exception:
|
||||
return False
|
||||
if platform_value != "telegram":
|
||||
return False
|
||||
|
||||
try:
|
||||
import json as _json
|
||||
import time as _time
|
||||
marker_path = _hermes_home / ".restart_last_processed.json"
|
||||
if not marker_path.exists():
|
||||
return False
|
||||
data = _json.loads(marker_path.read_text())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
if data.get("platform") != platform_value:
|
||||
return False
|
||||
recorded_uid = data.get("update_id")
|
||||
if not isinstance(recorded_uid, int):
|
||||
return False
|
||||
# Staleness guard: ignore markers older than 5 minutes. A legitimately
|
||||
# old marker (e.g. crash recovery where notify never fired) should not
|
||||
# swallow a fresh /restart from the user.
|
||||
requested_at = data.get("requested_at")
|
||||
if isinstance(requested_at, (int, float)):
|
||||
if _time.time() - requested_at > 300:
|
||||
return False
|
||||
return event.platform_update_id <= recorded_uid
|
||||
|
||||
|
||||
async def _handle_help_command(self, event: MessageEvent) -> str:
|
||||
"""Handle /help command - list available commands."""
|
||||
from hermes_cli.commands import gateway_help_lines
|
||||
|
||||
Reference in New Issue
Block a user