The shutdown banner promised "send any message after restart to resume where you left off" but the code did the opposite: a drain-timeout restart skipped the .clean_shutdown marker, which made the next startup call suspend_recently_active(), which marked the session suspended, which made get_or_create_session() spawn a fresh session_id with a 'Session automatically reset. Use /resume...' notice — contradicting the banner. Introduce a resume_pending state on SessionEntry that is distinct from suspended. Drain-timeout shutdown flags active sessions resume_pending instead of letting startup-wide suspension destroy them. The next message on the same session_key preserves the session_id, reloads the transcript, and the agent receives a reason-aware restart-resume system note that subsumes the existing tool-tail auto-continue note (PR #9934). Terminal escalation still flows through the existing .restart_failure_counts stuck-loop counter (PR #7536, threshold 3) — no parallel counter on SessionEntry. suspended still wins over resume_pending in get_or_create_session() so genuinely stuck sessions converge to a clean slate. Spec: PR #11852 (BrennerSpear). Implementation follows the spec with the approved correction (reuse .restart_failure_counts rather than adding a resume_attempts field). Changes: - gateway/session.py: SessionEntry.resume_pending/resume_reason/ last_resume_marked_at + to_dict/from_dict; SessionStore .mark_resume_pending()/clear_resume_pending(); get_or_create_session() returns existing entry when resume_pending (suspended still wins); suspend_recently_active() skips resume_pending entries. - gateway/run.py: _stop_impl() drain-timeout branch marks active sessions resume_pending before _interrupt_running_agents(); _run_agent() injects reason-aware restart-resume system note that subsumes the tool-tail case; successful-turn cleanup also clears resume_pending next to _clear_restart_failure_count(); _notify_active_sessions_of_shutdown() softens the restart banner to 'I'll try to resume where you left off' (honest about stuck-loop escalation). - tests/gateway/test_restart_resume_pending.py: 29 new tests covering SessionEntry roundtrip, mark/clear helpers, get_or_create_session precedence (suspended > resume_pending), suspend_recently_active skip, drain-timeout mark reason (restart vs shutdown), system-note injection decision tree (including tool-tail subsumption), banner wording, and stuck-loop escalation override.
This commit is contained in:
@@ -1539,7 +1539,7 @@ class GatewayRunner:
|
||||
action = "restarting" if self._restart_requested else "shutting down"
|
||||
hint = (
|
||||
"Your current task will be interrupted. "
|
||||
"Send any message after restart to resume where it left off."
|
||||
"Send any message after restart and I'll try to resume where you left off."
|
||||
if self._restart_requested
|
||||
else "Your current task will be interrupted."
|
||||
)
|
||||
@@ -2373,6 +2373,27 @@ class GatewayRunner:
|
||||
timeout,
|
||||
self._running_agent_count(),
|
||||
)
|
||||
# Mark forcibly-interrupted sessions as resume_pending BEFORE
|
||||
# interrupting the agents. This preserves each session's
|
||||
# session_id + transcript so the next message on the same
|
||||
# session_key auto-resumes from the existing conversation
|
||||
# instead of getting routed through suspend_recently_active()
|
||||
# and converted into a fresh session. Terminal escalation
|
||||
# for genuinely stuck sessions still flows through the
|
||||
# existing ``.restart_failure_counts`` stuck-loop counter
|
||||
# (incremented below, threshold 3), which sets
|
||||
# ``suspended=True`` and overrides resume_pending.
|
||||
_resume_reason = (
|
||||
"restart_timeout" if self._restart_requested else "shutdown_timeout"
|
||||
)
|
||||
for _sk in list(active_agents.keys()):
|
||||
try:
|
||||
self.session_store.mark_resume_pending(_sk, _resume_reason)
|
||||
except Exception as _e:
|
||||
logger.debug(
|
||||
"mark_resume_pending failed for %s: %s",
|
||||
_sk[:20], _e,
|
||||
)
|
||||
self._interrupt_running_agents(
|
||||
"Gateway restarting" if self._restart_requested else "Gateway shutting down"
|
||||
)
|
||||
@@ -4152,8 +4173,20 @@ class GatewayRunner:
|
||||
# Successful turn — clear any stuck-loop counter for this session.
|
||||
# This ensures the counter only accumulates across CONSECUTIVE
|
||||
# restarts where the session was active (never completed).
|
||||
#
|
||||
# Also clear the resume_pending flag (set by drain-timeout
|
||||
# shutdown) — the turn ran to completion, so recovery
|
||||
# succeeded and subsequent messages should no longer receive
|
||||
# the restart-interruption system note.
|
||||
if session_key:
|
||||
self._clear_restart_failure_count(session_key)
|
||||
try:
|
||||
self.session_store.clear_resume_pending(session_key)
|
||||
except Exception as _e:
|
||||
logger.debug(
|
||||
"clear_resume_pending failed for %s: %s",
|
||||
session_key[:20], _e,
|
||||
)
|
||||
|
||||
# Surface error details when the agent failed silently (final_response=None)
|
||||
if not response and agent_result.get("failed"):
|
||||
@@ -9427,7 +9460,40 @@ class GatewayRunner:
|
||||
# restart, crash, SIGTERM). Prepend a system note so the model
|
||||
# finishes processing the pending tool results before addressing
|
||||
# the user's new message. (#4493)
|
||||
if agent_history and agent_history[-1].get("role") == "tool":
|
||||
#
|
||||
# Session-level resume_pending (set on drain-timeout shutdown)
|
||||
# escalates the wording — the transcript's last role may be
|
||||
# anything (tool, assistant with unfinished work, etc.), so we
|
||||
# give a stronger, reason-aware instruction that subsumes the
|
||||
# tool-tail case.
|
||||
_resume_entry = None
|
||||
if session_key:
|
||||
try:
|
||||
_resume_entry = self.session_store._entries.get(session_key)
|
||||
except Exception:
|
||||
_resume_entry = None
|
||||
_is_resume_pending = bool(
|
||||
_resume_entry is not None and getattr(_resume_entry, "resume_pending", False)
|
||||
)
|
||||
|
||||
if _is_resume_pending:
|
||||
_reason = getattr(_resume_entry, "resume_reason", None) or "restart_timeout"
|
||||
_reason_phrase = (
|
||||
"a gateway restart"
|
||||
if _reason == "restart_timeout"
|
||||
else "a gateway shutdown"
|
||||
if _reason == "shutdown_timeout"
|
||||
else "a gateway interruption"
|
||||
)
|
||||
message = (
|
||||
f"[System note: Your previous turn in this session was interrupted "
|
||||
f"by {_reason_phrase}. The conversation history below is intact. "
|
||||
f"If it contains unfinished tool result(s), process them first and "
|
||||
f"summarize what was accomplished, then address the user's new "
|
||||
f"message below.]\n\n"
|
||||
+ message
|
||||
)
|
||||
elif agent_history and agent_history[-1].get("role") == "tool":
|
||||
message = (
|
||||
"[System note: Your previous turn was interrupted before you could "
|
||||
"process the last tool result(s). The conversation history contains "
|
||||
|
||||
Reference in New Issue
Block a user