fix: Nous Portal rate limit guard — prevent retry amplification (#10568)
When Nous returns a 429, the retry amplification chain burns up to 9 API requests per conversation turn (3 SDK retries × 3 Hermes retries), each counting against RPH and deepening the rate limit. With multiple concurrent sessions (cron + gateway + auxiliary), this creates a spiral where retries keep the limit tapped indefinitely. New module: agent/nous_rate_guard.py - Shared file-based rate limit state (~/.hermes/rate_limits/nous.json) - Parses reset time from x-ratelimit-reset-requests-1h, x-ratelimit- reset-requests, retry-after headers, or error context - Falls back to 5-minute default cooldown if no header data - Atomic writes (tempfile + rename) for cross-process safety - Auto-cleanup of expired state files run_agent.py changes: - Top-of-retry-loop guard: when another session already recorded Nous as rate-limited, skip the API call entirely. Try fallback provider first, then return a clear message with the reset time. - On 429 from Nous: record rate limit state and skip further retries (sets retry_count = max_retries to trigger fallback path) - On success from Nous: clear the rate limit state so other sessions know they can resume auxiliary_client.py changes: - _try_nous() checks rate guard before attempting Nous in the auxiliary fallback chain. When rate-limited, returns (None, None) so the chain skips to the next provider instead of piling more requests onto Nous. This eliminates three sources of amplification: 1. Hermes-level retries (saves 6 of 9 calls per turn) 2. Cross-session retries (cron + gateway all skip Nous) 3. Auxiliary fallback to Nous (compression/session_search skip too) Includes 24 tests covering the rate guard module, header parsing, state lifecycle, and auxiliary client integration.
This commit is contained in:
88
run_agent.py
88
run_agent.py
@@ -8660,6 +8660,53 @@ class AIAgent:
|
||||
api_kwargs = None # Guard against UnboundLocalError in except handler
|
||||
|
||||
while retry_count < max_retries:
|
||||
# ── Nous Portal rate limit guard ──────────────────────
|
||||
# If another session already recorded that Nous is rate-
|
||||
# limited, skip the API call entirely. Each attempt
|
||||
# (including SDK-level retries) counts against RPH and
|
||||
# deepens the rate limit hole.
|
||||
if self.provider == "nous":
|
||||
try:
|
||||
from agent.nous_rate_guard import (
|
||||
nous_rate_limit_remaining,
|
||||
format_remaining as _fmt_nous_remaining,
|
||||
)
|
||||
_nous_remaining = nous_rate_limit_remaining()
|
||||
if _nous_remaining is not None and _nous_remaining > 0:
|
||||
_nous_msg = (
|
||||
f"Nous Portal rate limit active — "
|
||||
f"resets in {_fmt_nous_remaining(_nous_remaining)}."
|
||||
)
|
||||
self._vprint(
|
||||
f"{self.log_prefix}⏳ {_nous_msg} Trying fallback...",
|
||||
force=True,
|
||||
)
|
||||
self._emit_status(f"⏳ {_nous_msg}")
|
||||
if self._try_activate_fallback():
|
||||
retry_count = 0
|
||||
compression_attempts = 0
|
||||
primary_recovery_attempted = False
|
||||
continue
|
||||
# No fallback available — return with clear message
|
||||
self._persist_session(messages, conversation_history)
|
||||
return {
|
||||
"final_response": (
|
||||
f"⏳ {_nous_msg}\n\n"
|
||||
"No fallback provider available. "
|
||||
"Try again after the reset, or add a "
|
||||
"fallback provider in config.yaml."
|
||||
),
|
||||
"messages": messages,
|
||||
"api_calls": api_call_count,
|
||||
"completed": False,
|
||||
"failed": True,
|
||||
"error": _nous_msg,
|
||||
}
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
pass # Never let rate guard break the agent loop
|
||||
|
||||
try:
|
||||
self._reset_stream_delivery_tracking()
|
||||
api_kwargs = self._build_api_kwargs(api_messages)
|
||||
@@ -9248,6 +9295,15 @@ class AIAgent:
|
||||
self._vprint(f"{self.log_prefix} 💾 Cache: {cached:,}/{prompt:,} tokens ({hit_pct:.0f}% hit, {written:,} written)")
|
||||
|
||||
has_retried_429 = False # Reset on success
|
||||
# Clear Nous rate limit state on successful request —
|
||||
# proves the limit has reset and other sessions can
|
||||
# resume hitting Nous.
|
||||
if self.provider == "nous":
|
||||
try:
|
||||
from agent.nous_rate_guard import clear_nous_rate_limit
|
||||
clear_nous_rate_limit()
|
||||
except Exception:
|
||||
pass
|
||||
self._touch_activity(f"API call #{api_call_count} completed")
|
||||
break # Success, exit retry loop
|
||||
|
||||
@@ -9659,6 +9715,38 @@ class AIAgent:
|
||||
primary_recovery_attempted = False
|
||||
continue
|
||||
|
||||
# ── Nous Portal: record rate limit & skip retries ─────
|
||||
# When Nous returns a 429, record the reset time to a
|
||||
# shared file so ALL sessions (cron, gateway, auxiliary)
|
||||
# know not to pile on. Then skip further retries —
|
||||
# each one burns another RPH request and deepens the
|
||||
# rate limit hole. The retry loop's top-of-iteration
|
||||
# guard will catch this on the next pass and try
|
||||
# fallback or bail with a clear message.
|
||||
if (
|
||||
is_rate_limited
|
||||
and self.provider == "nous"
|
||||
and classified.reason == FailoverReason.rate_limit
|
||||
and not recovered_with_pool
|
||||
):
|
||||
try:
|
||||
from agent.nous_rate_guard import record_nous_rate_limit
|
||||
_err_resp = getattr(api_error, "response", None)
|
||||
_err_hdrs = (
|
||||
getattr(_err_resp, "headers", None)
|
||||
if _err_resp else None
|
||||
)
|
||||
record_nous_rate_limit(
|
||||
headers=_err_hdrs,
|
||||
error_context=error_context,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
# Skip straight to max_retries — the top-of-loop
|
||||
# guard will handle fallback or bail cleanly.
|
||||
retry_count = max_retries
|
||||
continue
|
||||
|
||||
is_payload_too_large = (
|
||||
classified.reason == FailoverReason.payload_too_large
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user