common/hermes
2
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Honor provider reset windows in pooled credential failover

Browse Source 
Persist structured exhaustion metadata from provider errors, use explicit reset timestamps when available, and expose label-based credential targeting in the auth CLI. This keeps long-lived Codex cooldowns from being misreported as one-hour waits and avoids forcing operators to manage entries by list position alone.

Constraint: Existing credential pool JSON needs to remain backward compatible with stored entries that only record status code and timestamp
Constraint: Runtime recovery must keep the existing retry-then-rotate semantics for 429s while enriching pool state with provider metadata
Rejected: Add a separate credential scheduler subsystem | too large for the Hermes pool architecture and unnecessary for this fix
Rejected: Only change CLI formatting | would leave runtime rotation blind to resets_at and preserve the serial-failure behavior
Confidence: high
Scope-risk: moderate
Reversibility: clean
Directive: Preserve structured rate-limit metadata when new providers expose reset hints; do not collapse back to status-code-only exhaustion tracking
Tested: Focused pytest slice for auth commands, credential pool recovery, and routing (272 passed); py_compile on changed Python files; hermes -w auth list/remove smoke test with temporary HERMES_HOME
Not-tested: Full repository pytest suite, broader gateway/integration flows outside the touched auth and pool paths
This commit is contained in:
kshitijk4poor
2026-04-05 12:03:20 +05:30
committed by Teknium
parent ed4a605696
commit 65952ac00c
8 changed files with 446 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
agent/credential_pool.py
View File

@@ -8,7 +8,9 @@ import threading
import time
import uuid
import os
import re
from dataclasses import dataclass, fields, replace
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, Set, Tuple
from hermes_constants import OPENROUTER_BASE_URL
@@ -95,6 +97,9 @@ class PooledCredential:
last_status: Optional[str] = None
last_status_at: Optional[float] = None
last_error_code: Optional[int] = None
last_error_reason: Optional[str] = None
last_error_message: Optional[str] = None
last_error_reset_at: Optional[float] = None
base_url: Optional[str] = None
expires_at: Optional[str] = None
expires_at_ms: Optional[int] = None
@@ -129,7 +134,14 @@ class PooledCredential:
return cls(provider=provider, **data)
def to_dict(self) -> Dict[str, Any]:
_ALWAYS_EMIT = {"last_status", "last_status_at", "last_error_code"}
_ALWAYS_EMIT = {
"last_status",
"last_status_at",
"last_error_code",
"last_error_reason",
"last_error_message",
"last_error_reset_at",
}
result: Dict[str, Any] = {}
for field_def in fields(self):
if field_def.name in ("provider", "extra"):
@@ -180,6 +192,85 @@ def _exhausted_ttl(error_code: Optional[int]) -> int:
return EXHAUSTED_TTL_DEFAULT_SECONDS
def _parse_absolute_timestamp(value: Any) -> Optional[float]:
"""Best-effort parse for provider reset timestamps.
Accepts epoch seconds, epoch milliseconds, and ISO-8601 strings.
Returns seconds since epoch.
"""
if value is None or value == "":
return None
if isinstance(value, (int, float)):
numeric = float(value)
if numeric <= 0:
return None
return numeric / 1000.0 if numeric > 1_000_000_000_000 else numeric
if isinstance(value, str):
raw = value.strip()
if not raw:
return None
try:
numeric = float(raw)
except ValueError:
numeric = None
if numeric is not None:
return numeric / 1000.0 if numeric > 1_000_000_000_000 else numeric
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00")).timestamp()
except ValueError:
return None
return None
def _extract_retry_delay_seconds(message: str) -> Optional[float]:
if not message:
return None
delay_match = re.search(r"quotaResetDelay[:\s\"]+(\d+(?:\.\d+)?)(ms|s)", message, re.IGNORECASE)
if delay_match:
value = float(delay_match.group(1))
return value / 1000.0 if delay_match.group(2).lower() == "ms" else value
sec_match = re.search(r"retry\s+(?:after\s+)?(\d+(?:\.\d+)?)\s*(?:sec|secs|seconds|s\b)", message, re.IGNORECASE)
if sec_match:
return float(sec_match.group(1))
return None
def _normalize_error_context(error_context: Optional[Dict[str, Any]]) -> Dict[str, Any]:
if not isinstance(error_context, dict):
return {}
normalized: Dict[str, Any] = {}
reason = error_context.get("reason")
if isinstance(reason, str) and reason.strip():
normalized["reason"] = reason.strip()
message = error_context.get("message")
if isinstance(message, str) and message.strip():
normalized["message"] = message.strip()
reset_at = (
error_context.get("reset_at")
or error_context.get("resets_at")
or error_context.get("retry_until")
)
parsed_reset_at = _parse_absolute_timestamp(reset_at)
if parsed_reset_at is None and isinstance(message, str):
retry_delay_seconds = _extract_retry_delay_seconds(message)
if retry_delay_seconds is not None:
parsed_reset_at = time.time() + retry_delay_seconds
if parsed_reset_at is not None:
normalized["reset_at"] = parsed_reset_at
return normalized
def _exhausted_until(entry: PooledCredential) -> Optional[float]:
if entry.last_status != STATUS_EXHAUSTED:
return None
reset_at = _parse_absolute_timestamp(getattr(entry, "last_error_reset_at", None))
if reset_at is not None:
return reset_at
if entry.last_status_at:
return entry.last_status_at + _exhausted_ttl(entry.last_error_code)
return None
def _normalize_custom_pool_name(name: str) -> str:
"""Normalize a custom provider name for use as a pool key suffix."""
return name.strip().lower().replace(" ", "-")
@@ -292,12 +383,21 @@ class CredentialPool:
[entry.to_dict() for entry in self._entries],
)
def _mark_exhausted(self, entry: PooledCredential, status_code: Optional[int]) -> PooledCredential:
def _mark_exhausted(
self,
entry: PooledCredential,
status_code: Optional[int],
error_context: Optional[Dict[str, Any]] = None,
) -> PooledCredential:
normalized_error = _normalize_error_context(error_context)
updated = replace(
entry,
last_status=STATUS_EXHAUSTED,
last_status_at=time.time(),
last_error_code=status_code,
last_error_reason=normalized_error.get("reason"),
last_error_message=normalized_error.get("message"),
last_error_reset_at=normalized_error.get("reset_at"),
)
self._replace_entry(entry, updated)
self._persist()
@@ -462,7 +562,15 @@ class CredentialPool:
self._mark_exhausted(entry, None)
return None
updated = replace(updated, last_status=STATUS_OK, last_status_at=None, last_error_code=None)
updated = replace(
updated,
last_status=STATUS_OK,
last_status_at=None,
last_error_code=None,
last_error_reason=None,
last_error_message=None,
last_error_reset_at=None,
)
self._replace_entry(entry, updated)
self._persist()
return updated
@@ -522,11 +630,19 @@ class CredentialPool:
entry = synced
cleared_any = True
if entry.last_status == STATUS_EXHAUSTED:
ttl = _exhausted_ttl(entry.last_error_code)
if entry.last_status_at and now - entry.last_status_at < ttl:
exhausted_until = _exhausted_until(entry)
if exhausted_until is not None and now < exhausted_until:
continue
if clear_expired:
cleared = replace(entry, last_status=STATUS_OK, last_status_at=None, last_error_code=None)
cleared = replace(
entry,
last_status=STATUS_OK,
last_status_at=None,
last_error_code=None,
last_error_reason=None,
last_error_message=None,
last_error_reset_at=None,
)
self._replace_entry(entry, cleared)
entry = cleared
cleared_any = True
@@ -576,12 +692,17 @@ class CredentialPool:
available = self._available_entries()
return available[0] if available else None
def mark_exhausted_and_rotate(self, *, status_code: Optional[int]) -> Optional[PooledCredential]:
def mark_exhausted_and_rotate(
self,
*,
status_code: Optional[int],
error_context: Optional[Dict[str, Any]] = None,
) -> Optional[PooledCredential]:
with self._lock:
entry = self.current() or self._select_unlocked()
if entry is None:
return None
self._mark_exhausted(entry, status_code)
self._mark_exhausted(entry, status_code, error_context)
self._current_id = None
return self._select_unlocked()
@@ -603,7 +724,17 @@ class CredentialPool:
new_entries = []
for entry in self._entries:
if entry.last_status or entry.last_status_at or entry.last_error_code:
new_entries.append(replace(entry, last_status=None, last_status_at=None, last_error_code=None))
new_entries.append(
replace(
entry,
last_status=None,
last_status_at=None,
last_error_code=None,
last_error_reason=None,
last_error_message=None,
last_error_reset_at=None,
)
)
count += 1
else:
new_entries.append(entry)
@@ -625,6 +756,31 @@ class CredentialPool:
self._current_id = None
return removed
def resolve_target(self, target: Any) -> Tuple[Optional[int], Optional[PooledCredential], Optional[str]]:
raw = str(target or "").strip()
if not raw:
return None, None, "No credential target provided."
if raw.isdigit():
index = int(raw)
if 1 <= index <= len(self._entries):
return index, self._entries[index - 1], None
return None, None, f"No credential #{index}."
for idx, entry in enumerate(self._entries, start=1):
if entry.id == raw:
return idx, entry, None
label_matches = [
(idx, entry)
for idx, entry in enumerate(self._entries, start=1)
if entry.label.strip().lower() == raw.lower()
]
if len(label_matches) == 1:
return label_matches[0][0], label_matches[0][1], None
if len(label_matches) > 1:
return None, None, f'Ambiguous credential label "{raw}". Use the numeric index or entry id instead.'
return None, None, f'No credential matching "{raw}".'
def add_entry(self, entry: PooledCredential) -> PooledCredential:
entry = replace(entry, priority=_next_priority(self._entries))
self._entries.append(entry)