fix: sanitize api_messages and extra string fields during ASCII-codec recovery (#6843)
The ASCII-locale recovery path in run_agent.py sanitized the canonical
'messages' list but left 'api_messages' untouched. api_messages is a
separate API-copy built before the retry loop and may carry extra fields
(reasoning_content, extra_body entries) that are not present in
'messages'. This caused the retry to still raise UnicodeEncodeError even
after the 'System encoding is ASCII — stripped...' log line appeared.
Two changes:
- _sanitize_messages_non_ascii now walks all extra top-level string fields
in each message dict (any key not in {content, name, tool_calls, role})
so reasoning_content and future extras are cleaned in both 'messages'
and 'api_messages'.
- The ASCII-codec recovery block now also calls sanitize on api_messages
and api_kwargs so no non-ASCII survives into the next retry attempt.
Adds regression tests covering:
- reasoning_content with non-ASCII in api_messages
- extra_body with non-ASCII in api_kwargs
- canonical messages clean but api_messages dirty
Fixes #6843
This commit is contained in:
committed by
Teknium
parent
d4eba82a37
commit
efd1ddc6e1
21
run_agent.py
21
run_agent.py
@@ -457,6 +457,15 @@ def _sanitize_messages_non_ascii(messages: list) -> bool:
|
||||
if sanitized != fn_args:
|
||||
fn["arguments"] = sanitized
|
||||
found = True
|
||||
# Sanitize any additional top-level string fields (e.g. reasoning_content)
|
||||
for key, value in msg.items():
|
||||
if key in {"content", "name", "tool_calls", "role"}:
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
sanitized = _strip_non_ascii(value)
|
||||
if sanitized != value:
|
||||
msg[key] = sanitized
|
||||
found = True
|
||||
return found
|
||||
|
||||
|
||||
@@ -9107,7 +9116,19 @@ class AIAgent:
|
||||
# ASCII codec: the system encoding can't handle
|
||||
# non-ASCII characters at all. Sanitize all
|
||||
# non-ASCII content from messages/tool schemas and retry.
|
||||
# Sanitize both the canonical `messages` list and
|
||||
# `api_messages` (the API-copy built before the retry
|
||||
# loop, which may contain extra fields like
|
||||
# reasoning_content that are not in `messages`).
|
||||
_messages_sanitized = _sanitize_messages_non_ascii(messages)
|
||||
if isinstance(api_messages, list):
|
||||
_sanitize_messages_non_ascii(api_messages)
|
||||
# Also sanitize the last api_kwargs if already built,
|
||||
# so a leftover non-ASCII value in a transformed field
|
||||
# (e.g. extra_body, reasoning_content) doesn't survive
|
||||
# into the next attempt via _build_api_kwargs cache paths.
|
||||
if isinstance(api_kwargs, dict):
|
||||
_sanitize_structure_non_ascii(api_kwargs)
|
||||
_prefill_sanitized = False
|
||||
if isinstance(getattr(self, "prefill_messages", None), list):
|
||||
_prefill_sanitized = _sanitize_messages_non_ascii(self.prefill_messages)
|
||||
|
||||
Reference in New Issue
Block a user