fix(computer-use): address Copilot review on max_elements cap
Four findings from Copilot's review on PR #22891, all in the AX elements-array cap added by 22fa1ed: 1. The truncation note ("response truncated to N of M elements") was appended unconditionally — including in the som/vision multimodal path, whose response carries a screenshot rather than an `elements` array. The note described a payload field that wasn't present. Moved the note into the AX-text branch where the array actually appears. 2. `_format_elements(cap.elements)` ran on the full untrimmed list with its own `max_lines=40` cap, so a caller passing `max_elements=10` would see summary lines referencing `#11..#40` even though the JSON `elements` array only held #1..#10. Format on `visible_elements` instead so the summary indices always exist in the response. 3. `_coerce_max_elements` enforced a lower bound but no upper bound, so `max_elements=10_000_000` silently disabled the safeguard and reintroduced the original context-blow-up. Added a hard cap (`_MAX_ALLOWED_MAX_ELEMENTS = 1000`) that clamps oversized values. 4. The schema string said "Default 100" but the property carried no `default` field, and claimed `max_elements` had no effect on som/ vision while the image-missing fallback path can still return an elements array. Added `"default": 100`, `"maximum": 1000`, and clarified the fallback-path wording. Each finding gets a regression test: - test_capture_ax_clamps_oversized_max_elements_to_hard_cap - test_capture_ax_summary_indices_match_returned_elements - test_capture_multimodal_summary_omits_truncation_note - test_schema_max_elements_documents_default_and_upper_bound Verified with `pytest tests/tools/test_computer_use.py` (53 passed, including the 5 new cases). Confirmed each new test fails on the pre-fix code path before applying the production change. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -79,18 +79,23 @@ COMPUTER_USE_SCHEMA: Dict[str, Any] = {
|
||||
"type": "integer",
|
||||
"description": (
|
||||
"Optional cap on the AX `elements` array returned by "
|
||||
"`action='capture'`. Default 100. Dense UIs (Electron "
|
||||
"apps such as Obsidian or VS Code, JetBrains IDEs) can "
|
||||
"publish 500+ AX nodes — capping prevents a single "
|
||||
"capture from blowing session context. When the cap "
|
||||
"trims the response, `total_elements` and "
|
||||
"`truncated_elements` are surfaced in the result so "
|
||||
"you can re-call with `app=` to narrow scope or raise "
|
||||
"`max_elements` when the full tree is required. Has no "
|
||||
"effect on `mode='som'` / `mode='vision'` (those return "
|
||||
"a screenshot, not the elements array)."
|
||||
"`action='capture'`. Default 100, hard maximum 1000. "
|
||||
"Dense UIs (Electron apps such as Obsidian or VS Code, "
|
||||
"JetBrains IDEs) can publish 500+ AX nodes — capping "
|
||||
"prevents a single capture from blowing session "
|
||||
"context. When the cap trims the response, "
|
||||
"`total_elements` and `truncated_elements` are "
|
||||
"surfaced in the result so you can re-call with "
|
||||
"`app=` to narrow scope or raise `max_elements` when "
|
||||
"the full tree is required. Has no effect on "
|
||||
"`mode='som'` / `mode='vision'` when a screenshot is "
|
||||
"included in the response; only the rare image-"
|
||||
"missing fallback returns an `elements` array and is "
|
||||
"subject to the cap."
|
||||
),
|
||||
"default": 100,
|
||||
"minimum": 1,
|
||||
"maximum": 1000,
|
||||
},
|
||||
# ── click / drag / scroll targeting ────────────────────
|
||||
"element": {
|
||||
|
||||
@@ -421,6 +421,10 @@ def _text_response(res: ActionResult) -> str:
|
||||
# can exhaust session context after a single capture. The model-facing
|
||||
# `max_elements` argument lets callers raise this when they need the full tree.
|
||||
_DEFAULT_MAX_ELEMENTS = 100
|
||||
# Hard upper bound on caller-supplied `max_elements`. Without this, a tool
|
||||
# call passing a very large integer would silently disable the safeguard and
|
||||
# reintroduce the original unbounded behavior.
|
||||
_MAX_ALLOWED_MAX_ELEMENTS = 1000
|
||||
|
||||
|
||||
def _coerce_max_elements(value: Any) -> int:
|
||||
@@ -428,7 +432,9 @@ def _coerce_max_elements(value: Any) -> int:
|
||||
|
||||
Falls back to :data:`_DEFAULT_MAX_ELEMENTS` for missing / non-integer /
|
||||
sub-1 inputs so the cap can never be silently disabled by a malformed
|
||||
tool-call argument.
|
||||
tool-call argument. Clamps oversized values to
|
||||
:data:`_MAX_ALLOWED_MAX_ELEMENTS` so a caller cannot bypass the
|
||||
safeguard by passing a very large integer.
|
||||
"""
|
||||
if value is None:
|
||||
return _DEFAULT_MAX_ELEMENTS
|
||||
@@ -438,6 +444,8 @@ def _coerce_max_elements(value: Any) -> int:
|
||||
return _DEFAULT_MAX_ELEMENTS
|
||||
if n < 1:
|
||||
return _DEFAULT_MAX_ELEMENTS
|
||||
if n > _MAX_ALLOWED_MAX_ELEMENTS:
|
||||
return _MAX_ALLOWED_MAX_ELEMENTS
|
||||
return n
|
||||
|
||||
|
||||
@@ -446,7 +454,11 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
|
||||
visible_elements = cap.elements[:max_elements]
|
||||
truncated_elements = max(0, total_elements - len(visible_elements))
|
||||
|
||||
element_index = _format_elements(cap.elements)
|
||||
# Index only what's actually surfaced in the response — otherwise the
|
||||
# human-readable summary references element indices the model cannot
|
||||
# find in the JSON `elements` array (e.g. max_elements=10 vs the default
|
||||
# 40-line index window).
|
||||
element_index = _format_elements(visible_elements)
|
||||
summary_lines = [
|
||||
f"capture mode={cap.mode} {cap.width}x{cap.height}"
|
||||
+ (f" app={cap.app}" if cap.app else "")
|
||||
@@ -455,12 +467,6 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
|
||||
]
|
||||
if element_index:
|
||||
summary_lines.extend(element_index)
|
||||
if truncated_elements:
|
||||
summary_lines.append(
|
||||
f" (response truncated to {len(visible_elements)} of {total_elements} elements; "
|
||||
f"raise max_elements or pass app= to narrow)"
|
||||
)
|
||||
summary = "\n".join(summary_lines)
|
||||
|
||||
if cap.png_b64 and cap.mode != "ax":
|
||||
# Decide whether to hand the screenshot to the auxiliary.vision
|
||||
@@ -483,6 +489,10 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
|
||||
# JPEG: base64 starts with /9j/ PNG: starts with iVBOR
|
||||
_b64_prefix = cap.png_b64[:8]
|
||||
_mime = "image/jpeg" if _b64_prefix.startswith("/9j/") else "image/png"
|
||||
# The multimodal response carries the screenshot, not the AX
|
||||
# elements array, so a "response truncated to N of M elements"
|
||||
# note would be inaccurate — skip it on this branch.
|
||||
summary = "\n".join(summary_lines)
|
||||
return {
|
||||
"_multimodal": True,
|
||||
"content": [
|
||||
@@ -494,7 +504,14 @@ def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEME
|
||||
"meta": {"mode": cap.mode, "width": cap.width, "height": cap.height,
|
||||
"elements": total_elements, "png_bytes": cap.png_bytes_len},
|
||||
}
|
||||
# AX-only (or image missing): text path.
|
||||
# AX-only (or image-missing fallback): text path actually carries the
|
||||
# `elements` array, so the truncation note applies here.
|
||||
if truncated_elements:
|
||||
summary_lines.append(
|
||||
f" (response truncated to {len(visible_elements)} of {total_elements} elements; "
|
||||
f"raise max_elements or pass app= to narrow)"
|
||||
)
|
||||
summary = "\n".join(summary_lines)
|
||||
payload: Dict[str, Any] = {
|
||||
"mode": cap.mode,
|
||||
"width": cap.width,
|
||||
|
||||
Reference in New Issue
Block a user