fix(computer_use): correct type_text MCP tool name and implement drag action

Bug 3: The cua_backend type_text() method called MCP tool 'type_text_chars'
which does not exist in current cua-driver. Changed to 'type_text' which is
the correct MCP tool name.

Bug 4: The drag() method returned a hardcoded 'not supported' error even
though cua-driver exposes a 'drag' MCP tool. Implemented proper drag
dispatching with coordinate-based and element-based targeting.

Added dispatch-level validation for drag to ensure from/to coordinates
or elements are provided before calling any backend.

Fixes #24170 (bugs 3 and 4)
This commit is contained in:
liuhao1024
2026-05-12 13:02:28 +08:00
committed by Teknium
parent 0ce12a9241
commit 18cd1e5c72
3 changed files with 76 additions and 7 deletions

View File

@@ -155,6 +155,56 @@ class TestDispatch:
click_kw = next(c[1] for c in noop_backend.calls if c[0] == "click")
assert click_kw["button"] == "right"
def test_type_action_routes_to_type_text_backend(self, noop_backend):
"""type action must call backend.type_text, not type_text_chars (issue #24170, bug 3)."""
from tools.computer_use.tool import handle_computer_use
out = handle_computer_use({"action": "type", "text": "hello"})
parsed = json.loads(out)
assert "error" not in parsed
call_names = [c[0] for c in noop_backend.calls]
assert "type" in call_names
type_kw = next(c[1] for c in noop_backend.calls if c[0] == "type")
assert type_kw["text"] == "hello"
def test_drag_action_routes_to_backend_by_coordinate(self, noop_backend):
"""drag action must dispatch to backend.drag with coordinates (issue #24170, bug 4)."""
from tools.computer_use.tool import handle_computer_use
out = handle_computer_use({
"action": "drag",
"from_coordinate": [100, 200],
"to_coordinate": [400, 500],
})
parsed = json.loads(out)
assert "error" not in parsed
call_names = [c[0] for c in noop_backend.calls]
assert "drag" in call_names
drag_kw = next(c[1] for c in noop_backend.calls if c[0] == "drag")
assert drag_kw["from_xy"] == (100, 200)
assert drag_kw["to_xy"] == (400, 500)
def test_drag_action_routes_to_backend_by_element(self, noop_backend):
"""drag action must dispatch to backend.drag with element indices (issue #24170, bug 4)."""
from tools.computer_use.tool import handle_computer_use
out = handle_computer_use({
"action": "drag",
"from_element": 1,
"to_element": 5,
})
parsed = json.loads(out)
assert "error" not in parsed
call_names = [c[0] for c in noop_backend.calls]
assert "drag" in call_names
drag_kw = next(c[1] for c in noop_backend.calls if c[0] == "drag")
assert drag_kw["from_element"] == 1
assert drag_kw["to_element"] == 5
def test_drag_action_requires_coordinates_or_elements(self, noop_backend):
"""drag without from/to must return an error."""
from tools.computer_use.tool import handle_computer_use
out = handle_computer_use({"action": "drag"})
parsed = json.loads(out)
assert "error" in parsed
# ---------------------------------------------------------------------------
# Safety guards (type / key block lists)