diff --git a/api.md b/api.md index 243a45d..7fb98c6 100644 --- a/api.md +++ b/api.md @@ -505,7 +505,7 @@ from yutori.navigator import ( TOOL_SET_COMPUTER_USE_20260825, TOOL_SET_COMPUTER_USE_20260830, # Navigator n2 loop helpers N2Computer, N2ComputerAgent, N2Compactor, N2InlineCompactor, - parse_n2_tool_calls, execute_n2_computer_call, retain_n2_image_window, + parse_n2_tool_calls, execute_n2_computer_call, prune_n2_screenshots_to_budget, # Navigator n2 bash/file-tool reference implementations ShellFileToolsMixin, format_shell_output, render_image_result, FILE_TOOL_SCRIPT, # Screenshots @@ -569,7 +569,7 @@ Window scope drives one application window in the background instead of the visi The run ends when the model answers with text and no tool calls, a callback's `on_run_continue` returns `False`, a `max_steps`/`agent_timeout_seconds` budget is spent, or the next request would exceed `context_window_tokens`; `agent.stopped_by` records which (`"final_answer"`, `"callback"`, `"max_steps"`, `"timeout"`, `"context_limit"`). Final text passes through untouched. `resume(message)` appends a user message to `agent.trajectory` and continues the same conversation, so the caller decides what a text-only turn means — answer a question, steer, or stop; a caller who wants an explicit completion convention (say, a `[DONE]` marker) asks for it in `system_prompt` and resumes until it appears. Each request echoes the previous response's `request_id` as `prev_request_id` (`run()` starts a new chain, `resume()` continues it), so the platform reports the whole conversation as one session. -Request rendering: the run starts without a screenshot — the model asks for one with a `screenshot` batch member. Frames are sent at the handler's own capture size, re-encoded to `image_format` (never resized); older frames are replaced by `[older image omitted]`; prior-turn reasoning is re-sent as the assistant message's `reasoning`/`reasoning_content` fields. Two responses are re-requested once instead of kept: a turn whose text carries literal `` markup but parsed no tool calls (retried with a format reminder, `TOOL_CALL_FORMAT_NUDGE`; the check is `needs_tool_call_format_nudge`), and a turn cut off at the output cap (`finish_reason == "length"`) with no tool calls. Neither attempt enters the kept trajectory. +Request rendering: the run starts without a screenshot — the model asks for one with a `screenshot` batch member. Frames are sent at the handler's own capture size, re-encoded to `image_format` (never resized). Every frame the run has taken rides along on every request: the server keeps images only in the two newest image-bearing messages before it serves the model, so a client-side window changes nothing the model reads while emptying the request log the run's replay is built from. `prune_n2_screenshots_to_budget` drops frames only when the serialized request would exceed the 10 MB cap, oldest first, never the newest, and leaves no marker in their place — a dropped frame reads exactly like one the server's own window stripped. It raises `ValueError` if the request cannot fit even with one frame left. Prior-turn reasoning is re-sent as the assistant message's `reasoning`/`reasoning_content` fields. Two responses are re-requested once instead of kept: a turn whose text carries literal `` markup but parsed no tool calls (retried with a format reminder, `TOOL_CALL_FORMAT_NUDGE`; the check is `needs_tool_call_format_nudge`), and a turn cut off at the output cap (`finish_reason == "length"`) with no tool calls. Neither attempt enters the kept trajectory. #### The adapter contract (`N2Computer`) diff --git a/tests/test_navigator_n2.py b/tests/test_navigator_n2.py index bf19e9a..f2b547c 100644 --- a/tests/test_navigator_n2.py +++ b/tests/test_navigator_n2.py @@ -27,6 +27,7 @@ parse_n2_key_expression, parse_n2_tool_calls, prepare_n2_image_data_url, + prune_n2_screenshots_to_budget, retain_n2_image_window, translate_n2_action, translate_n2_bash, @@ -37,13 +38,20 @@ from yutori.navigator.macos.types import CancellationLatch, N2Observation from yutori.navigator.n2 import _CallbackDispatcher from yutori.navigator.n2_payload import ( - fit_n2_request_images_to_budget, + DEFAULT_MAX_MESSAGES_BYTES, image_dimensions, ) from .conftest import FakeCompletions +def _images(message: dict[str, Any]) -> list[dict[str, Any]]: + content = message.get("content") + if not isinstance(content, list): + return [] + return [part for part in content if part.get("type") == "image_url"] + + def _png_data_url(width: int = 200, height: int = 100) -> str: buffer = io.BytesIO() Image.new("RGB", (width, height), (10, 20, 30)).save(buffer, format="PNG") @@ -341,17 +349,68 @@ def image_message(url): assert all(message["content"] for message in messages) -def test_budget_drops_the_older_image_then_raises(): +def test_a_history_that_fits_keeps_every_frame(): + def image_message(index): + return { + "role": "tool", + "content": [ + {"type": "text", "text": f"[{index}:left_click]"}, + {"type": "image_url", "image_url": {"url": _png_data_url(40, 30)}}, + ], + } + + messages = [image_message(index) for index in range(8)] + assert prune_n2_screenshots_to_budget(messages, DEFAULT_MAX_MESSAGES_BYTES) == 0 + assert sum(1 for message in messages if _images(message)) == 8 + + +def test_budget_drops_oldest_frames_first_and_leaves_no_marker(): + big_url = _png_data_url(600, 400) + + def image_message(): + return { + "role": "tool", + "content": [ + {"type": "text", "text": "[0:left_click]"}, + {"type": "image_url", "image_url": {"url": big_url}}, + ], + } + + messages = [image_message() for _ in range(4)] + # Room for the newest frame and the run's text, and nothing else. + dropped = prune_n2_screenshots_to_budget(messages, len(big_url) + 400) + assert dropped == 3 + assert [bool(_images(message)) for message in messages] == [False, False, False, True] + # A dropped frame leaves the message's own text and nothing in its place: + # the server strips its own window's frames the same way. + assert messages[0]["content"] == [{"type": "text", "text": "[0:left_click]"}] + + +def test_budget_raises_when_even_the_newest_frame_cannot_fit(): big_url = _png_data_url(600, 400) def image_message(): return {"role": "user", "content": [{"type": "image_url", "image_url": {"url": big_url}}]} - budget = len(big_url) + 200 - fitted = fit_n2_request_images_to_budget([image_message(), image_message()], budget) - assert not fitted[0]["content"] and fitted[1]["content"] with pytest.raises(ValueError, match="cannot fit"): - fit_n2_request_images_to_budget([image_message(), image_message()], 100) + prune_n2_screenshots_to_budget([image_message(), image_message()], 100) + + +def test_budget_drains_extra_images_sharing_the_newest_message(): + big_url = _png_data_url(600, 400) + messages = [ + { + "role": "tool", + "content": [ + {"type": "text", "text": "read"}, + {"type": "image_url", "image_url": {"url": big_url}}, + {"type": "image_url", "image_url": {"url": big_url}}, + ], + } + ] + # Only the last image part is protected, not every image in its message. + assert prune_n2_screenshots_to_budget(messages, len(big_url) + 400) == 1 + assert len(_images(messages[0])) == 1 # --------------------------------------------------------------------------- @@ -1203,7 +1262,7 @@ def test_agent_rejects_unknown_tool_sets_and_missing_credentials(): N2ComputerAgent(computer=FakeComputer(), tool_set=TOOL_SET_COMPUTER_USE_HYBRID_BATCH) -async def test_agent_requests_stay_within_the_two_image_window(): +async def test_agent_requests_carry_every_frame_the_run_took(): turns = [] for index in range(3): turns.append( @@ -1239,7 +1298,10 @@ async def test_agent_requests_stay_within_the_two_image_window(): for part in (message.get("content") if isinstance(message.get("content"), list) else []) if isinstance(part, dict) and part.get("type") == "image_url" ] - assert len(image_parts) == 2 + # Three clicks, three frames — the whole run, not a two-image window. The + # server applies its own window before serving the model; what the client + # sends is what the run's replay is built from. + assert len(image_parts) == 3 # Every image the wire carries is the default WebP re-encode of the raw capture. assert all(part["image_url"]["url"].startswith("data:image/webp;") for part in image_parts) diff --git a/tests/test_navigator_n2_harness.py b/tests/test_navigator_n2_harness.py index 8a1c0f7..603aff0 100644 --- a/tests/test_navigator_n2_harness.py +++ b/tests/test_navigator_n2_harness.py @@ -277,7 +277,13 @@ async def test_harness_loop_starts_blind_and_attaches_one_frame_per_gui_turn(): assert final == "All set. [DONE]" -async def test_pruned_frames_leave_the_harness_marker_in_place(): +async def test_every_frame_of_the_run_rides_along_with_no_marker(): + """The request carries the whole history, which is what the replay is built from. + + The server keeps images only in the two newest image-bearing messages before + it serves the model, so windowing here would change nothing the model reads + while emptying the run's replay of every step but the last two. + """ click = _batch({"name": "left_click", "arguments": {"coordinates": [500, 500]}}) completions = FakeCompletions( [{"content": "", "tool_calls": [click]} for _ in range(3)] + [{"content": "done [DONE]", "tool_calls": []}] @@ -287,10 +293,11 @@ async def test_pruned_frames_leave_the_harness_marker_in_place(): pass last = completions.requests[-1]["messages"] tool_messages = [message for message in last if message["role"] == "tool"] - assert [len(_images(message)) for message in tool_messages] == [0, 1, 1] - # The marker concatenates into the preceding text part, one merged block — - # the reference builder's rendering of a pruned frame. - assert tool_messages[0]["content"] == [{"type": "text", "text": "[0:left_click][older image omitted]"}] + assert [len(_images(message)) for message in tool_messages] == [1, 1, 1] + # Each frame still rides with its own result text, and nothing stands in for + # a frame that was never dropped. + assert tool_messages[0]["content"][0] == {"type": "text", "text": "[0:left_click]"} + assert "omitted" not in json.dumps(last) async def test_harness_loop_sizes_a_blind_start_from_the_handler_dimensions(): diff --git a/yutori/navigator/__init__.py b/yutori/navigator/__init__.py index dd6746b..41b0c95 100644 --- a/yutori/navigator/__init__.py +++ b/yutori/navigator/__init__.py @@ -82,6 +82,7 @@ DEFAULT_IMAGE_FORMAT, OLDER_IMAGE_OMITTED_TEXT, prepare_n2_image_data_url, + prune_n2_screenshots_to_budget, retain_n2_image_window, ) from .payload import ( @@ -159,6 +160,7 @@ "parse_n2_tool_calls", "playwright_screenshot_to_data_url", "prepare_n2_image_data_url", + "prune_n2_screenshots_to_budget", "retain_n2_image_window", "screenshot_to_data_url", "translate_n2_action", diff --git a/yutori/navigator/n2.py b/yutori/navigator/n2.py index 2696f8c..92e22d1 100644 --- a/yutori/navigator/n2.py +++ b/yutori/navigator/n2.py @@ -89,10 +89,9 @@ DEFAULT_MAX_MESSAGES_BYTES, MAX_REQUEST_BODY_BYTES, convert_request_images, - fit_n2_request_images_to_budget, image_dimensions, latest_image_url, - retain_n2_image_window, + prune_n2_screenshots_to_budget, serialized_messages_bytes, ) @@ -1339,13 +1338,14 @@ def _prepare_completion_messages(self, items: list[dict[str, Any]]) -> list[dict completion_messages = convert_n2_items_to_completion_messages(copy.deepcopy(items)) if self.system_prompt: completion_messages.insert(0, {"role": "system", "content": self.system_prompt}) - # Strip historical screenshots before compression so long-running - # trajectories do not repeatedly re-encode images that will not be - # sent. Apply the byte budget after conversion because it measures the - # actual request representation. - completion_messages = retain_n2_image_window(completion_messages) + # Send every frame the run has taken and let the budget decide, rather + # than windowing first: the server keeps images only in the two newest + # image-bearing messages before it serves the model, so trimming here + # changes nothing the model sees — it only empties the request log the + # run's replay is built from. Convert before pruning, because the budget + # has to measure the representation that actually goes on the wire. convert_request_images(completion_messages, self.image_format) - completion_messages = fit_n2_request_images_to_budget(completion_messages, DEFAULT_MAX_MESSAGES_BYTES) + prune_n2_screenshots_to_budget(completion_messages, DEFAULT_MAX_MESSAGES_BYTES) request_bytes = serialized_messages_bytes(completion_messages) if request_bytes > MAX_REQUEST_BODY_BYTES: diff --git a/yutori/navigator/n2_payload.py b/yutori/navigator/n2_payload.py index 62237c2..467fada 100644 --- a/yutori/navigator/n2_payload.py +++ b/yutori/navigator/n2_payload.py @@ -2,9 +2,17 @@ n2 requests carry the computer handler's screenshots as captured — the handler defines the viewport (with any DPR scaling already removed); the SDK never -resizes, only re-encodes to ``image_format`` (WebP by default). Requests keep -images only in the two newest image-bearing messages (older ones leave an -``[older image omitted]`` marker) and must fit a 10 MB serialized request. +resizes, only re-encodes to ``image_format`` (WebP by default). + +Requests carry the WHOLE screenshot history and must fit a 10 MB serialized +request. Sending every frame is deliberate, and matches the reference harness: +the server already keeps images only in the two newest image-bearing messages +before it serves the model, so a client-side window buys the model nothing — +while the request log the run's replay is built from records exactly the +messages the client sent, so a client that trims first is the only reason a +replay has gaps. :func:`prune_n2_screenshots_to_budget` therefore drops frames +only to fit the wire cap, oldest first. + Coordinates are the model's 0-1000 space mapped onto the capture's dimensions. """ @@ -13,6 +21,7 @@ import base64 import copy import io +import json from typing import Any, Optional from PIL import Image @@ -24,7 +33,13 @@ DEFAULT_IMAGE_FORMAT = "webp" MAX_REQUEST_BODY_BYTES = 10_000_000 -REQUEST_ENVELOPE_ALLOWANCE_BYTES = 500_000 +# Slack left below the cap when deciding how much screenshot history fits: the +# budget is measured over the messages array alone, so this covers the rest of +# the serialized body (model, tool_set, sampling fields, JSON structure) and +# keeps the loop's own exact-size guard from tripping after a prune. Matches the +# reference harness's headroom; a larger allowance only throws away frames that +# would have fit. +REQUEST_ENVELOPE_ALLOWANCE_BYTES = 64 * 1024 DEFAULT_MAX_MESSAGES_BYTES = MAX_REQUEST_BODY_BYTES - REQUEST_ENVELOPE_ALLOWANCE_BYTES @@ -44,15 +59,29 @@ def image_dimensions(url: str) -> "tuple[int, int]": return image.size +def _data_url_media_type(url: str) -> str: + """The media type of a base64 data URL, without decoding its payload.""" + if not isinstance(url, str) or not url.startswith("data:") or "," not in url: + raise ValueError("n2 screenshots must be base64 data URLs") + header = url.split(",", 1)[0] + if ";base64" not in header: + raise ValueError("n2 screenshots must use base64 data URLs") + return header[5:].split(";", 1)[0] + + def prepare_n2_image_data_url(url: str, image_format: str = DEFAULT_IMAGE_FORMAT) -> str: """Re-encode an image data URL to ``image_format``; returned unchanged when it already is. Never resizes: the frame stays at whatever size the computer handler captured (its viewport, with any DPR scaling already removed). """ - image_bytes, media_type = _decode_data_url(url) - if media_type.lower() == f"image/{image_format.lower()}": + # Read the media type off the header rather than decoding first: a request + # now carries the whole history, so the pass-through case is walked once per + # frame per step, and base64-decoding megabytes only to compare a string is + # the kind of cost that shows up as latency on a long run. + if _data_url_media_type(url).lower() == f"image/{image_format.lower()}": return url + image_bytes, _ = _decode_data_url(url) with Image.open(io.BytesIO(image_bytes)) as source: output = io.BytesIO() source.convert("RGB").save(output, format=image_format.upper()) @@ -112,6 +141,11 @@ def _strip_images_from_message(message: dict[str, Any], omitted_text: Optional[s serialized_messages_bytes = estimate_messages_size_bytes +def _serialized_bytes(value: Any) -> int: + """The JSON-serialized byte size of one content part, matching the messages estimate.""" + return len(json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode("utf-8")) + + def retain_n2_image_window( messages: list[dict[str, Any]], *, omitted_text: Optional[str] = OLDER_IMAGE_OMITTED_TEXT ) -> list[dict[str, Any]]: @@ -120,6 +154,13 @@ def retain_n2_image_window( Each pruned image is replaced in place by the ``omitted_text`` block (by default :data:`OLDER_IMAGE_OMITTED_TEXT`); with ``None`` the image part is dropped. + + The loop does NOT apply this — it sends the full history and lets + :func:`prune_n2_screenshots_to_budget` drop only what will not fit, because + the server applies this same window itself before serving the model. Kept + for a harness that has its own reason to send less than it has (a metered + uplink, say), and as the executable statement of what the server's window + does. """ request_messages = copy.deepcopy(messages) image_indices = [index for index, message in enumerate(request_messages) if _message_image_parts(message)] @@ -128,27 +169,81 @@ def retain_n2_image_window( return request_messages -def fit_n2_request_images_to_budget( - messages: list[dict[str, Any]], - max_messages_bytes: int = DEFAULT_MAX_MESSAGES_BYTES, -) -> list[dict[str, Any]]: - """Copy an already-windowed request and drop its older image message if needed.""" - request_messages = copy.deepcopy(messages) - image_indices = [index for index, message in enumerate(request_messages) if _message_image_parts(message)] - - if serialized_messages_bytes(request_messages) <= max_messages_bytes: - return request_messages +def _drop_first_image(content: list[Any]) -> "dict[str, Any] | None": + """Remove the first ``image_url`` part from a content list, returning it.""" + for position, part in enumerate(content): + if isinstance(part, dict) and part.get("type") == "image_url": + return content.pop(position) + return None - retained_indices = image_indices[-2:] - if len(retained_indices) == 2: - _strip_images_from_message(request_messages[retained_indices[0]]) - if serialized_messages_bytes(request_messages) <= max_messages_bytes: - return request_messages - raise ValueError( - "The newest n2 screenshot message cannot fit within the serialized messages budget. " - "Reduce screenshot dimensions/quality or shorten non-image request content." - ) +def prune_n2_screenshots_to_budget( + messages: list[dict[str, Any]], + max_messages_bytes: int = DEFAULT_MAX_MESSAGES_BYTES, +) -> int: + """Drop the oldest screenshots, in place, until *messages* fits the budget. + + Returns how many were dropped, so a caller can surface that the run's replay + was truncated. Nothing is dropped when the history already fits, which is the + common case and the whole point: every frame reaches the request log the + replay is built from. + + A dropped frame leaves NO marker. That is what the server's own window does + to the frames it strips, so a request pruned here and a request pruned there + reach the model as the same conversation; injecting a marker per dropped + frame instead hands the model text the reference harness never produces. + The newest image is never dropped — it is the observation the model is being + asked to act on. + + Raises: + ValueError: if the request cannot fit even with one frame left. + """ + size_bytes = serialized_messages_bytes(messages) + if size_bytes <= max_messages_bytes: + return 0 + + # One entry per image part, oldest first; a message holding several images + # appears once per image, and each visit takes that message's first + # remaining one. The last entry is the current observation and is never + # visited. + image_contents: list[list[Any]] = [] + for message in messages: + content = message.get("content") + if not isinstance(content, list): + continue + image_contents.extend(content for part in _message_image_parts(message)) + + dropped = 0 + # Running estimate rather than a re-serialization per drop: the payload is + # megabytes by construction, so measuring it once per dropped frame turned a + # trim of N frames into N passes over all of it. Dropping an array element + # removes its serialization plus one separating comma; an image that was the + # only part of its content list has no comma to remove, so the estimate can + # run one byte low per such frame. The exact re-measure below settles it. + for content in image_contents[:-1]: + if size_bytes <= max_messages_bytes: + break + part = _drop_first_image(content) + if part is None: + continue + size_bytes -= _serialized_bytes(part) + 1 + dropped += 1 + + size_bytes = serialized_messages_bytes(messages) + for content in image_contents[:-1]: + if size_bytes <= max_messages_bytes: + break + if _drop_first_image(content) is None: + continue + dropped += 1 + size_bytes = serialized_messages_bytes(messages) + + if size_bytes > max_messages_bytes: + raise ValueError( + "The newest n2 screenshot message cannot fit within the serialized messages budget. " + "Reduce screenshot dimensions/quality or shorten non-image request content." + ) + return dropped def convert_request_images(messages: list[dict[str, Any]], image_format: str = DEFAULT_IMAGE_FORMAT) -> None: