fix(n2): send the whole screenshot history, pruning only to fit the wire cap

api.md

@@ -505,7 +505,7 @@ from yutori.navigator import (
     TOOL_SET_COMPUTER_USE_20260825, TOOL_SET_COMPUTER_USE_20260830,
     # Navigator n2 loop helpers
     N2Computer, N2ComputerAgent, N2Compactor, N2InlineCompactor,
-    parse_n2_tool_calls, execute_n2_computer_call, retain_n2_image_window,
+    parse_n2_tool_calls, execute_n2_computer_call, prune_n2_screenshots_to_budget,
     # Navigator n2 bash/file-tool reference implementations
     ShellFileToolsMixin, format_shell_output, render_image_result, FILE_TOOL_SCRIPT,
     # Screenshots
@@ -569,7 +569,7 @@ Window scope drives one application window in the background instead of the visi
 
 The run ends when the model answers with text and no tool calls, a callback's `on_run_continue` returns `False`, a `max_steps`/`agent_timeout_seconds` budget is spent, or the next request would exceed `context_window_tokens`; `agent.stopped_by` records which (`"final_answer"`, `"callback"`, `"max_steps"`, `"timeout"`, `"context_limit"`). Final text passes through untouched. `resume(message)` appends a user message to `agent.trajectory` and continues the same conversation, so the caller decides what a text-only turn means — answer a question, steer, or stop; a caller who wants an explicit completion convention (say, a `[DONE]` marker) asks for it in `system_prompt` and resumes until it appears. Each request echoes the previous response's `request_id` as `prev_request_id` (`run()` starts a new chain, `resume()` continues it), so the platform reports the whole conversation as one session.
 
-Request rendering: the run starts without a screenshot — the model asks for one with a `screenshot` batch member. Frames are sent at the handler's own capture size, re-encoded to `image_format` (never resized); older frames are replaced by `[older image omitted]`; prior-turn reasoning is re-sent as the assistant message's `reasoning`/`reasoning_content` fields. Two responses are re-requested once instead of kept: a turn whose text carries literal `<tool_call>` markup but parsed no tool calls (retried with a format reminder, `TOOL_CALL_FORMAT_NUDGE`; the check is `needs_tool_call_format_nudge`), and a turn cut off at the output cap (`finish_reason == "length"`) with no tool calls. Neither attempt enters the kept trajectory.
+Request rendering: the run starts without a screenshot — the model asks for one with a `screenshot` batch member. Frames are sent at the handler's own capture size, re-encoded to `image_format` (never resized). Every frame the run has taken rides along on every request: the server keeps images only in the two newest image-bearing messages before it serves the model, so a client-side window changes nothing the model reads while emptying the request log the run's replay is built from. `prune_n2_screenshots_to_budget` drops frames only when the serialized request would exceed the 10 MB cap, oldest first, never the newest, and leaves no marker in their place — a dropped frame reads exactly like one the server's own window stripped. It raises `ValueError` if the request cannot fit even with one frame left. Prior-turn reasoning is re-sent as the assistant message's `reasoning`/`reasoning_content` fields. Two responses are re-requested once instead of kept: a turn whose text carries literal `<tool_call>` markup but parsed no tool calls (retried with a format reminder, `TOOL_CALL_FORMAT_NUDGE`; the check is `needs_tool_call_format_nudge`), and a turn cut off at the output cap (`finish_reason == "length"`) with no tool calls. Neither attempt enters the kept trajectory.
 
 #### The adapter contract (`N2Computer`)
 

tests/test_navigator_n2.py

@@ -27,6 +27,7 @@
     parse_n2_key_expression,
     parse_n2_tool_calls,
     prepare_n2_image_data_url,
+    prune_n2_screenshots_to_budget,
     retain_n2_image_window,
     translate_n2_action,
     translate_n2_bash,
@@ -37,13 +38,20 @@
 from yutori.navigator.macos.types import CancellationLatch, N2Observation
 from yutori.navigator.n2 import _CallbackDispatcher
 from yutori.navigator.n2_payload import (
-    fit_n2_request_images_to_budget,
+    DEFAULT_MAX_MESSAGES_BYTES,
     image_dimensions,
 )
 
 from .conftest import FakeCompletions
 
 
+def _images(message: dict[str, Any]) -> list[dict[str, Any]]:
+    content = message.get("content")
+    if not isinstance(content, list):
+        return []
+    return [part for part in content if part.get("type") == "image_url"]
+
+
 def _png_data_url(width: int = 200, height: int = 100) -> str:
     buffer = io.BytesIO()
     Image.new("RGB", (width, height), (10, 20, 30)).save(buffer, format="PNG")
@@ -341,17 +349,68 @@ def image_message(url):
     assert all(message["content"] for message in messages)
 
 
-def test_budget_drops_the_older_image_then_raises():
+def test_a_history_that_fits_keeps_every_frame():
+    def image_message(index):
+        return {
+            "role": "tool",
+            "content": [
+                {"type": "text", "text": f"[{index}:left_click]"},
+                {"type": "image_url", "image_url": {"url": _png_data_url(40, 30)}},
+            ],
+        }
+
+    messages = [image_message(index) for index in range(8)]
+    assert prune_n2_screenshots_to_budget(messages, DEFAULT_MAX_MESSAGES_BYTES) == 0
+    assert sum(1 for message in messages if _images(message)) == 8
+
+
+def test_budget_drops_oldest_frames_first_and_leaves_no_marker():
+    big_url = _png_data_url(600, 400)
+
+    def image_message():
+        return {
+            "role": "tool",
+            "content": [
+                {"type": "text", "text": "[0:left_click]"},
+                {"type": "image_url", "image_url": {"url": big_url}},
+            ],
+        }
+
+    messages = [image_message() for _ in range(4)]
+    # Room for the newest frame and the run's text, and nothing else.
+    dropped = prune_n2_screenshots_to_budget(messages, len(big_url) + 400)
+    assert dropped == 3
+    assert [bool(_images(message)) for message in messages] == [False, False, False, True]
+    # A dropped frame leaves the message's own text and nothing in its place:
+    # the server strips its own window's frames the same way.
+    assert messages[0]["content"] == [{"type": "text", "text": "[0:left_click]"}]
+
+
+def test_budget_raises_when_even_the_newest_frame_cannot_fit():
     big_url = _png_data_url(600, 400)
 
     def image_message():
         return {"role": "user", "content": [{"type": "image_url", "image_url": {"url": big_url}}]}
 
-    budget = len(big_url) + 200
-    fitted = fit_n2_request_images_to_budget([image_message(), image_message()], budget)
-    assert not fitted[0]["content"] and fitted[1]["content"]
     with pytest.raises(ValueError, match="cannot fit"):
-        fit_n2_request_images_to_budget([image_message(), image_message()], 100)
+        prune_n2_screenshots_to_budget([image_message(), image_message()], 100)
+
+
+def test_budget_drains_extra_images_sharing_the_newest_message():
+    big_url = _png_data_url(600, 400)
+    messages = [
+        {
+            "role": "tool",
+            "content": [
+                {"type": "text", "text": "read"},
+                {"type": "image_url", "image_url": {"url": big_url}},
+                {"type": "image_url", "image_url": {"url": big_url}},
+            ],
+        }
+    ]
+    # Only the last image part is protected, not every image in its message.
+    assert prune_n2_screenshots_to_budget(messages, len(big_url) + 400) == 1
+    assert len(_images(messages[0])) == 1
 
 
 # ---------------------------------------------------------------------------
@@ -1203,7 +1262,7 @@ def test_agent_rejects_unknown_tool_sets_and_missing_credentials():
         N2ComputerAgent(computer=FakeComputer(), tool_set=TOOL_SET_COMPUTER_USE_HYBRID_BATCH)
 
 
-async def test_agent_requests_stay_within_the_two_image_window():
+async def test_agent_requests_carry_every_frame_the_run_took():
     turns = []
     for index in range(3):
         turns.append(
@@ -1239,7 +1298,10 @@ async def test_agent_requests_stay_within_the_two_image_window():
         for part in (message.get("content") if isinstance(message.get("content"), list) else [])
         if isinstance(part, dict) and part.get("type") == "image_url"
     ]
-    assert len(image_parts) == 2
+    # Three clicks, three frames — the whole run, not a two-image window. The
+    # server applies its own window before serving the model; what the client
+    # sends is what the run's replay is built from.
+    assert len(image_parts) == 3
     # Every image the wire carries is the default WebP re-encode of the raw capture.
     assert all(part["image_url"]["url"].startswith("data:image/webp;") for part in image_parts)
 

tests/test_navigator_n2_harness.py

@@ -277,7 +277,13 @@ async def test_harness_loop_starts_blind_and_attaches_one_frame_per_gui_turn():
     assert final == "All set. [DONE]"
 
 
-async def test_pruned_frames_leave_the_harness_marker_in_place():
+async def test_every_frame_of_the_run_rides_along_with_no_marker():
+    """The request carries the whole history, which is what the replay is built from.
+
+    The server keeps images only in the two newest image-bearing messages before
+    it serves the model, so windowing here would change nothing the model reads
+    while emptying the run's replay of every step but the last two.
+    """
     click = _batch({"name": "left_click", "arguments": {"coordinates": [500, 500]}})
     completions = FakeCompletions(
         [{"content": "", "tool_calls": [click]} for _ in range(3)] + [{"content": "done [DONE]", "tool_calls": []}]
@@ -287,10 +293,11 @@ async def test_pruned_frames_leave_the_harness_marker_in_place():
         pass
     last = completions.requests[-1]["messages"]
     tool_messages = [message for message in last if message["role"] == "tool"]
-    assert [len(_images(message)) for message in tool_messages] == [0, 1, 1]
-    # The marker concatenates into the preceding text part, one merged block —
-    # the reference builder's rendering of a pruned frame.
-    assert tool_messages[0]["content"] == [{"type": "text", "text": "[0:left_click][older image omitted]"}]
+    assert [len(_images(message)) for message in tool_messages] == [1, 1, 1]
+    # Each frame still rides with its own result text, and nothing stands in for
+    # a frame that was never dropped.
+    assert tool_messages[0]["content"][0] == {"type": "text", "text": "[0:left_click]"}
+    assert "omitted" not in json.dumps(last)
 
 
 async def test_harness_loop_sizes_a_blind_start_from_the_handler_dimensions():

yutori/navigator/__init__.py

@@ -82,6 +82,7 @@
     DEFAULT_IMAGE_FORMAT,
     OLDER_IMAGE_OMITTED_TEXT,
     prepare_n2_image_data_url,
+    prune_n2_screenshots_to_budget,
     retain_n2_image_window,
 )
 from .payload import (
@@ -159,6 +160,7 @@
     "parse_n2_tool_calls",
     "playwright_screenshot_to_data_url",
     "prepare_n2_image_data_url",
+    "prune_n2_screenshots_to_budget",
     "retain_n2_image_window",
     "screenshot_to_data_url",
     "translate_n2_action",

yutori/navigator/n2.py

@@ -89,10 +89,9 @@
     DEFAULT_MAX_MESSAGES_BYTES,
     MAX_REQUEST_BODY_BYTES,
     convert_request_images,
-    fit_n2_request_images_to_budget,
     image_dimensions,
     latest_image_url,
-    retain_n2_image_window,
+    prune_n2_screenshots_to_budget,
     serialized_messages_bytes,
 )
 
@@ -1339,13 +1338,14 @@ def _prepare_completion_messages(self, items: list[dict[str, Any]]) -> list[dict
         completion_messages = convert_n2_items_to_completion_messages(copy.deepcopy(items))
         if self.system_prompt:
             completion_messages.insert(0, {"role": "system", "content": self.system_prompt})
-        # Strip historical screenshots before compression so long-running
-        # trajectories do not repeatedly re-encode images that will not be
-        # sent. Apply the byte budget after conversion because it measures the
-        # actual request representation.
-        completion_messages = retain_n2_image_window(completion_messages)
+        # Send every frame the run has taken and let the budget decide, rather
+        # than windowing first: the server keeps images only in the two newest
+        # image-bearing messages before it serves the model, so trimming here
+        # changes nothing the model sees — it only empties the request log the
+        # run's replay is built from. Convert before pruning, because the budget
+        # has to measure the representation that actually goes on the wire.
         convert_request_images(completion_messages, self.image_format)
-        completion_messages = fit_n2_request_images_to_budget(completion_messages, DEFAULT_MAX_MESSAGES_BYTES)
+        prune_n2_screenshots_to_budget(completion_messages, DEFAULT_MAX_MESSAGES_BYTES)
 
         request_bytes = serialized_messages_bytes(completion_messages)
         if request_bytes > MAX_REQUEST_BODY_BYTES: