[decision_models] Compact the browser front's state for Laya's window

CHANGELOG.md

@@ -4,6 +4,13 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver
 
 ## Unreleased
 
+### Added
+
+- `laya_state` (`s1a/decision_models/laya.py`): folds a browser-front state to fit Laya's 512 to 1024 token
+  window before every call — `page.text` dropped, one short line per element row instead of a JSON object, the
+  last three actions instead of ten — roughly a tenfold reduction in the JSON-shaped state on the pages measured.
+  On by default; `LAYA_COMPACT_BROWSER_STATE=0` turns it off. `docs/decision-models.md`.
+
 ### Changed
 
 - `--model` picks the model on every agent, on `decide` and on `probe`: `jev`, `laya`, `cua`, `llm`, `random` or

docs/decision-models.md

@@ -90,6 +90,16 @@ real; `bodies` records every request). `ScriptedModel` fakes the interface for f
 4. An optional extra in `pyproject.toml` and an env block in `.env.example` when it needs a dependency.
 
 Laya is text only and reads a 512 to 1024 token window; it fits the tool front first. The browser front's element
-tables are wider than that window. The window check sums `input_tokens` over the request's questions. On the
-browser front (two to four questions per tick) only a cut on every head raises the config error above; a cut on
-one head goes unseen. `--model laya` on a browser agent needs `LAYA_MAX_LEN` raised to the page's size.
+tables, sent to Jev as-is, ran well past that window on a real page before a single instruction token was spent:
+a JSON object per row, the full page text, and ten actions of history. The window check sums `input_tokens` over
+the request's questions. On the browser front (two to four questions per tick) only a cut on every head raises
+the config error above; a cut on one head goes unseen.
+
+`laya_state` (`s1a/decision_models/laya.py`) folds a browser-shaped state before every call: `page.text` dropped
+(the choice heads already carry each candidate's own text; the free-form dump is for the chat model's DONE
+answer, which Laya never writes), each element row rendered as one short line instead of a JSON object, and the
+last three actions kept instead of ten. On the WebVoyager-style pages measured while adding this, that is
+roughly a tenfold reduction in the JSON-shaped state's size before the tokenizer sees it — the difference between
+routinely filling a 512-token window and, on most pages, comfortably fitting it. It is on by default and skips
+anything that is not the browser front's shape; `LAYA_COMPACT_BROWSER_STATE=0` turns it off. `--model laya` on a
+page whose element table is still too wide for the window needs `LAYA_MAX_LEN` raised, same as before.

s1a/decision_models/laya.py

@@ -21,6 +21,16 @@
 
 LAYA_DEFAULT_MODEL = "convaiinnovations/laya"
 LAYA_DEFAULT_MAX_LEN = 512  # the window Laya assumes when a checkpoint config names none
+LAYA_BROWSER_LABEL_CHARS = 40  # a row's label/value, kept over its full text (Jev's window is 32K; Laya's is not)
+LAYA_BROWSER_TITLE_CHARS = 80
+LAYA_BROWSER_HISTORY_KEPT = 3  # of the browser front's last ten actions; the freshest ones carry the signal
+_FLAG_LETTERS = (
+    ("checked", "C"),
+    ("selected", "S"),
+    ("expanded", "X"),
+    ("blocked_by", "B"),
+    ("click_did_nothing", "D"),
+)
 
 
 def laya_question(question: Question) -> Json:
@@ -32,25 +42,72 @@ def laya_question(question: Question) -> Json:
     return {"type": "noul", "instructions": question.question, **criteria}
 
 
+def _laya_browser_row(row: Json) -> str:
+    """One element row as a short line instead of a JSON object: the repeated key names (``role``, ``label``, ...)
+    are what a tiny window can least afford. ``[CX]``-style flags stand in for the sparse boolean/id fields."""
+    label = str(row.get("label") or "")[:LAYA_BROWSER_LABEL_CHARS]
+    value = str(row.get("value") or "")[:LAYA_BROWSER_LABEL_CHARS]
+    flags = "".join(letter for key, letter in _FLAG_LETTERS if row.get(key))
+    parts = [str(row.get("index", "")), str(row.get("role") or ""), label]
+    if value:
+        parts.append(f"={value}")
+    if flags:
+        parts.append(f"[{flags}]")
+    return " ".join(part for part in parts if part)
+
+
+def laya_state(state: Json | str) -> Json | str:
+    """The browser front's per-tick state, folded to fit Laya's window: no ``page.text`` (the choice heads already
+    carry each candidate's own text; the free-form page dump is for the chat model's DONE answer, which Laya never
+    writes), the element table as one short line per row instead of a JSON object per row, and the last
+    ``LAYA_BROWSER_HISTORY_KEPT`` actions instead of ten. Anything that is not this shape (a plain string, the tool
+    front's state, a rail's) passes through: it already fits the window Laya was sized for.
+
+    This is what made Laya's real-page window error mean anything other than "raise LAYA_MAX_LEN and hope": on a
+    dozen-element page the JSON-shaped state alone ran well past a 512-token window before a single instruction
+    token was spent.
+    """
+    if not (
+        isinstance(state, dict) and isinstance(state.get("page"), dict) and isinstance(state.get("elements"), list)
+    ):
+        return state
+    compact: Json = {
+        "page": {
+            "url": str(state["page"].get("url", "")),
+            "title": str(state["page"].get("title", ""))[:LAYA_BROWSER_TITLE_CHARS],
+        },
+        "elements": [_laya_browser_row(row) for row in state["elements"]],
+    }
+    recent = state.get("recent_actions")
+    if recent:
+        compact["recent_actions"] = [
+            f"{entry.get('kind', '')}:{entry.get('action', '')}" + ("" if entry.get("page_changed") else " (no change)")
+            for entry in recent[-LAYA_BROWSER_HISTORY_KEPT:]
+        ]
+    return compact
+
+
 class LayaModel(DecisionModel):
     """Laya's ``Agent`` (or anything with ``system_one(state, questions)`` and a ``cfg``) behind the interface."""
 
     name = "laya"
     deterministic = True
 
-    def __init__(self, agent: Any, *, model: str) -> None:
+    def __init__(self, agent: Any, *, model: str, compact_browser_state: bool = True) -> None:
         self._agent = agent
         self._model = model
+        self._compact_browser_state = compact_browser_state
 
     @property
     def model(self) -> str:
         return self._model
 
     async def _decide(self, observation: Observation, questions: dict[str, Question]) -> Reply:
         asked = {name: laya_question(question) for name, question in questions.items()}
+        state = laya_state(observation.state) if self._compact_browser_state else observation.state
         started = time.perf_counter()
         try:
-            payload = await asyncio.to_thread(self._agent.system_one, observation.state, asked)
+            payload = await asyncio.to_thread(self._agent.system_one, state, asked)
         except (ValueError, RuntimeError) as exc:  # option overflow, and torch (CUDA included) failures
             raise build_error(
                 StatusCode.MODEL_CALL_FAILED, cause=exc, error_msg=f"laya forward pass failed: {exc}"
@@ -88,7 +145,8 @@ def _check_the_window(self, usage: Usage, questions: int) -> None:
     @classmethod
     def from_env(cls) -> "LayaModel":
         """``LAYA_MODEL`` (a hub id or a path), ``LAYA_SUBFOLDER``, ``LAYA_DEVICE``; ``LAYA_MAX_LEN`` and
-        ``LAYA_HEAD_MAX_LEN`` override the checkpoint's window."""
+        ``LAYA_HEAD_MAX_LEN`` override the checkpoint's window. ``LAYA_COMPACT_BROWSER_STATE`` (default on;
+        ``0``/``false``/``no`` turns it off) folds a browser-shaped state through ``laya_state`` before every call."""
         try:
             import laya
         except ImportError as exc:
@@ -115,4 +173,5 @@ def from_env(cls) -> "LayaModel":
             value = os.getenv(variable)
             if value:
                 agent.cfg[key] = int(value)
-        return cls(agent, model=f"{model}/{subfolder}" if subfolder else model)
+        compact = (os.getenv("LAYA_COMPACT_BROWSER_STATE") or "1").strip().lower() not in ("0", "false", "no")
+        return cls(agent, model=f"{model}/{subfolder}" if subfolder else model, compact_browser_state=compact)

tests/test_decision_models_laya.py

@@ -136,6 +136,82 @@ async def test_the_model_is_deterministic_and_text_only(self) -> None:
         self.assertEqual((decision_model.name, decision_model.model), ("laya", "convaiinnovations/laya"))
 
 
+_BROWSER_STATE = {
+    "page": {"url": "https://example.com/flights", "title": "Google Flights" + "!" * 100, "text": "x" * 5000},
+    "elements": [
+        {
+            "index": "1",
+            "role": "textbox",
+            "label": "Where from?" + " padding" * 20,
+            "value": "",
+            "operations": ["TYPE_TEXT"],
+        },
+        {
+            "index": "2",
+            "role": "button",
+            "label": "Search",
+            "value": "",
+            "checked": True,
+            "operations": ["CLICK"],
+        },
+    ],
+    "recent_actions": [
+        {"action": f"step-{i}", "kind": "click", "text": "", "page_changed": i % 2 == 0} for i in range(10)
+    ],
+}
+
+
+class TestLayaState(TestCase):
+    """``laya_state`` folds the browser front's state to fit Laya's window; anything else passes through."""
+
+    def test_a_non_browser_state_passes_through(self) -> None:
+        for state in ({"page": "x"}, "plain text", {"score": 1}, {"page": {"url": "u"}, "elements": "not a list"}):
+            self.assertEqual(laya_module.laya_state(state), state)
+
+    def test_page_text_is_dropped_and_the_title_is_capped(self) -> None:
+        compact = laya_module.laya_state(_BROWSER_STATE)
+        self.assertEqual(compact["page"]["url"], "https://example.com/flights")
+        self.assertNotIn("text", compact["page"])
+        self.assertLessEqual(len(compact["page"]["title"]), laya_module.LAYA_BROWSER_TITLE_CHARS)
+
+    def test_each_element_row_becomes_one_short_line_not_a_json_object(self) -> None:
+        compact = laya_module.laya_state(_BROWSER_STATE)
+        self.assertEqual(len(compact["elements"]), 2)
+        self.assertTrue(all(isinstance(row, str) for row in compact["elements"]))
+        self.assertLess(len(compact["elements"][0]), len("label") * 20)  # far short of the padded label
+        self.assertIn("[C]", compact["elements"][1])  # the checked flag survives as a letter, not a key
+
+    def test_history_is_capped_at_the_last_few_actions(self) -> None:
+        compact = laya_module.laya_state(_BROWSER_STATE)
+        self.assertEqual(len(compact["recent_actions"]), laya_module.LAYA_BROWSER_HISTORY_KEPT)
+        self.assertEqual(compact["recent_actions"][-1], "click:step-9 (no change)")
+
+    def test_compaction_shrinks_the_json_size_by_an_order_of_magnitude(self) -> None:
+        import json
+
+        raw = json.dumps(_BROWSER_STATE)
+        compact = json.dumps(laya_module.laya_state(_BROWSER_STATE))
+        self.assertGreater(len(raw) / len(compact), 8)
+
+
+class TestLayaModelCompaction(IsolatedAsyncioTestCase):
+    async def test_the_browser_state_reaching_the_agent_is_compacted_by_default(self) -> None:
+        agent = FakeLayaAgent()
+        question = ChoiceQuestion({"1": {"element": "[1] Search"}})
+        await _model(agent).decide_many(Observation(_BROWSER_STATE), {"operation": question})
+        ((state, _asked),) = agent.calls
+        self.assertEqual(state, laya_module.laya_state(_BROWSER_STATE))
+        self.assertNotIn("text", state["page"])
+
+    async def test_compaction_turns_off_with_compact_browser_state_false(self) -> None:
+        agent = FakeLayaAgent()
+        question = ChoiceQuestion({"1": {"element": "[1] Search"}})
+        model = LayaModel(agent, model="convaiinnovations/laya", compact_browser_state=False)
+        await model.decide_many(Observation(_BROWSER_STATE), {"operation": question})
+        ((state, _asked),) = agent.calls
+        self.assertEqual(state, _BROWSER_STATE)
+
+
 class TestFailures(IsolatedAsyncioTestCase):
     async def test_option_overflow_and_torch_errors_are_model_call_failures(self) -> None:
         for error in (ValueError("question 'pick' options exceed head_max_len=192"), RuntimeError("CUDA error")):
@@ -204,3 +280,13 @@ def test_the_defaults_when_the_env_is_empty(self) -> None:
                 decision_model = LayaModel.from_env()
         self.assertEqual(decision_model.model, laya_module.LAYA_DEFAULT_MODEL)
         self.assertEqual(decision_model._agent.cfg, {"max_len": 512, "head_max_len": 192})
+        self.assertTrue(decision_model._compact_browser_state)
+
+    def test_laya_compact_browser_state_env_var_turns_compaction_off(self) -> None:
+        for off in ("0", "false", "False", "no"):
+            with self.subTest(off=off):
+                env = {"LAYA_COMPACT_BROWSER_STATE": off}
+                with patch.dict(sys.modules, {"laya": SimpleNamespace(load=lambda *a, **k: FakeLayaAgent())}):
+                    with patch.dict(os.environ, env):
+                        decision_model = LayaModel.from_env()
+                self.assertFalse(decision_model._compact_browser_state)