fix(autoresearch): deduplicate strategy state losslessly

Intern repeated branch statements and evidence once so the complete active research state fits its hard Prefill budget without truncation or summarization.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: 2 people

autoresearch/prefill/program.md

@@ -61,6 +61,8 @@ Prefill budgets are hard admission limits, never truncation instructions.
 Strategy input must fit 8192 tokens by carrying the complete active leaf
 ancestry and its exact experiment records. Generator and Critic inputs must fit
 6144 tokens; the Critic always receives the complete current Generator output.
+Repeated Strategy strings are interned once in `text_by_id`; `_ref` fields
+losslessly reference that exact text.
 If any complete semantic unit exceeds its budget, reject it before remote
 Prefill and preserve the checkpoint. Never slice, sample, summarize, or drop
 the tail of an over-budget input.

autoresearch/prefill/supervisor.py

@@ -424,6 +424,21 @@ def build_strategy_research_state(
     ledger: dict,
     results_text: str,
 ) -> dict:
+    text_by_id: dict[str, str] = {}
+    id_by_text: dict[str, str] = {}
+
+    def intern(value) -> str:
+        text = str(value or "")
+        if not text:
+            return ""
+        existing = id_by_text.get(text)
+        if existing is not None:
+            return existing
+        text_id = f"t{len(text_by_id) + 1}"
+        id_by_text[text] = text_id
+        text_by_id[text_id] = text
+        return text_id
+
     target_id = _select_repair_target(current, ledger)
     obligations = {
         str(item.get("obligation_id", "")): item
@@ -437,11 +452,11 @@ def build_strategy_research_state(
         item = obligations[cursor]
         ancestry.append({
             "obligation_id": cursor,
-            "statement": item.get("statement", ""),
+            "statement_ref": intern(item.get("statement", "")),
             "status": item.get("status", ""),
             "parent_id": item.get("parent_id", ""),
             "last_run_id": item.get("last_run_id", ""),
-            "last_evidence": item.get("last_evidence", ""),
+            "last_evidence_ref": intern(item.get("last_evidence", "")),
         })
         cursor = str(item.get("parent_id", ""))
     ancestry.reverse()
@@ -464,12 +479,14 @@ def build_strategy_research_state(
                 ),
                 "hypothesis_sha256": row.get("hypothesis_sha256", ""),
                 "research_outcome": row.get("research_outcome", ""),
-                "research_evidence": row.get("research_evidence", ""),
-                "new_frontier": row.get("new_frontier", ""),
+                "research_evidence_ref": intern(
+                    row.get("research_evidence", ""),
+                ),
+                "new_frontier_ref": intern(row.get("new_frontier", "")),
                 "kept": row.get("kept", ""),
                 "error": row.get("error", ""),
             })
-    return {
+    state = {
         "target_leaf_id": target_id,
         "target_ancestry": ancestry,
         "relevant_experiments": relevant_results,
@@ -479,12 +496,14 @@ def build_strategy_research_state(
                 "target_obligation_id",
                 "",
             ),
-            "hypothesis": current.get("hypothesis", ""),
+            "hypothesis_ref": intern(current.get("hypothesis", "")),
             "prefill_compute_chunk_tokens": current.get(
                 "prefill_compute_chunk_tokens",
             ),
         },
     }
+    state["text_by_id"] = text_by_id
+    return state
 
 
 def build_strategy_prompt(
@@ -511,7 +530,9 @@ def build_strategy_prompt(
         "It must either construct a concrete object or attempt a concrete "
         "counterexample for the target leaf. target_obligation_id must equal "
         "TARGET_LEAF_ID. Every statement and evidence item below is complete; "
-        "do not infer omitted text from unrelated branches."
+        "do not infer omitted text from unrelated branches. Fields ending in "
+        "_ref resolve through text_by_id; this is lossless deduplication, not "
+        "summary or truncation."
         f"\n\nPROGRAM:\n{program}"
         "\n\nRESEARCH_STATE:\n"
         f"{json.dumps(research_state, ensure_ascii=False)}"

tests/inference_engine/bench/test_autoresearch_supervisor.py

@@ -327,6 +327,40 @@ def test_strategy_state_keeps_complete_active_ancestry_only():
     assert "unrelated result" not in serialized
 
 
+def test_strategy_state_deduplicates_text_losslessly():
+    shared = "Complete exact evidence that must appear once without truncation."
+    ledger = {"obligations": [
+        {
+            "obligation_id": "RH-C2",
+            "statement": "Root statement.",
+            "status": "UNRESOLVED",
+            "parent_id": "",
+            "last_evidence": shared,
+        },
+    ]}
+    results = (
+        "candidate_id\ttarget_obligation_id\tresearch_outcome\t"
+        "research_evidence\tnew_frontier\tkept\terror\t"
+        "hypothesis_sha256\n"
+        f"c2\tRH-C2\tINCONCLUSIVE\t{shared}\tRoot statement."
+        "\tFalse\t\th2\n"
+    )
+    state = build_strategy_research_state(
+        current={**_candidate(), "target_obligation_id": "RH-C2"},
+        ledger=ledger,
+        results_text=results,
+    )
+    serialized = str(state)
+    assert serialized.count(shared) == 1
+    assert serialized.count("Root statement.") == 1
+    evidence_ref = state["target_ancestry"][0]["last_evidence_ref"]
+    result_ref = state["relevant_experiments"][0][
+        "research_evidence_ref"
+    ]
+    assert evidence_ref == result_ref
+    assert state["text_by_id"][evidence_ref] == shared
+
+
 def test_results_are_append_only_and_best_is_selected(tmp_path):
     path = tmp_path / "results.tsv"
     common = {