fix(agents): replace Critic scoring with claim audit

Make Gemma reconstruct and adversarially challenge the complete response claim by claim instead of producing supportive numeric scores and blanket approval.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: 2 people

docs/ops/distributed-prefill-kv-network.md

@@ -349,6 +349,11 @@ and semantic fallback are forbidden. A global Critic score is valid only when
 `critic_omitted_tokens=0`. Long Prefill operations emit a heartbeat every 30
 seconds; on the 16GB allens worker, full-context Critic Prefill may take 15–25
 minutes.
+The Critic uses `adversarial_claim_audit_v1`: numeric scores and blanket
+approval are forbidden. It must reconstruct the thesis, quote and challenge
+each material claim, present the strongest objection, provide a corrected
+response, and state residual uncertainty. Epistemic honesty is evaluated
+separately from literal task completion.
 The worker reserves estimated final-snapshot capacity before model compute,
 prevents adaptive shrink from consuming active reservations, then atomically
 publishes and leases the final snapshot before adding optional intermediate

scripts/agent_gan_inference_demo.py

@@ -47,6 +47,7 @@ def build_critic_context(tokenizer, text: str) -> tuple[str, dict]:
         "critic_context_tokens": len(full_ids),
         "critic_omitted_tokens": 0,
         "review_scope": "full",
+        "critic_protocol": "adversarial_claim_audit_v1",
     }
 
 
@@ -178,14 +179,19 @@ def main() -> int:
     critic_history = [{
         "role": "system",
         "content": (
-            "You are the Critic/Discriminator agent. Attack the proposal, "
-            "identify false assumptions and bottlenecks, score it from 0 to "
-            "10, and demand specific corrections. Do not call a response "
-            "incomplete merely because it refuses to fabricate a solution to "
-            "an open problem. Claim truncation only when completion_status is "
-            "not EOS or the text is syntactically cut off."
-            " Review the complete Generator response as one semantic argument. "
-            "Do not sample, summarize, or infer claims from partial text."
+            "You are an adversarial peer reviewer, not a supportive grader. "
+            "Review the complete Generator response as one semantic argument. "
+            "Reconstruct its thesis and audit every material factual, logical, "
+            "and task-completion claim against the strongest counterargument. "
+            "Quote claims before challenging them. Distinguish an honest "
+            "boundary from literal task completion; unknown does not mean "
+            "impossible. Check numbers, quantifiers, awards, assumptions, and "
+            "claimed consequences. Never output a numeric score or blanket "
+            "approval without an explicit claim-by-claim audit. Return Thesis "
+            "Reconstruction, Claim-by-Claim Audit, Strongest Objection, "
+            "Corrected Response, and Residual Uncertainty. Claim truncation "
+            "only when completion_status is not EOS or syntax is cut off. "
+            "Do not sample, summarize, simplify, or use fallback review."
         ),
     }]
 

scripts/agent_gan_repl.py

@@ -52,7 +52,9 @@ def build_generator_messages(prompt: str) -> list[dict[str, str]]:
             "role": "system",
             "content": (
                 "Answer rigorously. For open problems, state the accepted "
-                "boundary and never fabricate a proof."
+                "boundary and never fabricate a proof. Distinguish unknown "
+                "from impossible; qualify awards, quantities, and claimed "
+                "downstream consequences precisely."
             ),
         },
         {"role": "user", "content": prompt},
@@ -70,10 +72,20 @@ def build_critic_messages(
         {
             "role": "system",
             "content": (
-                "Score the answer 0-10, identify false claims, and give "
-                "specific corrections. Do not penalize an honest statement "
-                "that an open problem is unsolved. Review the complete response "
-                "as one semantic argument; do not sample or summarize it."
+                "Act as an adversarial peer reviewer, not a supportive grader. "
+                "Read the complete response as one semantic argument. Rebuild "
+                "its thesis and audit every material factual, logical, and "
+                "task-completion claim against the strongest counterargument. "
+                "Quote the exact claim before challenging it. Distinguish "
+                "epistemic honesty from literal task completion: unknown or "
+                "unsolved does not mean impossible, and an honest refusal does "
+                "not complete a requested proof. Check numbers, awards, "
+                "quantifiers, and claimed consequences. Never output a numeric "
+                "score. Never issue blanket approval unless every material "
+                "claim has been explicitly audited. Use exactly these sections: "
+                "Thesis Reconstruction; Claim-by-Claim Audit; Strongest "
+                "Objection; Corrected Response; Residual Uncertainty. Do not "
+                "sample, summarize, simplify, or use a fallback review."
             ),
         },
         {

tests/inference_engine/bridge/test_agent_gan_demo.py

@@ -120,3 +120,4 @@ def test_critic_context_preserves_complete_generator_response():
     assert metrics["critic_context_tokens"] == 10
     assert metrics["critic_omitted_tokens"] == 0
     assert metrics["review_scope"] == "full"
+    assert metrics["critic_protocol"] == "adversarial_claim_audit_v1"

tests/inference_engine/bridge/test_agent_gan_repl.py

@@ -140,11 +140,13 @@ def test_stage_includes_full_context_metrics():
             "critic_context_tokens": 100,
             "critic_omitted_tokens": 0,
             "review_scope": "full",
+            "critic_protocol": "adversarial_claim_audit_v1",
         },
     )
     assert stage["critic_context_tokens"] == 100
     assert stage["critic_omitted_tokens"] == 0
     assert stage["review_scope"] == "full"
+    assert stage["critic_protocol"] == "adversarial_claim_audit_v1"
 
 
 def test_telemetry_timeout_warns_without_stopping_inference(
@@ -193,5 +195,8 @@ def test_interactive_prompts_are_deterministic_for_kv_reuse():
     combined = repr(generator_a + critic_a)
     assert "Internal run" not in combined
     assert "open problem" in combined
-    assert "Review the complete response" in combined
-    assert "do not sample or summarize" in combined
+    assert "adversarial peer reviewer" in combined
+    assert "Never output a numeric score" in combined
+    assert "Claim-by-Claim Audit" in combined
+    assert "unknown or unsolved does not mean impossible" in combined
+    assert "sample, summarize, simplify" in combined