Validate robustness corpus release evidence

src/format_bench/release.py

@@ -65,6 +65,9 @@ def _artifact_references(manifest: dict, results: dict) -> list[tuple[str, bool]
             for record in case.get("artifact_records", []):
                 if isinstance(record, dict) and isinstance(record.get("path"), str):
                     references.append((record["path"], True))
+            for record in case.get("corpus_records", []):
+                if isinstance(record, dict) and isinstance(record.get("path"), str):
+                    references.append((record["path"], True))
     return references
 
 

src/format_bench/robustness/cases.py

@@ -74,7 +74,8 @@ def generated_cases(seed: int, count: int) -> tuple[CaseSpec, ...]:
             "rows": rng.randint(0, 2049),
             "string_seed": rng.getrandbits(32),
         }
-        parameters["cardinality"] = rng.randint(1, max(parameters["rows"], 1))
+        rows = parameters["rows"]
+        parameters["cardinality"] = 0 if rows == 0 else rng.randint(1, rows)
         encoded = json.dumps(parameters, sort_keys=True, separators=(",", ":")).encode()
         suffix = hashlib.sha256(encoded).hexdigest()[:10]
         cases.append(

src/format_bench/robustness/runner.py

@@ -113,7 +113,13 @@ def _worker_response(stdout: str) -> WorkerResponse:
 
 
 def _bounded_details(details: dict[str, object]) -> dict[str, object]:
-    encoded = (json.dumps(details, indent=2, sort_keys=True) + "\n").encode()
+    try:
+        encoded = (json.dumps(details, indent=2, sort_keys=True) + "\n").encode()
+    except (RecursionError, TypeError, ValueError) as error:
+        return {
+            "truncated": True,
+            "error_type": type(error).__name__,
+        }
     if len(encoded) <= MAX_WORKER_DETAILS_BYTES:
         return details
     return {
@@ -333,7 +339,7 @@ def _outcome(
     try:
         response = _worker_response(stdout)
         return ObservedOutcome(response["observed"]), response["details"]
-    except (json.JSONDecodeError, KeyError, ValueError, TypeError):
+    except (json.JSONDecodeError, KeyError, RecursionError, ValueError, TypeError):
         return ObservedOutcome.HARNESS_FAILED, {}
 
 

src/format_bench/robustness/worker.py

@@ -70,8 +70,14 @@ def run_request(request_path: Path) -> dict:
                     "observed": ObservedOutcome.UNSUPPORTED,
                     "details": {"error_type": type(error).__name__},
                 }
-            except Exception as error:
+            except ValueError as error:
                 observed = ObservedOutcome.VALUE_MISMATCH
+                details = {
+                    "error_type": type(error).__name__,
+                    "message": str(error)[-500:],
+                }
+            except Exception as error:
+                observed = ObservedOutcome.HARNESS_FAILED
                 details = {"error_type": type(error).__name__, "message": str(error)[-500:]}
         else:
             effective_manifest = json.loads(manifest.read_text(encoding="utf-8"))

tests/test_release.py

@@ -29,6 +29,7 @@ def _robustness_run(root: Path) -> Path:
         "artifact.csv",
         "stdout.txt",
         "stderr.txt",
+        "source.csv",
         "request.json",
         "result.json",
     ):
@@ -53,6 +54,11 @@ def _robustness_run(root: Path) -> Path:
                                 "path": "robustness/cases/csv/rows-1/artifact.csv"
                             }
                         ],
+                        "corpus_records": [
+                            {
+                                "path": "robustness/cases/csv/rows-1/source.csv"
+                            }
+                        ],
                     }
                 ]
             }
@@ -121,6 +127,22 @@ def test_release_package_is_deterministic_and_relative(
     assert all(not Path(name).is_absolute() for name in names)
 
 
+def test_release_rejects_missing_corpus_record(tmp_path: Path) -> None:
+    run = _robustness_run(tmp_path)
+    (run / "robustness/cases/csv/rows-1/source.csv").unlink()
+
+    with pytest.raises(FileNotFoundError, match="source.csv"):
+        package_run(run, tmp_path / "out", "linux-x86_64")
+
+
+def test_release_packages_corpus_record(tmp_path: Path) -> None:
+    run = _robustness_run(tmp_path)
+
+    archive = package_run(run, tmp_path / "out", "linux-x86_64")
+
+    assert "run-1/robustness/cases/csv/rows-1/source.csv" in _archive_names(archive)
+
+
 def test_release_rejects_missing_referenced_artifact(tmp_path: Path) -> None:
     run = tmp_path / "run"
     (run / "input").mkdir(parents=True)
@@ -210,6 +232,7 @@ def test_release_streams_all_robustness_evidence(
         "run-1/robustness/cases/csv/rows-1/input.arrow",
         "run-1/robustness/cases/csv/rows-1/request.json",
         "run-1/robustness/cases/csv/rows-1/result.json",
+        "run-1/robustness/cases/csv/rows-1/source.csv",
         "run-1/robustness/cases/csv/rows-1/stderr.txt",
         "run-1/robustness/cases/csv/rows-1/stdout.txt",
     ]

tests/test_robustness_cases.py

@@ -57,6 +57,8 @@ def test_generated_cases_preserve_requested_rows() -> None:
         assert table.num_rows == case.options["rows"]
         if case.options["rows"]:
             assert case.options["cardinality"] <= case.options["rows"]
+        else:
+            assert case.options["cardinality"] == 0
 
 
 def test_malformed_case_has_no_valid_arrow_materialization() -> None:

tests/test_robustness_runner.py

@@ -56,6 +56,14 @@ def test_runner_classifies_signal_and_timeout(tmp_path: Path) -> None:
     assert timed_out["process"]["signal"] is None
 
 
+def test_runner_classifies_timeout_after_bytes_output(tmp_path: Path) -> None:
+    code = "import os,time; os.write(1,b'worker-bytes\\n'); time.sleep(5)"
+    timed_out = _run(tmp_path, code)
+    assert timed_out["observed"] is ObservedOutcome.TIMED_OUT
+    assert timed_out["process"]["signal"] is None
+    assert (tmp_path / timed_out["stdout"]).read_bytes() == b"worker-bytes\n"
+
+
 def test_process_reaps_descendant_after_leader_exit(tmp_path: Path) -> None:
     pid_path = tmp_path / "descendant.pid"
     ready_path = tmp_path / "descendant.ready"
@@ -152,6 +160,22 @@ def test_runner_classifies_invalid_output_and_valid_roundtrip_failure(tmp_path:
     assert valid["verdict"] is RobustnessVerdict.FAIL
 
 
+def test_runner_bounds_deep_worker_details_without_escaping(tmp_path: Path) -> None:
+    code = (
+        "depth=1000; "
+        "payload='{" + '\"observed\":\"REJECTED\",\"details\":' + "' "
+        "+ '{\"nested\":' * depth + '0' + '}' * depth + '}'"
+        "; print(payload)"
+    )
+    result = _run(tmp_path, code)
+    assert result["observed"] is ObservedOutcome.REJECTED
+    assert result["verdict"] is RobustnessVerdict.PASS
+    assert result["details"] == {
+        "truncated": True,
+        "error_type": "RecursionError",
+    }
+
+
 def test_runner_reuses_unused_stream_budget(tmp_path: Path) -> None:
     code = "import json; print(json.dumps({'observed':'REJECTED','details':{'message':'x'*200}}))"
     result = _run(tmp_path, code, output_budget_bytes=512)

tests/test_robustness_targets.py

@@ -152,7 +152,7 @@ def test_worker_reports_raised_builtin_verification_as_value_mismatch(
     assert result["details"]["error_type"] == "ValueError"
 
 
-def test_worker_reports_non_value_verification_errors_as_value_mismatch(
+def test_worker_reports_non_value_verification_errors_as_harness_failure(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:
     class BrokenVerifier:
@@ -176,5 +176,5 @@ def verify_roundtrip(self, path, manifest):
 
     result = run_request(Path("request.json"))
 
-    assert result["observed"] is ObservedOutcome.VALUE_MISMATCH
+    assert result["observed"] is ObservedOutcome.HARNESS_FAILED
     assert result["details"]["error_type"] == "TypeError"