Harden robustness evidence contracts

src/format_bench/model.py

@@ -98,10 +98,14 @@ def transition(current: ExecutionState, target: ExecutionState) -> ExecutionStat
 
 
 def robustness_verdict(
-    expectation: RobustnessExpectation,
-    observed: ObservedOutcome,
-    applicability: Applicability = Applicability.APPLICABLE,
+    expectation: RobustnessExpectation | str,
+    observed: ObservedOutcome | str,
+    applicability: Applicability | str = Applicability.APPLICABLE,
 ) -> RobustnessVerdict:
+    # Evidence is persisted as JSON strings and may be evaluated after reload.
+    expectation = RobustnessExpectation(expectation)
+    observed = ObservedOutcome(observed)
+    applicability = Applicability(applicability)
     if applicability is Applicability.NOT_APPLICABLE:
         return RobustnessVerdict.NOT_APPLICABLE
     if observed in {

src/format_bench/report.py

@@ -121,13 +121,22 @@ def _claims(results: dict) -> list[str]:
     for name, item in sorted(results["results"].items()):
         if name == "negative_research":
             for research_name, record in sorted(item.items()):
+                summary = record.get("claim_summary")
+                if summary is None:
+                    attempts = record.get("attempts", [])
+                    summary = (
+                        attempts[-1].get("result")
+                        if attempts and isinstance(attempts[-1], dict)
+                        else None
+                    )
+                summary = summary or "no claim summary recorded"
                 rows.append(
                     [
                         research_name,
                         "RESEARCH_RECORD",
                         record["comparability"],
                         record["state"],
-                        record["claim_summary"],
+                        summary,
                     ]
                 )
         else:

src/format_bench/robustness/cases.py

@@ -70,11 +70,11 @@ def generated_cases(seed: int, count: int) -> tuple[CaseSpec, ...]:
     cases = []
     for index in range(count):
         parameters = {
-            "cardinality": rng.randint(1, 256),
             "null_stride": rng.randint(0, 17),
             "rows": rng.randint(0, 2049),
             "string_seed": rng.getrandbits(32),
         }
+        parameters["cardinality"] = rng.randint(1, max(parameters["rows"], 1))
         encoded = json.dumps(parameters, sort_keys=True, separators=(",", ":")).encode()
         suffix = hashlib.sha256(encoded).hexdigest()[:10]
         cases.append(
@@ -101,7 +101,7 @@ def materialize_case(base: pa.Table, case: CaseSpec) -> pa.Table:
         raise ValueError(f"case {case.case_id} has no valid Arrow input")
     options = case.options
     count = options.get("rows", options.get("cardinality", 17))
-    table = _rows(base, max(count, options.get("cardinality", 0)))
+    table = _rows(base, count)
 
     if case.category in {"dictionary", "generated"}:
         cardinality = options["cardinality"]

src/format_bench/robustness/evidence.py

@@ -73,6 +73,8 @@ def import_path(self, source: Path, relative: str | Path) -> tuple[ArtifactRecor
             raise ValueError("artifact source must not contain symlinks")
         size = sum(item.stat().st_size for item in files)
         target = self._target(relative)
+        if source.is_dir() and target.is_relative_to(source):
+            raise ValueError("artifact destination must not be inside source directory")
         if target.exists():
             raise FileExistsError(target)
         self._reserve(size)

src/format_bench/robustness/mutations.py

@@ -29,7 +29,7 @@ def mutation_recipes(size: int, seed: int, count: int) -> tuple[MutationRecipe,
         operation = operations[index % len(operations)]
         parameters: dict[str, MutationValue] = {}
         if operation == "truncate":
-            parameters["offset"] = rng.randint(0, size)
+            parameters["offset"] = rng.randrange(size) if size else 0
         elif operation.startswith("flip_"):
             region = operation.removeprefix("flip_")
             if region == "header":

src/format_bench/robustness/profile.py

@@ -279,6 +279,8 @@ def run_bounded(
 ) -> Path:
     if mutations_per_target < 0:
         raise ValueError("mutations_per_target must be non-negative")
+    root = root.resolve()
+    run_dir = run_dir.resolve()
     run, dataset = _load(run_dir)
     base = read_csv(run_dir / run["input"]["source"], dataset)
     cases = list(named_cases()) + list(generated_cases(seed, generated_count))
@@ -376,5 +378,6 @@ def run_bounded(
             "summary": summary,
         }
     }
-    # LLM contract: ROUNDTRIP_VERIFIED -> BENCHMARKED; report performs -> REPORTED.
+    # LLM contract: DISCOVERED -> ENCODED -> ROUNDTRIP_VERIFIED -> BENCHMARKED -> REPORTED.
+    # Robustness failures remain UNSUPPORTED or FAILED evidence and never rank.
     return _finish(root, run_dir, run, Lane.ROBUSTNESS, evidence)

src/format_bench/robustness/worker.py

@@ -37,15 +37,40 @@ def run_request(request_path: Path) -> dict:
     try:
         if expectation is RobustnessExpectation.MUST_ROUNDTRIP:
             effective_manifest = json.loads(manifest.read_text(encoding="utf-8"))
-            adapter.read(artifact, effective_manifest)
+            try:
+                adapter.read(artifact, effective_manifest)
+            except ModuleNotFoundError as error:
+                return {
+                    "schema_version": "1",
+                    "case_id": case_id,
+                    "observed": ObservedOutcome.UNSUPPORTED,
+                    "details": {"error_type": type(error).__name__},
+                }
+            except Exception as error:
+                return {
+                    "schema_version": "1",
+                    "case_id": case_id,
+                    "observed": ObservedOutcome.REJECTED,
+                    "details": {
+                        "error_type": type(error).__name__,
+                        "message": str(error)[-500:],
+                    },
+                }
             try:
                 verification = adapter.verify_roundtrip(artifact, effective_manifest)
                 observed = (
                     ObservedOutcome.ROUNDTRIP_EQUAL
                     if verification["passed"]
                     else ObservedOutcome.VALUE_MISMATCH
                 )
-            except ValueError as error:
+            except ModuleNotFoundError as error:
+                return {
+                    "schema_version": "1",
+                    "case_id": case_id,
+                    "observed": ObservedOutcome.UNSUPPORTED,
+                    "details": {"error_type": type(error).__name__},
+                }
+            except Exception as error:
                 observed = ObservedOutcome.VALUE_MISMATCH
                 details = {"error_type": type(error).__name__, "message": str(error)[-500:]}
         else:

tests/test_model.py

@@ -87,6 +87,11 @@ def test_not_applicable_takes_precedence_over_observation() -> None:
     ) is RobustnessVerdict.NOT_APPLICABLE
 
 
+def test_robustness_verdict_normalizes_persisted_strings() -> None:
+    assert robustness_verdict("MUST_ROUNDTRIP", "REJECTED") is RobustnessVerdict.FAIL
+    assert robustness_verdict("MUST_REJECT", "REJECTED") is RobustnessVerdict.PASS
+
+
 def test_dataset_asset_path_stays_under_the_run_root() -> None:
     spec = DatasetSpec(
         schema_version="1",

tests/test_report.py

@@ -161,6 +161,40 @@ def test_claim_report_preserves_terminal_observations(tmp_path: Path) -> None:
     assert "robustness crash" not in (tmp_path / "report.md").read_text()
 
 
+def test_claim_report_falls_back_to_legacy_research_attempt(tmp_path: Path) -> None:
+    manifest = {"state": "BENCHMARKED", "dataset_id": "fixture", "formats": []}
+    results = {
+        "state": "BENCHMARKED",
+        "dataset_id": "fixture",
+        "run_id": "legacy-claims-fixture",
+        "profile": "claims",
+        "environment": {
+            "git_commit": "abc",
+            "flake_lock_sha256": "def",
+            "platform": "test-os",
+            "machine": "test-cpu",
+            "python": "3.12.0",
+        },
+        "results": {
+            "negative_research": {
+                "legacy": {
+                    "comparability": "PARTIAL",
+                    "state": "FAILED",
+                    "attempts": [{"result": "legacy result"}],
+                }
+            }
+        },
+    }
+    (tmp_path / "manifest.json").write_text(json.dumps(manifest))
+    (tmp_path / "results.json").write_text(json.dumps(results))
+
+    render_report(tmp_path)
+
+    assert "| legacy | RESEARCH_RECORD | PARTIAL | FAILED | legacy result |" in (
+        tmp_path / "report.md"
+    ).read_text()
+
+
 def test_robustness_report_separates_case_contract_and_is_deterministic(
     tmp_path: Path,
 ) -> None:

tests/test_robustness_artifacts.py

@@ -56,6 +56,16 @@ def test_evidence_store_imports_file_and_directory(tmp_path: Path) -> None:
     assert store.used_bytes == 6
 
 
+def test_evidence_store_rejects_destination_inside_source(tmp_path: Path) -> None:
+    source = tmp_path / "evidence"
+    source.mkdir()
+    (source / "data.bin").write_bytes(b"data")
+    store = EvidenceStore(source, budget_bytes=100)
+
+    with pytest.raises(ValueError, match="inside source"):
+        store.import_path(source, "copy")
+
+
 def test_evidence_store_rejects_unsafe_paths_and_symlinks(tmp_path: Path) -> None:
     store = EvidenceStore(tmp_path / "evidence", budget_bytes=100)
     with pytest.raises(ValueError, match="safe relative"):
@@ -74,3 +84,11 @@ def test_invalid_mutation_arguments_and_operations_are_rejected() -> None:
     unknown = type(recipe)("bad", "unknown")
     with pytest.raises(ValueError, match="unknown mutation"):
         apply_mutation(b"x", unknown)
+
+
+def test_non_empty_truncate_mutation_always_changes_size() -> None:
+    data = bytes(range(256))
+    recipes = mutation_recipes(len(data), 983, 2)
+    truncate = next(recipe for recipe in recipes if recipe.operation == "truncate")
+
+    assert len(apply_mutation(data, truncate)) < len(data)

tests/test_robustness_cases.py

@@ -50,6 +50,15 @@ def test_generated_cases_are_seeded_and_have_stable_semantics() -> None:
     assert first_hashes != [canonical_hash(materialize_case(base, case)) for case in other]
 
 
+def test_generated_cases_preserve_requested_rows() -> None:
+    base = _base()
+    for case in generated_cases(20260703, 32):
+        table = materialize_case(base, case)
+        assert table.num_rows == case.options["rows"]
+        if case.options["rows"]:
+            assert case.options["cardinality"] <= case.options["rows"]
+
+
 def test_malformed_case_has_no_valid_arrow_materialization() -> None:
     case = next(case for case in named_cases() if case.case_id == "malformed-extra-column")
     with pytest.raises(ValueError, match="no valid Arrow input"):

tests/test_robustness_profile.py

@@ -104,6 +104,26 @@ def test_public_cli_runs_and_reports_bounded_fixture(
     assert (run_dir / "report.md").read_text() == first
 
 
+def test_bounded_resolves_relative_run_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    root = Path(__file__).parents[1]
+    absolute_run = tmp_path / "relative-run"
+    adapter = CsvAdapter()
+    prepare_run(root, DATASET, absolute_run, fixture=True, selected=(adapter,))
+    verify_run(absolute_run, {"csv": adapter})
+    monkeypatch.chdir(tmp_path)
+
+    result_path = run_bounded(
+        root,
+        Path("relative-run"),
+        generated_count=0,
+        mutations_per_target=0,
+        targets=(core_targets()[0],),
+    )
+
+    assert result_path == absolute_run / "results.json"
+    assert result_path.is_file()
+
+
 def test_mutation_uses_actual_member_size_and_rejects_links(tmp_path: Path) -> None:
     artifact = tmp_path / "artifact.bin"
     artifact.write_bytes(bytes(range(256)))

tests/test_robustness_targets.py

@@ -150,3 +150,31 @@ def test_worker_reports_raised_builtin_verification_as_value_mismatch(
     result = run_request(Path("request.json"))
     assert result["observed"] is ObservedOutcome.VALUE_MISMATCH
     assert result["details"]["error_type"] == "ValueError"
+
+
+def test_worker_reports_non_value_verification_errors_as_value_mismatch(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    class BrokenVerifier:
+        def read(self, path, manifest):
+            return None
+
+        def verify_roundtrip(self, path, manifest):
+            raise TypeError("comparison unavailable")
+
+    (tmp_path / "manifest.json").write_text("{}")
+    (tmp_path / "artifact.bin").write_bytes(b"data")
+    (tmp_path / "request.json").write_text(json.dumps({
+        "case_id": "type-mismatch",
+        "target": "broken",
+        "manifest": "manifest.json",
+        "artifact": "artifact.bin",
+        "expectation": "MUST_ROUNDTRIP",
+    }))
+    monkeypatch.setattr(worker, "adapter_map", lambda: {"broken": BrokenVerifier()})
+    monkeypatch.chdir(tmp_path)
+
+    result = run_request(Path("request.json"))
+
+    assert result["observed"] is ObservedOutcome.VALUE_MISMATCH
+    assert result["details"]["error_type"] == "TypeError"