diff --git a/src/format_bench/model.py b/src/format_bench/model.py index 85231dd..a9bf983 100644 --- a/src/format_bench/model.py +++ b/src/format_bench/model.py @@ -98,10 +98,14 @@ def transition(current: ExecutionState, target: ExecutionState) -> ExecutionStat def robustness_verdict( - expectation: RobustnessExpectation, - observed: ObservedOutcome, - applicability: Applicability = Applicability.APPLICABLE, + expectation: RobustnessExpectation | str, + observed: ObservedOutcome | str, + applicability: Applicability | str = Applicability.APPLICABLE, ) -> RobustnessVerdict: + # Evidence is persisted as JSON strings and may be evaluated after reload. + expectation = RobustnessExpectation(expectation) + observed = ObservedOutcome(observed) + applicability = Applicability(applicability) if applicability is Applicability.NOT_APPLICABLE: return RobustnessVerdict.NOT_APPLICABLE if observed in { diff --git a/src/format_bench/report.py b/src/format_bench/report.py index f0127bf..a3b5d1d 100644 --- a/src/format_bench/report.py +++ b/src/format_bench/report.py @@ -121,13 +121,22 @@ def _claims(results: dict) -> list[str]: for name, item in sorted(results["results"].items()): if name == "negative_research": for research_name, record in sorted(item.items()): + summary = record.get("claim_summary") + if summary is None: + attempts = record.get("attempts", []) + summary = ( + attempts[-1].get("result") + if attempts and isinstance(attempts[-1], dict) + else None + ) + summary = summary or "no claim summary recorded" rows.append( [ research_name, "RESEARCH_RECORD", record["comparability"], record["state"], - record["claim_summary"], + summary, ] ) else: diff --git a/src/format_bench/robustness/cases.py b/src/format_bench/robustness/cases.py index 78f171c..725bed0 100644 --- a/src/format_bench/robustness/cases.py +++ b/src/format_bench/robustness/cases.py @@ -70,11 +70,11 @@ def generated_cases(seed: int, count: int) -> tuple[CaseSpec, ...]: cases = [] for index in range(count): parameters = { - "cardinality": rng.randint(1, 256), "null_stride": rng.randint(0, 17), "rows": rng.randint(0, 2049), "string_seed": rng.getrandbits(32), } + parameters["cardinality"] = rng.randint(1, max(parameters["rows"], 1)) encoded = json.dumps(parameters, sort_keys=True, separators=(",", ":")).encode() suffix = hashlib.sha256(encoded).hexdigest()[:10] cases.append( @@ -101,7 +101,7 @@ def materialize_case(base: pa.Table, case: CaseSpec) -> pa.Table: raise ValueError(f"case {case.case_id} has no valid Arrow input") options = case.options count = options.get("rows", options.get("cardinality", 17)) - table = _rows(base, max(count, options.get("cardinality", 0))) + table = _rows(base, count) if case.category in {"dictionary", "generated"}: cardinality = options["cardinality"] diff --git a/src/format_bench/robustness/evidence.py b/src/format_bench/robustness/evidence.py index 0420252..2695cf5 100644 --- a/src/format_bench/robustness/evidence.py +++ b/src/format_bench/robustness/evidence.py @@ -73,6 +73,8 @@ def import_path(self, source: Path, relative: str | Path) -> tuple[ArtifactRecor raise ValueError("artifact source must not contain symlinks") size = sum(item.stat().st_size for item in files) target = self._target(relative) + if source.is_dir() and target.is_relative_to(source): + raise ValueError("artifact destination must not be inside source directory") if target.exists(): raise FileExistsError(target) self._reserve(size) diff --git a/src/format_bench/robustness/mutations.py b/src/format_bench/robustness/mutations.py index 34ecc71..7ecaceb 100644 --- a/src/format_bench/robustness/mutations.py +++ b/src/format_bench/robustness/mutations.py @@ -29,7 +29,7 @@ def mutation_recipes(size: int, seed: int, count: int) -> tuple[MutationRecipe, operation = operations[index % len(operations)] parameters: dict[str, MutationValue] = {} if operation == "truncate": - parameters["offset"] = rng.randint(0, size) + parameters["offset"] = rng.randrange(size) if size else 0 elif operation.startswith("flip_"): region = operation.removeprefix("flip_") if region == "header": diff --git a/src/format_bench/robustness/profile.py b/src/format_bench/robustness/profile.py index 0d19fb8..edda478 100644 --- a/src/format_bench/robustness/profile.py +++ b/src/format_bench/robustness/profile.py @@ -279,6 +279,8 @@ def run_bounded( ) -> Path: if mutations_per_target < 0: raise ValueError("mutations_per_target must be non-negative") + root = root.resolve() + run_dir = run_dir.resolve() run, dataset = _load(run_dir) base = read_csv(run_dir / run["input"]["source"], dataset) cases = list(named_cases()) + list(generated_cases(seed, generated_count)) @@ -376,5 +378,6 @@ def run_bounded( "summary": summary, } } - # LLM contract: ROUNDTRIP_VERIFIED -> BENCHMARKED; report performs -> REPORTED. + # LLM contract: DISCOVERED -> ENCODED -> ROUNDTRIP_VERIFIED -> BENCHMARKED -> REPORTED. + # Robustness failures remain UNSUPPORTED or FAILED evidence and never rank. return _finish(root, run_dir, run, Lane.ROBUSTNESS, evidence) diff --git a/src/format_bench/robustness/worker.py b/src/format_bench/robustness/worker.py index b237f37..2b2d370 100644 --- a/src/format_bench/robustness/worker.py +++ b/src/format_bench/robustness/worker.py @@ -37,7 +37,25 @@ def run_request(request_path: Path) -> dict: try: if expectation is RobustnessExpectation.MUST_ROUNDTRIP: effective_manifest = json.loads(manifest.read_text(encoding="utf-8")) - adapter.read(artifact, effective_manifest) + try: + adapter.read(artifact, effective_manifest) + except ModuleNotFoundError as error: + return { + "schema_version": "1", + "case_id": case_id, + "observed": ObservedOutcome.UNSUPPORTED, + "details": {"error_type": type(error).__name__}, + } + except Exception as error: + return { + "schema_version": "1", + "case_id": case_id, + "observed": ObservedOutcome.REJECTED, + "details": { + "error_type": type(error).__name__, + "message": str(error)[-500:], + }, + } try: verification = adapter.verify_roundtrip(artifact, effective_manifest) observed = ( @@ -45,7 +63,14 @@ def run_request(request_path: Path) -> dict: if verification["passed"] else ObservedOutcome.VALUE_MISMATCH ) - except ValueError as error: + except ModuleNotFoundError as error: + return { + "schema_version": "1", + "case_id": case_id, + "observed": ObservedOutcome.UNSUPPORTED, + "details": {"error_type": type(error).__name__}, + } + except Exception as error: observed = ObservedOutcome.VALUE_MISMATCH details = {"error_type": type(error).__name__, "message": str(error)[-500:]} else: diff --git a/tests/test_model.py b/tests/test_model.py index 894108b..3cd1315 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -87,6 +87,11 @@ def test_not_applicable_takes_precedence_over_observation() -> None: ) is RobustnessVerdict.NOT_APPLICABLE +def test_robustness_verdict_normalizes_persisted_strings() -> None: + assert robustness_verdict("MUST_ROUNDTRIP", "REJECTED") is RobustnessVerdict.FAIL + assert robustness_verdict("MUST_REJECT", "REJECTED") is RobustnessVerdict.PASS + + def test_dataset_asset_path_stays_under_the_run_root() -> None: spec = DatasetSpec( schema_version="1", diff --git a/tests/test_report.py b/tests/test_report.py index 3a64920..79adfdd 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -161,6 +161,40 @@ def test_claim_report_preserves_terminal_observations(tmp_path: Path) -> None: assert "robustness crash" not in (tmp_path / "report.md").read_text() +def test_claim_report_falls_back_to_legacy_research_attempt(tmp_path: Path) -> None: + manifest = {"state": "BENCHMARKED", "dataset_id": "fixture", "formats": []} + results = { + "state": "BENCHMARKED", + "dataset_id": "fixture", + "run_id": "legacy-claims-fixture", + "profile": "claims", + "environment": { + "git_commit": "abc", + "flake_lock_sha256": "def", + "platform": "test-os", + "machine": "test-cpu", + "python": "3.12.0", + }, + "results": { + "negative_research": { + "legacy": { + "comparability": "PARTIAL", + "state": "FAILED", + "attempts": [{"result": "legacy result"}], + } + } + }, + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + (tmp_path / "results.json").write_text(json.dumps(results)) + + render_report(tmp_path) + + assert "| legacy | RESEARCH_RECORD | PARTIAL | FAILED | legacy result |" in ( + tmp_path / "report.md" + ).read_text() + + def test_robustness_report_separates_case_contract_and_is_deterministic( tmp_path: Path, ) -> None: diff --git a/tests/test_robustness_artifacts.py b/tests/test_robustness_artifacts.py index b390b1b..a67882c 100644 --- a/tests/test_robustness_artifacts.py +++ b/tests/test_robustness_artifacts.py @@ -56,6 +56,16 @@ def test_evidence_store_imports_file_and_directory(tmp_path: Path) -> None: assert store.used_bytes == 6 +def test_evidence_store_rejects_destination_inside_source(tmp_path: Path) -> None: + source = tmp_path / "evidence" + source.mkdir() + (source / "data.bin").write_bytes(b"data") + store = EvidenceStore(source, budget_bytes=100) + + with pytest.raises(ValueError, match="inside source"): + store.import_path(source, "copy") + + def test_evidence_store_rejects_unsafe_paths_and_symlinks(tmp_path: Path) -> None: store = EvidenceStore(tmp_path / "evidence", budget_bytes=100) with pytest.raises(ValueError, match="safe relative"): @@ -74,3 +84,11 @@ def test_invalid_mutation_arguments_and_operations_are_rejected() -> None: unknown = type(recipe)("bad", "unknown") with pytest.raises(ValueError, match="unknown mutation"): apply_mutation(b"x", unknown) + + +def test_non_empty_truncate_mutation_always_changes_size() -> None: + data = bytes(range(256)) + recipes = mutation_recipes(len(data), 983, 2) + truncate = next(recipe for recipe in recipes if recipe.operation == "truncate") + + assert len(apply_mutation(data, truncate)) < len(data) diff --git a/tests/test_robustness_cases.py b/tests/test_robustness_cases.py index 8870ffa..f3433de 100644 --- a/tests/test_robustness_cases.py +++ b/tests/test_robustness_cases.py @@ -50,6 +50,15 @@ def test_generated_cases_are_seeded_and_have_stable_semantics() -> None: assert first_hashes != [canonical_hash(materialize_case(base, case)) for case in other] +def test_generated_cases_preserve_requested_rows() -> None: + base = _base() + for case in generated_cases(20260703, 32): + table = materialize_case(base, case) + assert table.num_rows == case.options["rows"] + if case.options["rows"]: + assert case.options["cardinality"] <= case.options["rows"] + + def test_malformed_case_has_no_valid_arrow_materialization() -> None: case = next(case for case in named_cases() if case.case_id == "malformed-extra-column") with pytest.raises(ValueError, match="no valid Arrow input"): diff --git a/tests/test_robustness_profile.py b/tests/test_robustness_profile.py index 74a7c85..43be47c 100644 --- a/tests/test_robustness_profile.py +++ b/tests/test_robustness_profile.py @@ -104,6 +104,26 @@ def test_public_cli_runs_and_reports_bounded_fixture( assert (run_dir / "report.md").read_text() == first +def test_bounded_resolves_relative_run_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = Path(__file__).parents[1] + absolute_run = tmp_path / "relative-run" + adapter = CsvAdapter() + prepare_run(root, DATASET, absolute_run, fixture=True, selected=(adapter,)) + verify_run(absolute_run, {"csv": adapter}) + monkeypatch.chdir(tmp_path) + + result_path = run_bounded( + root, + Path("relative-run"), + generated_count=0, + mutations_per_target=0, + targets=(core_targets()[0],), + ) + + assert result_path == absolute_run / "results.json" + assert result_path.is_file() + + def test_mutation_uses_actual_member_size_and_rejects_links(tmp_path: Path) -> None: artifact = tmp_path / "artifact.bin" artifact.write_bytes(bytes(range(256))) diff --git a/tests/test_robustness_targets.py b/tests/test_robustness_targets.py index 90ca8a8..b2d8120 100644 --- a/tests/test_robustness_targets.py +++ b/tests/test_robustness_targets.py @@ -150,3 +150,31 @@ def test_worker_reports_raised_builtin_verification_as_value_mismatch( result = run_request(Path("request.json")) assert result["observed"] is ObservedOutcome.VALUE_MISMATCH assert result["details"]["error_type"] == "ValueError" + + +def test_worker_reports_non_value_verification_errors_as_value_mismatch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + class BrokenVerifier: + def read(self, path, manifest): + return None + + def verify_roundtrip(self, path, manifest): + raise TypeError("comparison unavailable") + + (tmp_path / "manifest.json").write_text("{}") + (tmp_path / "artifact.bin").write_bytes(b"data") + (tmp_path / "request.json").write_text(json.dumps({ + "case_id": "type-mismatch", + "target": "broken", + "manifest": "manifest.json", + "artifact": "artifact.bin", + "expectation": "MUST_ROUNDTRIP", + })) + monkeypatch.setattr(worker, "adapter_map", lambda: {"broken": BrokenVerifier()}) + monkeypatch.chdir(tmp_path) + + result = run_request(Path("request.json")) + + assert result["observed"] is ObservedOutcome.VALUE_MISMATCH + assert result["details"]["error_type"] == "TypeError"