From e365ed9b7b08acb60a81d3e65ee16949df519f2e Mon Sep 17 00:00:00 2001 From: Anionix Date: Fri, 17 Jul 2026 03:45:06 +0900 Subject: [PATCH] Validate robustness corpus release evidence --- src/format_bench/release.py | 3 +++ src/format_bench/robustness/cases.py | 3 ++- src/format_bench/robustness/runner.py | 10 ++++++++-- src/format_bench/robustness/worker.py | 8 +++++++- tests/test_release.py | 23 +++++++++++++++++++++++ tests/test_robustness_cases.py | 2 ++ tests/test_robustness_runner.py | 24 ++++++++++++++++++++++++ tests/test_robustness_targets.py | 4 ++-- 8 files changed, 71 insertions(+), 6 deletions(-) diff --git a/src/format_bench/release.py b/src/format_bench/release.py index 311c183..85c7cad 100644 --- a/src/format_bench/release.py +++ b/src/format_bench/release.py @@ -65,6 +65,9 @@ def _artifact_references(manifest: dict, results: dict) -> list[tuple[str, bool] for record in case.get("artifact_records", []): if isinstance(record, dict) and isinstance(record.get("path"), str): references.append((record["path"], True)) + for record in case.get("corpus_records", []): + if isinstance(record, dict) and isinstance(record.get("path"), str): + references.append((record["path"], True)) return references diff --git a/src/format_bench/robustness/cases.py b/src/format_bench/robustness/cases.py index 725bed0..c11f37a 100644 --- a/src/format_bench/robustness/cases.py +++ b/src/format_bench/robustness/cases.py @@ -74,7 +74,8 @@ def generated_cases(seed: int, count: int) -> tuple[CaseSpec, ...]: "rows": rng.randint(0, 2049), "string_seed": rng.getrandbits(32), } - parameters["cardinality"] = rng.randint(1, max(parameters["rows"], 1)) + rows = parameters["rows"] + parameters["cardinality"] = 0 if rows == 0 else rng.randint(1, rows) encoded = json.dumps(parameters, sort_keys=True, separators=(",", ":")).encode() suffix = hashlib.sha256(encoded).hexdigest()[:10] cases.append( diff --git a/src/format_bench/robustness/runner.py b/src/format_bench/robustness/runner.py index 09b8bd0..50dc14c 100644 --- a/src/format_bench/robustness/runner.py +++ b/src/format_bench/robustness/runner.py @@ -113,7 +113,13 @@ def _worker_response(stdout: str) -> WorkerResponse: def _bounded_details(details: dict[str, object]) -> dict[str, object]: - encoded = (json.dumps(details, indent=2, sort_keys=True) + "\n").encode() + try: + encoded = (json.dumps(details, indent=2, sort_keys=True) + "\n").encode() + except (RecursionError, TypeError, ValueError) as error: + return { + "truncated": True, + "error_type": type(error).__name__, + } if len(encoded) <= MAX_WORKER_DETAILS_BYTES: return details return { @@ -333,7 +339,7 @@ def _outcome( try: response = _worker_response(stdout) return ObservedOutcome(response["observed"]), response["details"] - except (json.JSONDecodeError, KeyError, ValueError, TypeError): + except (json.JSONDecodeError, KeyError, RecursionError, ValueError, TypeError): return ObservedOutcome.HARNESS_FAILED, {} diff --git a/src/format_bench/robustness/worker.py b/src/format_bench/robustness/worker.py index 2b2d370..50332bf 100644 --- a/src/format_bench/robustness/worker.py +++ b/src/format_bench/robustness/worker.py @@ -70,8 +70,14 @@ def run_request(request_path: Path) -> dict: "observed": ObservedOutcome.UNSUPPORTED, "details": {"error_type": type(error).__name__}, } - except Exception as error: + except ValueError as error: observed = ObservedOutcome.VALUE_MISMATCH + details = { + "error_type": type(error).__name__, + "message": str(error)[-500:], + } + except Exception as error: + observed = ObservedOutcome.HARNESS_FAILED details = {"error_type": type(error).__name__, "message": str(error)[-500:]} else: effective_manifest = json.loads(manifest.read_text(encoding="utf-8")) diff --git a/tests/test_release.py b/tests/test_release.py index cc18b03..1cb27b6 100644 --- a/tests/test_release.py +++ b/tests/test_release.py @@ -29,6 +29,7 @@ def _robustness_run(root: Path) -> Path: "artifact.csv", "stdout.txt", "stderr.txt", + "source.csv", "request.json", "result.json", ): @@ -53,6 +54,11 @@ def _robustness_run(root: Path) -> Path: "path": "robustness/cases/csv/rows-1/artifact.csv" } ], + "corpus_records": [ + { + "path": "robustness/cases/csv/rows-1/source.csv" + } + ], } ] } @@ -121,6 +127,22 @@ def test_release_package_is_deterministic_and_relative( assert all(not Path(name).is_absolute() for name in names) +def test_release_rejects_missing_corpus_record(tmp_path: Path) -> None: + run = _robustness_run(tmp_path) + (run / "robustness/cases/csv/rows-1/source.csv").unlink() + + with pytest.raises(FileNotFoundError, match="source.csv"): + package_run(run, tmp_path / "out", "linux-x86_64") + + +def test_release_packages_corpus_record(tmp_path: Path) -> None: + run = _robustness_run(tmp_path) + + archive = package_run(run, tmp_path / "out", "linux-x86_64") + + assert "run-1/robustness/cases/csv/rows-1/source.csv" in _archive_names(archive) + + def test_release_rejects_missing_referenced_artifact(tmp_path: Path) -> None: run = tmp_path / "run" (run / "input").mkdir(parents=True) @@ -210,6 +232,7 @@ def test_release_streams_all_robustness_evidence( "run-1/robustness/cases/csv/rows-1/input.arrow", "run-1/robustness/cases/csv/rows-1/request.json", "run-1/robustness/cases/csv/rows-1/result.json", + "run-1/robustness/cases/csv/rows-1/source.csv", "run-1/robustness/cases/csv/rows-1/stderr.txt", "run-1/robustness/cases/csv/rows-1/stdout.txt", ] diff --git a/tests/test_robustness_cases.py b/tests/test_robustness_cases.py index f3433de..70a9bb2 100644 --- a/tests/test_robustness_cases.py +++ b/tests/test_robustness_cases.py @@ -57,6 +57,8 @@ def test_generated_cases_preserve_requested_rows() -> None: assert table.num_rows == case.options["rows"] if case.options["rows"]: assert case.options["cardinality"] <= case.options["rows"] + else: + assert case.options["cardinality"] == 0 def test_malformed_case_has_no_valid_arrow_materialization() -> None: diff --git a/tests/test_robustness_runner.py b/tests/test_robustness_runner.py index 5f4dd02..d373293 100644 --- a/tests/test_robustness_runner.py +++ b/tests/test_robustness_runner.py @@ -56,6 +56,14 @@ def test_runner_classifies_signal_and_timeout(tmp_path: Path) -> None: assert timed_out["process"]["signal"] is None +def test_runner_classifies_timeout_after_bytes_output(tmp_path: Path) -> None: + code = "import os,time; os.write(1,b'worker-bytes\\n'); time.sleep(5)" + timed_out = _run(tmp_path, code) + assert timed_out["observed"] is ObservedOutcome.TIMED_OUT + assert timed_out["process"]["signal"] is None + assert (tmp_path / timed_out["stdout"]).read_bytes() == b"worker-bytes\n" + + def test_process_reaps_descendant_after_leader_exit(tmp_path: Path) -> None: pid_path = tmp_path / "descendant.pid" ready_path = tmp_path / "descendant.ready" @@ -152,6 +160,22 @@ def test_runner_classifies_invalid_output_and_valid_roundtrip_failure(tmp_path: assert valid["verdict"] is RobustnessVerdict.FAIL +def test_runner_bounds_deep_worker_details_without_escaping(tmp_path: Path) -> None: + code = ( + "depth=1000; " + "payload='{" + '\"observed\":\"REJECTED\",\"details\":' + "' " + "+ '{\"nested\":' * depth + '0' + '}' * depth + '}'" + "; print(payload)" + ) + result = _run(tmp_path, code) + assert result["observed"] is ObservedOutcome.REJECTED + assert result["verdict"] is RobustnessVerdict.PASS + assert result["details"] == { + "truncated": True, + "error_type": "RecursionError", + } + + def test_runner_reuses_unused_stream_budget(tmp_path: Path) -> None: code = "import json; print(json.dumps({'observed':'REJECTED','details':{'message':'x'*200}}))" result = _run(tmp_path, code, output_budget_bytes=512) diff --git a/tests/test_robustness_targets.py b/tests/test_robustness_targets.py index b2d8120..3baebba 100644 --- a/tests/test_robustness_targets.py +++ b/tests/test_robustness_targets.py @@ -152,7 +152,7 @@ def test_worker_reports_raised_builtin_verification_as_value_mismatch( assert result["details"]["error_type"] == "ValueError" -def test_worker_reports_non_value_verification_errors_as_value_mismatch( +def test_worker_reports_non_value_verification_errors_as_harness_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: class BrokenVerifier: @@ -176,5 +176,5 @@ def verify_roundtrip(self, path, manifest): result = run_request(Path("request.json")) - assert result["observed"] is ObservedOutcome.VALUE_MISMATCH + assert result["observed"] is ObservedOutcome.HARNESS_FAILED assert result["details"]["error_type"] == "TypeError"