From 495af4a3645b65cc90133ec43ca8b392604dfac1 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Thu, 8 Oct 2026 11:32:03 +0800 Subject: [PATCH] [Feat] Collect all seven Intern-Decision benchmark suites --- CHANGELOG.md | 3 + benchmarks/INTERN_DECISION.md | 37 ++ benchmarks/intern_decision.py | 130 +++++- benchmarks/results/intern-accuracy-v1-h800.md | 50 +++ tests/benchmarks/test_intern_decision.py | 371 ++++++++++++++++++ 5 files changed, 589 insertions(+), 2 deletions(-) create mode 100644 benchmarks/results/intern-accuracy-v1-h800.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7071118..2111e8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ### Added +- Intern-Decision seven-suite collection with raw HTTP evidence and H800 results + for the 0.8B, 2B and 4B checkpoints. + - Intern-Decision input verification and offline replay with the official evaluator and explicit coverage for incomplete runs. diff --git a/benchmarks/INTERN_DECISION.md b/benchmarks/INTERN_DECISION.md index 8bcc0c6..673213d 100644 --- a/benchmarks/INTERN_DECISION.md +++ b/benchmarks/INTERN_DECISION.md @@ -57,6 +57,40 @@ parse and refusal failures remain failed attempts. Scored ledger columns are ign by hash. Retain the original run because raw files are referenced, not copied. Recorded configuration is provenance, not proof of the loaded weights. +## Collect seven suites + +Use an already running authorized endpoint and an environment JSON containing +`checkpoint.repo`/`revision`, `backend.name`/`version`, positive +`sampling.temperature`, and `hardware`. Include actual dtype, libraries, context +limits and JobID when applicable. The original file is saved unchanged and hashed. +It must not contain collector-owned `endpoint`, `path`, `model_configured`, +`env_sha256`, `dataset`, `plan`, `stop_reason`, `attempted` or `collected` fields. + +```bash +python -m benchmarks.intern_decision collect \ + --intern-root /path/to/Intern-Decision --scorer-root /path/to/jevbench \ + --endpoint http://127.0.0.1:8000 --path /v1/decisions \ + --run-meta /path/to/actual-environment.json --out /path/to/fresh-seven-suite-run +python -m benchmarks.intern_decision replay \ + --intern-root /path/to/Intern-Decision --scorer-root /path/to/jevbench \ + --round /path/to/fresh-seven-suite-run/round-1 --out /path/to/fresh-seven-suite-replay +``` + +The shared client sends five separate warmups, then one serial accuracy round. +All fields of a Typed Decisions row stay in one POST. Only state and questions +(`type`, `instructions`, `criteria`) are model evidence; targets and IDs are omitted. +Use `--model` only if the endpoint requires it; `--timeout` defaults to 120 seconds. +`--path /v1/systemone` is also supported. No API key, automatic retry or resume is +provided. This is separate from the public231 three-round latency matrix. + +Every attempt saves original raw evidence before its ledger record. HTTP +401/403/429, three consecutive infrastructure failures, or an interrupt stop +collection; 422 remains failed and resets the consecutive-failure counter. +`run.meta.json` retains `stop_reason` and `attempted`. `collected=true` means every +request was attempted, including failed answers; it does not mean scoring passed. +A stop during warmup leaves all measured requests unattempted. Controlled early +stops exit 2; unexpected exceptions exit nonzero; full collection exits 0. + ## Scoring and completeness The unmodified upstream `src.eval.jev.evaluate` and table renderer perform scoring. @@ -79,3 +113,6 @@ different input. This entry calls official `evaluate` directly because upstream' main/merge CLI requires those 10 references even for the bundled input. The table's Brier/ECE columns refer to JevBench hard. A complete replay establishes scoring coverage, not reproduction of published model quality or inference latency. + +[H800 results](results/intern-accuracy-v1-h800.md) report one complete run for the +0.8B, 2B and 4B HF checkpoints. diff --git a/benchmarks/intern_decision.py b/benchmarks/intern_decision.py index fc6bae1..b6bd8e0 100644 --- a/benchmarks/intern_decision.py +++ b/benchmarks/intern_decision.py @@ -1,8 +1,9 @@ -"""Verify and replay pinned Intern-Decision suites with the official evaluator.""" +"""Verify, collect and replay pinned Intern-Decision suites with the official evaluator.""" from __future__ import annotations import argparse +from contextlib import closing import hashlib import importlib import importlib.util @@ -227,6 +228,117 @@ def load_round(intern_root, scorer_root, round_dir): return responses, provenance, config +def collect(args, *, transport=None): + """Collect one complete accuracy-v1 round, with five separate warmup requests.""" + from benchmarks.client import DecisionClient, decode + from benchmarks.evidence import dump, outside_repo, write_json + from benchmarks.legs.leg1_public231 import validate_environment + + verified = verify_sources(args.intern_root, args.scorer_root) + environment = args.run_meta.read_bytes() + meta = decode(environment) + validate_environment(meta) + controlled = { + "endpoint", + "path", + "model_configured", + "env_sha256", + "dataset", + "plan", + "stop_reason", + "attempted", + "collected", + } + if controlled.intersection(meta): + raise ValueError(f"run-meta contains collector-controlled fields: {sorted(controlled.intersection(meta))}") + bundle = Path(verified["intern_root"]) / "benchmarks/accuracy-v1" + items = [ + (suite, line, row) for suite in SUITES for line, row in _rows(bundle / verified["datasets"][suite]["path"]) + ] + output = outside_repo(args.out) + meta.update( + endpoint=args.endpoint, + path=args.path, + model_configured=args.model, + env_sha256=hashlib.sha256(environment).hexdigest(), + dataset=dict( + intern_revision=INTERN_REVISION, + jevbench_pin=SCORER_REVISION, + hashes=verified["files"], + dataset_hash=verified["manifest_sha256"], + ), + plan=dict(datasets=verified["datasets"], warmup=5, rounds=1), + stop_reason=None, + ) + with closing( + DecisionClient(args.endpoint, path=args.path, model=args.model, timeout_s=args.timeout, transport=transport) + ) as client: + output.mkdir(parents=True, exist_ok=False) + (output / "environment.json").write_bytes(environment) + write_json(output / "run.meta.json", meta) + stop, errors, attempted = None, 0, 0 + try: + for number, selected in ((0, items[:5]), (1, items)): + directory = output / ("warmup" if number == 0 else "round-1") + directory.mkdir() + with (directory / ("records.jsonl" if number == 0 else "results.jsonl")).open( + "x", encoding="utf-8" + ) as stream: + for suite, line, row in selected: + body = _body(row) + raw = client.request(body) + path = _raw_path(directory, (suite, line, row["id"]), True) + path.parent.mkdir(parents=True, exist_ok=True) + write_json(path, raw) + code, response = raw["http_status"], raw["response"] + answers = response.get("answers") if isinstance(response, dict) else None + parsed = isinstance(answers, dict) and all( + isinstance(answers.get(field), dict) and answers[field].get("type") == question["type"] + for field, question in body["questions"].items() + ) + status = "ok" if parsed else "parse_error" + if raw["client_error"] is not None: + status = "client_error" + elif code is None or not 200 <= code < 300: + status = "refused" if code == 422 else "http_error" + record = dict( + suite=suite, + source_line=line, + task_id=row["id"], + source_file=verified["datasets"][suite]["path"], + round=number, + kind="warmup" if number == 0 else "measured", + attempted=True, + ok=status == "ok", + status=status, + status_code=code, + error=raw["client_error"] or (status if status != "ok" else None), + ts=raw["ts"], + latency_s=raw["latency_s"], + raw_sha256=_sha256(path), + ) + stream.write(dump(record) + "\n") + stream.flush() + os.fsync(stream.fileno()) + attempted += number + errors = errors + 1 if status != "ok" and code != 422 else 0 + if raw["client_error"] == "KeyboardInterrupt" or code in (401, 403, 429) or errors >= 3: + stop = raw["client_error"] or "access/rate limit or three consecutive infrastructure errors" + break + if stop: + break + except KeyboardInterrupt: + stop = "interrupted" + except Exception as exc: + stop = f"{type(exc).__name__}: {exc}" + raise + finally: + meta.update(stop_reason=stop, attempted=attempted, collected=attempted == len(items) and stop is None) + write_json(output / "run.meta.final.json", meta) + (output / "run.meta.final.json").replace(output / "run.meta.json") + return output + + class SavedResponseEngine: """Sequential, unsharded evaluate() adapter; never normalizes saved answers.""" @@ -374,22 +486,36 @@ def replay_bundle(intern_root, scorer_root, responses, *, checkpoint, backend, t def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) - for command in ("verify", "replay"): + for command in ("verify", "replay", "collect"): child = commands.add_parser(command) child.add_argument("--intern-root", type=Path, required=True) child.add_argument("--scorer-root", type=Path, required=True) if command == "replay": child.add_argument("--round", type=Path, required=True, help="A's saved round-N directory") child.add_argument("--out", type=Path, required=True, help="New replay output directory") + if command == "collect": + child.add_argument("--endpoint", required=True) + child.add_argument("--path", choices=("/v1/decisions", "/v1/systemone"), required=True) + child.add_argument("--model") + child.add_argument("--timeout", type=float, default=120) + child.add_argument("--run-meta", type=Path, required=True) + child.add_argument("--out", type=Path, required=True) args = parser.parse_args() if args.command == "verify": report = verify_sources(args.intern_root, args.scorer_root) + elif args.command == "collect": + report = {"run": str(collect(args))} else: responses, provenance, config = load_round(args.intern_root, args.scorer_root, args.round) report = replay_bundle( args.intern_root, args.scorer_root, responses, **config, output=args.out, provenance=provenance ) print(json.dumps(report, indent=2)) + if ( + args.command == "collect" + and not json.loads((Path(report["run"]) / "run.meta.json").read_text(encoding="utf-8"))["collected"] + ): + raise SystemExit(2) if __name__ == "__main__": diff --git a/benchmarks/results/intern-accuracy-v1-h800.md b/benchmarks/results/intern-accuracy-v1-h800.md new file mode 100644 index 0000000..cce4f30 --- /dev/null +++ b/benchmarks/results/intern-accuracy-v1-h800.md @@ -0,0 +1,50 @@ +# Intern-Decision accuracy-v1 on one H800 + +Three pinned Intern checkpoints used the official HF service, BF16, SDPA and +temperature 1. Each received five warmups, then one serial pass over all seven +suites: 10,751 requests and 12,351 decisions. All responses were valid. + +## Results + +| Suite | Requests / decisions | Intern 0.8B | Intern 2B | Intern 4B | +|---|---:|---:|---:|---:| +| JevBench easy | 48 / 48 | 47/48 | 48/48 | 48/48 | +| JevBench original | 72 / 72 | 57/72 | 61/72 | 71/72 | +| JevBench hard | 111 / 111 | 60/111 | 72/111 | 81/111 | +| AG News | 7,600 / 7,600 | 6,735/7,600 | 6,833/7,600 | 6,903/7,600 | +| ToolACE | 310 / 310 | 293/310 | 299/310 | 299/310 | +| Typed Decisions | 400 / 2,000 | 1,552/2,000 | 1,582/2,000 | 1,613/2,000 | +| WildJailBreak | 2,210 / 2,210 | 1,414/2,210 | 1,731/2,210 | 1,982/2,210 | +| **Mean of seven accuracies** | Each suite has equal weight | **79.4077%** | **84.7675%** | **89.8854%** | + +The mean uses each suite's unrounded accuracy, not the pooled fraction over +12,351 decisions. Typed Decisions keeps all five fields in one request. +These are HF T=1 results; no XTuner calibration or published-score equivalence +is claimed. The pinned Hard input has no `gold_probs`, so TVD is unmeasured. + +## Reproduce + +Use the [verify, collect and replay commands](../INTERN_DECISION.md). Inputs and +the official evaluator are pinned to Intern-Decision +`3572c8a68b5df5dafe02d0e093989ba8ec0183bc`; JevBench is +`7ce310c7262ed49cc85853339a8a42459298e3f3`. + +| Checkpoint | Revision | +|---|---| +| Intern-Decision-0.8B | `85a0cc5a99d67ea8d56dfe98115689212867171d` | +| Intern-Decision-2B | `8797836c65fc91a2435b1fb6850b5f0aabd75cc3` | +| Intern-Decision-4B | `0e5e6aa7d6d750e2b1504ba11a8136cb58aeb3cd` | + +The service used PyTorch 2.9.1+cu128, Transformers 5.14.1, an 8192-token limit, +thinking off and no KV cache. Optional FLA/causal-conv1d paths were unavailable. +This run was collected on 2026-10-07 UTC, Slurm job 415106. It is a separate +allocation from the [five-model public231 comparison](public231-h800.md); +latencies from those two runs should not be combined. + +A non-author checked every saved request and response against the fixed inputs, +then independently re-scored them and called the unmodified official `evaluate` +on CPU. Counts and metrics agreed. Source, environment and response hashes are +retained with the run. The data bundle is not vendored; its manifest leaves +AG News licensing unresolved. The public attachment retains predictions, scores +and timing, omitting input states and returned legends. Full raw evidence stays +with the original run; the public copy is not a complete replay bundle. diff --git a/tests/benchmarks/test_intern_decision.py b/tests/benchmarks/test_intern_decision.py index 09ad5e9..66ac7f4 100644 --- a/tests/benchmarks/test_intern_decision.py +++ b/tests/benchmarks/test_intern_decision.py @@ -22,6 +22,22 @@ SUITES = MODULE.SUITES +def collection_args(intern, scorer, directory, meta): + environment = directory.parent / "collector-environment.json" + source = directory.parent / "environment.json" + environment.write_bytes(source.read_bytes() if source.exists() else json.dumps(meta).encode()) + return SimpleNamespace( + intern_root=intern, + scorer_root=scorer, + endpoint="http://fixture.invalid", + path="/v1/decisions", + model="protocol-fixture", + timeout=1.0, + run_meta=environment, + out=directory.parent / "collected", + ) + + def fixture_answer(body): answers = {} for field, question in body["questions"].items(): @@ -35,6 +51,71 @@ def fixture_answer(body): return {"answers": answers, "model": "protocol-fixture-no-model"} +@pytest.fixture(scope="module") +def full_collection(tmp_path_factory): + import httpx + + intern, scorer = references() + root = tmp_path_factory.mktemp("full-seven-suite-collection") + meta = { + "checkpoint": {"repo": "protocol-fixture-no-model", "revision": "a" * 40}, + "backend": {"name": "httpx-mock", "version": "fixture"}, + "sampling": {"temperature": 1}, + "hardware": "CPU protocol fixture", + } + args = collection_args(intern, scorer, root / "fixture", meta) + counts = {"requests": 0, "decisions": 0} + + def handler(request): + body = json.loads(request.content) + assert set(body) == {"state", "questions", "thinking", "model"} + assert all(set(question) <= {"type", "instructions", "criteria"} for question in body["questions"].values()) + counts["requests"] += 1 + counts["decisions"] += len(body["questions"]) + return httpx.Response(200, json=fixture_answer(body)) + + output = MODULE.collect(args, transport=httpx.MockTransport(handler)) + saved, provenance, config = MODULE.load_round(intern, scorer, output / "round-1") + return intern, scorer, output, saved, provenance, config, counts + + +def test_full_pinned_seven_suite_collection(full_collection): + _, _, output, saved, provenance, _, counts = full_collection + assert counts == {"requests": 10756, "decisions": 12356} + assert len(saved) == len(provenance["raw"]) == 10751 + assert sum(len(response["answers"]) for response in saved.values()) == 12351 + assert len(set(record["path"] for record in provenance["raw"])) == 10751 + assert json.loads((output / "run.meta.json").read_text())["collected"] is True + + +def test_full_pinned_collection_reuses_official_replay(full_collection): + if importlib.util.find_spec("torch") is None: + pytest.skip("full collection passed; official saved-response replay needs prepared Torch CPU dependencies") + intern, scorer, output, saved, provenance, config, _ = full_collection + report = MODULE.replay_bundle( + intern, scorer, saved, **config, output=output.parent / "official-replay", provenance=provenance + ) + assert report["complete"], json.dumps(report, indent=2) + assert report["valid"] == report["planned_decisions"] == 12351 + assert report["attempted"] == 10751 + + +def test_official_overflow_keeps_suite_failure_report(full_collection): + if importlib.util.find_spec("torch") is None: + pytest.skip("official OverflowError regression needs prepared Torch CPU dependencies") + intern, scorer, output, original, _, config, _ = full_collection + saved = dict(original) + key = next(iter(saved)) + saved[key] = copy.deepcopy(saved[key]) + probs = saved[key]["answers"]["decision"]["probabilities"] + probs[next(iter(probs))] = 10**400 + target = output.parent / "official-overflow-replay" + report = MODULE.replay_bundle(intern, scorer, saved, **config, output=target) + assert not report["complete"] and report["planned_decisions"] == 12351 + assert "OverflowError at source_line=1" in report["datasets"][SUITES[0]]["error"] + assert (target / "report.json").exists() and not (target / "complete.json").exists() + + def test_replay_catches_real_pinned_scorer_overflow_without_torch(tmp_path, monkeypatch): """A small evaluator adapter calls the real pinned scorer; it is not full official evaluate.""" from benchmarks.evidence import load_reference @@ -75,6 +156,264 @@ def evaluate(engine, data_path, output_path): assert max(probs.values()) == 10**400 +def test_seven_suite_collection_uses_one_post_for_all_fields(disk_round): + import httpx + + intern, scorer, directory, meta, _, _, _, _ = disk_round + verified = MODULE.verify_sources(intern, scorer) + for suite, entry in verified["datasets"].items(): + source = intern / "benchmarks/accuracy-v1" / entry["path"] + row = json.loads(source.read_text()) + row["questions"] = {"first": row.pop("question"), "second": {"type": "choice", "criteria": {"B": "b"}}} + row.update(targets={"first": "A", "second": "B"}, provenance={"private": "not model evidence"}) + source.write_text(json.dumps(row) + "\n") + entry["decisions"] = 2 + verified["files"][entry["path"]] = MODULE._sha256(source) + calls = [] + + def handler(request): + body = json.loads(request.content) + calls.append(body) + assert set(body) == {"state", "questions", "thinking", "model"} + assert list(body["questions"]) == ["first", "second"] + return httpx.Response(200, json=fixture_answer(body)) + + args = collection_args(intern, scorer, directory, meta) + output = MODULE.collect(args, transport=httpx.MockTransport(handler)) + saved, evidence, _ = MODULE.load_round(intern, scorer, output / "round-1") + assert len(calls) == 5 + 7 # Five warmups; never split the two fields into separate POSTs. + assert len(saved) == 7 + assert {key[2] for key in saved} == {"item"} # Reused IDs across suites cannot overwrite raw files. + assert len({row["path"] for row in evidence["raw"]}) == 7 + assert all(value == fixture_answer(calls[5]) for value in saved.values()) + final_meta = json.loads((output / "run.meta.json").read_text()) + assert final_meta["collected"] and final_meta["attempted"] == 7 + assert sum(entry["decisions"] for entry in final_meta["plan"]["datasets"].values()) == 14 + assert not (output / "complete.json").exists() # Collection does not establish official scoring success. + assert MODULE._sha256(output / "environment.json") == final_meta["env_sha256"] + + +@pytest.mark.parametrize("failure,attempted", [(401, 1), (503, 3), (422, 7), ("timeout", 3), ("interrupt", 1)]) +def test_collection_preserves_failures_and_stops_without_claiming_scores(disk_round, failure, attempted): + import httpx + + intern, scorer, directory, meta, _, _, _, _ = disk_round + calls = [] + + def handler(request): + body = json.loads(request.content) + calls.append(body) + if len(calls) <= 5: + return httpx.Response(200, json=fixture_answer(body)) + if failure == "interrupt": + raise KeyboardInterrupt + if failure == "timeout": + raise httpx.ReadTimeout("fixture timeout") + return httpx.Response(failure, json=fixture_answer(body)) + + output = MODULE.collect(collection_args(intern, scorer, directory, meta), transport=httpx.MockTransport(handler)) + saved, evidence, _ = MODULE.load_round(intern, scorer, output / "round-1") + assert len(saved) == attempted and all(value is None for value in saved.values()) + assert len(evidence["raw"]) == attempted + final_meta = json.loads((output / "run.meta.json").read_text()) + assert final_meta["attempted"] == attempted + assert final_meta["collected"] is (failure == 422) + assert bool(final_meta["stop_reason"]) is (failure != 422) + records = [json.loads(line) for line in (output / "round-1/results.jsonl").read_text().splitlines()] + assert all(row["latency_s"] >= 0 and not row["ok"] for row in records) + + +def test_collection_stopped_in_warmup_replays_as_unattempted(disk_round): + import httpx + + intern, scorer, directory, meta, _, _, _, _ = disk_round + output = MODULE.collect( + collection_args(intern, scorer, directory, meta), + transport=httpx.MockTransport(lambda _: httpx.Response(429, text="rate limit")), + ) + saved, evidence, config = MODULE.load_round(intern, scorer, output / "round-1") + assert saved == {} and evidence["results_sha256"] is None + report = MODULE.replay_bundle(intern, scorer, saved, **config, output=directory.parent / "stopped-replay") + assert report["planned"] == 7 and report["attempted"] == 0 and not report["complete"] + + +def test_collect_cli_controlled_stop_returns_nonzero(disk_round, monkeypatch, capsys): + import httpx + + intern, scorer, directory, meta, _, _, _, _ = disk_round + args = collection_args(intern, scorer, directory, meta) + original = MODULE.collect + monkeypatch.setattr( + MODULE, + "collect", + lambda options: original(options, transport=httpx.MockTransport(lambda _: httpx.Response(403, text="denied"))), + ) + monkeypatch.setattr( + "sys.argv", + [ + "intern_decision.py", + "collect", + "--intern-root", + str(intern), + "--scorer-root", + str(scorer), + "--endpoint", + args.endpoint, + "--path", + args.path, + "--run-meta", + str(args.run_meta), + "--out", + str(args.out), + ], + ) + with pytest.raises(SystemExit) as stopped: + MODULE.main() + assert stopped.value.code == 2 + assert json.loads(capsys.readouterr().out)["run"] == str(args.out) + assert json.loads((args.out / "run.meta.json").read_text())["collected"] is False + + +@pytest.mark.parametrize( + "endpoint", ["http://user:DUMMY_SECRET@fixture.invalid", "http://fixture.invalid?key=DUMMY_SECRET"] +) +def test_collect_cli_rejects_credentials_before_output(disk_round, monkeypatch, endpoint): + import httpx + + intern, scorer, directory, meta, _, _, _, _ = disk_round + args = collection_args(intern, scorer, directory, meta) + original = MODULE.collect + calls = [] + monkeypatch.setattr( + MODULE, + "collect", + lambda options: original(options, transport=httpx.MockTransport(lambda request: calls.append(request))), + ) + monkeypatch.setattr( + "sys.argv", + [ + "intern_decision", + "collect", + "--intern-root", + str(intern), + "--scorer-root", + str(scorer), + "--endpoint", + endpoint, + "--path", + args.path, + "--run-meta", + str(args.run_meta), + "--out", + str(args.out), + ], + ) + with pytest.raises(ValueError, match="without credentials"): + MODULE.main() + assert not args.out.exists() and not calls + assert all("DUMMY_SECRET" not in path.read_text() for path in directory.parent.rglob("*") if path.is_file()) + + +@pytest.mark.parametrize("failure", ["mkdir", "environment.json", "run.meta.json"]) +def test_collect_closes_client_when_output_setup_fails(disk_round, monkeypatch, failure): + import httpx + from benchmarks import client + + intern, scorer, directory, meta, _, _, _, _ = disk_round + args = collection_args(intern, scorer, directory, meta) + created, calls = [], [] + original_client, original_open = client.DecisionClient, Path.open + + def make_client(*args, **kwargs): + result = original_client(*args, **kwargs) + created.append(result) + return result + + def open_file(path, *options, **kwargs): + if path == args.out / failure: + raise OSError("fixture output failure") + return original_open(path, *options, **kwargs) + + if failure == "mkdir": + args.out.mkdir() + monkeypatch.setattr(client, "DecisionClient", make_client) + monkeypatch.setattr(Path, "open", open_file) + with pytest.raises(OSError): + MODULE.collect(args, transport=httpx.MockTransport(lambda request: calls.append(request))) + assert len(created) == 1 and created[0].http.is_closed + assert not calls + + +@pytest.mark.parametrize("unicode_field", ["task_id", "environment"]) +def test_collect_cli_writes_and_reads_utf8_under_non_utf8_locale(disk_round, unicode_field): + import subprocess + import sys + + intern, scorer, directory, meta, _, _, _, _ = disk_round + args = collection_args(intern, scorer, directory, meta) + plan = MODULE.verify_sources(intern, scorer)["datasets"] + if unicode_field == "task_id": + for entry in plan.values(): + path = intern / "benchmarks/accuracy-v1" / entry["path"] + row = json.loads(path.read_text(encoding="utf-8")) + row["id"] = "item\u2028unicode" + path.write_text(json.dumps(row, ensure_ascii=False) + "\n", encoding="utf-8") + else: + environment = json.loads(args.run_meta.read_text(encoding="utf-8")) + environment["hardware"] = "CPU\u2028fixture" + args.run_meta.write_text(json.dumps(environment, ensure_ascii=False), encoding="utf-8") + script = r""" +import json, locale, sys +from pathlib import Path +import httpx +from benchmarks import intern_decision as module +encoding = locale.getpreferredencoding(False) +if encoding.lower().replace("-", "") == "utf8": + sys.exit(77) +print("locale=" + encoding) +intern, scorer, environment, output = map(Path, sys.argv[1:5]) +plan = json.loads(sys.argv[5]) +module.verify_sources = lambda *_: dict( + intern_root=str(intern), scorer_root=str(scorer), datasets=plan, + files={}, manifest_sha256="fixture", source_revisions={}) +calls = [] +def handler(request): + calls.append(request) + return httpx.Response(200, json={"answers": {"decision": {"type": "choice", "probabilities": {"A": 1}}}}) +collect = module.collect +module.collect = lambda args: collect(args, transport=httpx.MockTransport(handler)) +sys.argv = ["intern_decision", "collect", "--intern-root", str(intern), "--scorer-root", str(scorer), + "--endpoint", "http://fixture.invalid", "--path", "/v1/decisions", + "--run-meta", str(environment), "--out", str(output)] +module.main() +assert len(calls) == 12 +""" + result = subprocess.run( + [sys.executable, "-c", script, str(intern), str(scorer), str(args.run_meta), str(args.out), json.dumps(plan)], + env=dict( + os.environ, + PYTHONPATH=str(Path(__file__).resolve().parents[2]), + PYTHONUTF8="0", + PYTHONCOERCECLOCALE="0", + LC_ALL="C", + PYTHONDONTWRITEBYTECODE="1", + ), + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode == 77: + pytest.skip("This platform keeps locale C in UTF-8 mode") + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.startswith("locale=") + assert json.loads((args.out / "run.meta.json").read_text(encoding="utf-8"))["collected"] is True + with (args.out / "round-1/results.jsonl").open(encoding="utf-8") as stream: + records = [json.loads(line) for line in stream] + assert len(records) == 7 + if unicode_field == "task_id": + assert all(row["task_id"] == "item\u2028unicode" for row in records) + + def test_actual_a_loopback_output_replays_without_layout_assumptions(tmp_path, monkeypatch): from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import threading @@ -362,6 +701,38 @@ def test_replay_rejects_changed_complete_request_envelope(disk_round, change): MODULE.load_round(intern, scorer, directory) +@pytest.mark.parametrize( + "field", + [ + "endpoint", + "path", + "model_configured", + "env_sha256", + "dataset", + "plan", + "stop_reason", + "attempted", + "collected", + ], +) +def test_collector_rejects_environment_fields_it_controls(disk_round, field): + import httpx + + intern, scorer, directory, meta, _, _, _, _ = disk_round + args = collection_args(intern, scorer, directory, meta) + environment = json.loads(args.run_meta.read_text()) + environment[field] = None + args.run_meta.write_text(json.dumps(environment)) + with pytest.raises(ValueError, match="controlled"): + MODULE.collect( + args, + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json=fixture_answer(json.loads(request.content))) + ), + ) + assert not args.out.exists() + + def test_disk_round_keeps_original_response_and_evidence(disk_round): intern, scorer, directory, meta, raw, _, raw_path, _ = disk_round saved, provenance, config = MODULE.load_round(intern, scorer, directory)