From c6f2271efd95abb014b427e1e44127f3a7c5af9b Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:41:06 +0800 Subject: [PATCH 01/12] test(agent-loop): add reproducible paired benchmark lane --- docs/experiments/agent-loop-baseline.md | 59 +- scripts/agent_loop_benchmark.py | 509 ++++++++++++++++++ scripts/agent_loop_cases.json | 45 ++ scripts/tests/test_agent_loop_benchmark.py | 174 ++++++ scripts/tests/test_agent_loop_report.py | 4 +- .../fixtures/agent-loop-multi-file/alpha.txt | 2 + tests/fixtures/agent-loop-multi-file/beta.txt | 2 + 7 files changed, 792 insertions(+), 3 deletions(-) create mode 100644 scripts/agent_loop_benchmark.py create mode 100644 scripts/tests/test_agent_loop_benchmark.py create mode 100644 tests/fixtures/agent-loop-multi-file/alpha.txt create mode 100644 tests/fixtures/agent-loop-multi-file/beta.txt diff --git a/docs/experiments/agent-loop-baseline.md b/docs/experiments/agent-loop-baseline.md index 619860c55..d2c66a2ae 100644 --- a/docs/experiments/agent-loop-baseline.md +++ b/docs/experiments/agent-loop-baseline.md @@ -179,12 +179,15 @@ expectation. The profiler includes a deterministic case fingerprint so changed case definitions cannot silently compare as the same pair. -The corpus contains sixteen cases: ten coding/inspection cases and six general +The corpus contains seventeen cases: eleven coding/inspection cases and six general file/command cases. The coding/inspection cases are: - `readonly_review`: read-only status/read/search/diff inspection. - `focused_edit_validation`: one focused edit, diff review, and successful `cargo check`. +- `guarded_multi_file_edit`: search → selected reads → one guarded two-file edit → + post-edit reads/diff, with an independent fixture-side oracle for both changed + files and preserved unrelated sentinels. - `failed_validation_recovery`: a deliberately failing `cargo test`, diagnostic inspection, fix, and successful rerun. - `long_validation_handoff`: real same-execution Job handoff and terminal @@ -255,6 +258,60 @@ Runtime Project, Workflow Session, Window, tunnel, and host-internal identifiers may be used transiently to select authoritative evidence. They must not be written into the manifest, docs, tests, run annotation, or durable benchmark result. +## Reproducible paired benchmark command + +`scripts/agent_loop_benchmark.py` is a thin orchestration layer over this existing +manifest and `agent_loop_report.py`. It does not contain another agent loop or a +benchmark-only WebCodex execution path. A caller-supplied Host driver performs the +real task using the ordinary Direct or Code Mode contracts. The orchestrator owns +only reproducibility mechanics: + +- one fresh detached Git worktree per sample at one exact base revision; +- the requested base revision must equal the benchmark checkout's current `HEAD`, so the manifest, fixtures, and code under test come from one revision; +- the same case fingerprint/prompt/correctness definition for both variants; +- alternating pair order (`direct/code_mode`, then `code_mode/direct`) to expose + one-sided process/cache ordering effects; +- bounded annotation and the existing report/correctness compatibility gates; +- fixture-side disk/diff oracles where the manifest provides one; +- explicit `pass`, `fail`, `partial`, and `unsupported` samples. None are + silently filtered from the aggregate; +- cleanup of only the worktrees/temp state created by the orchestrator. + +Example: + +```bash +python3 scripts/agent_loop_benchmark.py \ + --base-revision "$(git rev-parse HEAD)" \ + --driver "python3 /path/to/real_host_driver.py" \ + --pairs 2 \ + --output /tmp/webcodex-agent-loop-benchmark.json +``` + +For each invocation the driver receives `WEBCODEX_BENCH_WORKSPACE`, +`WEBCODEX_BENCH_CASE_ID`, `WEBCODEX_BENCH_VARIANT`, +`WEBCODEX_BENCH_SURFACE`, `WEBCODEX_BENCH_BASE_REVISION`, +`WEBCODEX_BENCH_CASE_FINGERPRINT`, `WEBCODEX_BENCH_PROMPT`, and +`WEBCODEX_BENCH_DRIVER_RESULT`. It writes one bounded JSON receipt to the last +path. The receipt is capped at 64 KiB; driver stdout/stderr is not retained by the orchestrator. `status` is required and is one of `pass/fail/partial/unsupported`. +Non-unsupported samples also provide the authoritative ActionAudit database path +and Workflow Session id; optional trace root, repair-turn annotation, task timing, +and correctness verdicts use the existing reporter semantics. The durable benchmark +result does not persist runtime Project, Window, tunnel, or Host-private ids. + +The default lane is exactly `readonly_review`, `guarded_multi_file_edit`, and +`long_validation_handoff`, matching #822. The handoff case is valid only when the +driver observes a genuine pending validation and follows the same Job to terminal; +a synchronous completion or substitute pending process must be reported +`partial`/`unsupported`, not upgraded to success. + +This command coordinates real captures; it is not itself evidence that a model made +fewer inference turns. Deterministic runtime/script acceptance and fixture oracles +prove runtime mechanics and final state. Direct-vs-Code Mode ergonomics require a +real Host/model capture. Exact model round trips remain unavailable unless the Host +records turn identity; meaningful outer calls remain only the documented proxy. +Report cold/warm context, pair order, sample count, and dispersion rather than +treating one run or one ordering as causal proof. + ## Typed Surface v1 observed dogfood — 2026-09-18 A real self-hosted dogfood build from the Typed Surface v1 branch ran the three diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py new file mode 100644 index 000000000..d7c1a7c4a --- /dev/null +++ b/scripts/agent_loop_benchmark.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 +"""Run reproducible paired Agent Loop benchmark samples over fresh Git worktrees. + +The script is orchestration only. A caller-supplied Host driver performs the real +Direct/Code Mode task. This runner owns fresh worktrees, bounded annotation, +existing agent_loop_report summarization/comparison, fixture-side oracles, sample +status retention, and cleanup of its own temporary resources. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import shlex +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +try: + from scripts import agent_loop_report as report +except ModuleNotFoundError: + import agent_loop_report as report + +STATUS_VALUES = frozenset(("pass", "fail", "partial", "unsupported")) +DEFAULT_CASES = ("readonly_review", "guarded_multi_file_edit", "long_validation_handoff") +VARIANTS = ("direct", "code_mode") +MAX_DRIVER_RECEIPT_BYTES = 64 * 1024 + + +class BenchmarkError(ValueError): + pass + + +def _stable_json(value: Any) -> str: + return json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def _run(argv: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + argv, + cwd=cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + +def _run_driver(argv: list[str], *, cwd: Path, env: dict[str, str]) -> subprocess.CompletedProcess[bytes]: + try: + return subprocess.run( + argv, + cwd=cwd, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + except OSError as exc: + raise BenchmarkError("could not start benchmark Host driver") from exc + + +def _require_exact_revision(repo: Path, revision: str) -> str: + completed = _run(["git", "rev-parse", "--verify", f"{revision}^{{commit}}"], cwd=repo) + if completed.returncode != 0: + raise BenchmarkError(f"could not resolve base revision: {revision}") + resolved = completed.stdout.strip() + if not report._is_exact_git_revision(resolved): + raise BenchmarkError("resolved base revision is not an exact 40-hex commit") + return resolved + + +def _head_revision(repo: Path) -> str: + completed = _run(["git", "rev-parse", "HEAD"], cwd=repo) + if completed.returncode != 0: + raise BenchmarkError("could not resolve current checkout HEAD") + head = completed.stdout.strip() + if not report._is_exact_git_revision(head): + raise BenchmarkError("current checkout HEAD is not an exact 40-hex commit") + return head + + +def _case(manifest: dict[str, Any], case_id: str) -> dict[str, Any]: + return report._case_by_id(manifest, case_id) + + +def _surface(case: dict[str, Any], variant: str) -> str: + if variant == "direct": + return "direct" + surface = case.get("code_mode_surface") + canonical = report._canonical_code_mode_surface(surface) + if canonical is None: + raise BenchmarkError(f"case {case['id']} has no usable Code Mode surface") + return canonical + + +def _worktree_add(repo: Path, path: Path, base_revision: str) -> None: + completed = _run(["git", "worktree", "add", "--detach", str(path), base_revision], cwd=repo) + if completed.returncode != 0: + raise BenchmarkError(completed.stderr.strip() or "git worktree add failed") + + +def _worktree_remove(repo: Path, path: Path) -> None: + _run(["git", "worktree", "remove", "--force", str(path)], cwd=repo) + if path.exists(): + shutil.rmtree(path, ignore_errors=True) + _run(["git", "worktree", "prune"], cwd=repo) + + +def _workspace_status(workspace: Path) -> str: + completed = _run(["git", "status", "--porcelain", "--untracked-files=all"], cwd=workspace) + if completed.returncode != 0: + raise BenchmarkError(completed.stderr.strip() or "git status failed") + return completed.stdout + + +def _fixture_oracle(case: dict[str, Any], workspace: Path) -> dict[str, Any]: + spec = (case.get("correctness") or {}).get("fixture_oracle") + if not isinstance(spec, dict): + return {"available": False, "passed": None, "checks": [], "reason": "case has no fixture_oracle"} + + checks: list[dict[str, Any]] = [] + for rel_path, expected in sorted((spec.get("files") or {}).items()): + target = workspace / rel_path + try: + text = target.read_text(encoding="utf-8") + except OSError: + checks.append({"path": rel_path, "passed": False, "reason": "file unavailable"}) + continue + required = expected.get("required_text") or [] + forbidden = expected.get("forbidden_text") or [] + passed = all(item in text for item in required) and all(item not in text for item in forbidden) + checks.append({"path": rel_path, "passed": passed, "reason": None if passed else "text oracle mismatch"}) + + expected_changed = sorted(spec.get("changed_files") or []) + if expected_changed: + completed = _run(["git", "diff", "--name-only", "HEAD", "--"], cwd=workspace) + changed = sorted(line for line in completed.stdout.splitlines() if line) + checks.append( + { + "kind": "changed_files", + "passed": completed.returncode == 0 and changed == expected_changed, + "expected": expected_changed, + "actual": changed, + } + ) + + passed = bool(checks) and all(check.get("passed") is True for check in checks) + return {"available": True, "passed": passed, "checks": checks, "reason": None} + + +def _load_driver_receipt(path: Path) -> dict[str, Any]: + try: + if path.stat().st_size > MAX_DRIVER_RECEIPT_BYTES: + raise BenchmarkError("driver result exceeds the 64 KiB receipt limit") + except OSError as exc: + raise BenchmarkError("driver did not create its result receipt") from exc + try: + value = json.loads(path.read_text(encoding="utf-8")) + except OSError as exc: + raise BenchmarkError("driver did not create its result receipt") from exc + except json.JSONDecodeError as exc: + raise BenchmarkError(f"driver result is not valid JSON: {exc}") from exc + if not isinstance(value, dict): + raise BenchmarkError("driver result must be a JSON object") + status = value.get("status") + if status not in STATUS_VALUES: + raise BenchmarkError("driver result status must be pass, fail, partial, or unsupported") + return value + + +def _annotation( + case: dict[str, Any], + *, + variant: str, + surface: str, + base_revision: str, + receipt: dict[str, Any], + oracle: dict[str, Any], +) -> dict[str, Any]: + correctness = receipt.get("correctness") + if correctness is not None and not isinstance(correctness, dict): + raise BenchmarkError("driver correctness must be an object when present") + correctness = dict(correctness or {}) + + if oracle["available"]: + if oracle["passed"] is not True: + correctness["task_verdict"] = "fail" + elif correctness.get("task_verdict") is None: + correctness["task_verdict"] = "pass" + if case["validation"]["required"] is False and correctness.get("validation_verdict") is None: + correctness["validation_verdict"] = "not_required" + + value: dict[str, Any] = { + "schema_version": report.RUN_ANNOTATION_SCHEMA_VERSION, + "case_id": case["id"], + "variant": variant, + "surface": surface, + "base_revision": base_revision, + "case_fingerprint": report._case_fingerprint(case), + } + for field in ("repair_turns", "task_timing"): + if receipt.get(field) is not None: + value[field] = receipt[field] + if correctness: + value["correctness"] = correctness + return report.validate_run_annotation(value) + + +def _summary_for_receipt( + case: dict[str, Any], + *, + variant: str, + surface: str, + base_revision: str, + receipt: dict[str, Any], + annotation_path: Path, +) -> dict[str, Any] | None: + if receipt["status"] == "unsupported": + return None + audit_db = receipt.get("audit_db") + workflow_session_id = receipt.get("workflow_session_id") + trace_root = receipt.get("trace_root") + if not isinstance(audit_db, str) or not audit_db: + raise BenchmarkError("non-unsupported driver result requires audit_db") + if not isinstance(workflow_session_id, str) or not workflow_session_id: + raise BenchmarkError("non-unsupported driver result requires workflow_session_id") + return report.summarize( + trace_root=Path(trace_root) if isinstance(trace_root, str) and trace_root else None, + audit_db=Path(audit_db), + workflow_session_id=workflow_session_id, + case_manifest=report.DEFAULT_CASE_MANIFEST, + case_id=case["id"], + variant=variant, + surface=surface, + base_revision=base_revision, + run_annotation=annotation_path, + ) + + +def _run_sample( + repo: Path, + root: Path, + driver_argv: list[str], + case: dict[str, Any], + *, + variant: str, + pair_index: int, + ordinal: int, + base_revision: str, +) -> dict[str, Any]: + label = f"{case['id']}-p{pair_index + 1}-{ordinal + 1}-{variant}" + workspace = root / "worktrees" / label + receipt_path = root / "receipts" / f"{label}.json" + annotation_path = root / "annotations" / f"{label}.json" + surface = _surface(case, variant) + _worktree_add(repo, workspace, base_revision) + started = time.time_ns() // 1_000_000 + try: + if _workspace_status(workspace): + raise BenchmarkError("fresh benchmark worktree is not clean") + env = dict(os.environ) + env.update( + { + "WEBCODEX_BENCH_WORKSPACE": str(workspace), + "WEBCODEX_BENCH_CASE_ID": case["id"], + "WEBCODEX_BENCH_VARIANT": variant, + "WEBCODEX_BENCH_SURFACE": surface, + "WEBCODEX_BENCH_BASE_REVISION": base_revision, + "WEBCODEX_BENCH_CASE_FINGERPRINT": report._case_fingerprint(case), + "WEBCODEX_BENCH_PROMPT": case["prompt"], + "WEBCODEX_BENCH_DRIVER_RESULT": str(receipt_path), + } + ) + completed = _run_driver(driver_argv, cwd=workspace, env=env) + ended = time.time_ns() // 1_000_000 + if not receipt_path.exists(): + return { + "case_id": case["id"], + "variant": variant, + "surface": surface, + "status": "fail", + "driver_exit_code": completed.returncode, + "reason_code": "driver_receipt_missing", + } + + receipt = _load_driver_receipt(receipt_path) + if receipt.get("task_timing") is None: + receipt["task_timing"] = {"started_at_ms": started, "ended_at_ms": ended} + oracle = _fixture_oracle(case, workspace) + annotation = _annotation( + case, + variant=variant, + surface=surface, + base_revision=base_revision, + receipt=receipt, + oracle=oracle, + ) + annotation_path.write_text(_stable_json(annotation), encoding="utf-8") + summary = _summary_for_receipt( + case, + variant=variant, + surface=surface, + base_revision=base_revision, + receipt=receipt, + annotation_path=annotation_path, + ) + effective_status = receipt["status"] + if completed.returncode != 0 and effective_status == "pass": + effective_status = "fail" + if oracle["available"] and oracle["passed"] is not True and effective_status == "pass": + effective_status = "fail" + if summary is not None and effective_status == "pass": + correctness = summary.get("correctness") or {} + task_verdict = correctness.get("task_verdict") + validation_verdict = correctness.get("validation_verdict") + if task_verdict == "fail" or validation_verdict == "fail": + effective_status = "fail" + elif task_verdict != "pass": + effective_status = "partial" + elif case["validation"]["required"] and validation_verdict != "pass": + effective_status = "partial" + elif not case["validation"]["required"] and validation_verdict not in ("pass", "not_required"): + effective_status = "partial" + + if ( + case["id"] == "long_validation_handoff" + and summary is not None + and effective_status == "pass" + and report._get_path(summary, "job_convergence.pending_handoff_count") in (None, 0) + ): + effective_status = "partial" + return { + "case_id": case["id"], + "variant": variant, + "surface": surface, + "status": effective_status, + "driver_exit_code": completed.returncode, + "fixture_oracle": oracle, + "summary": summary, + } + except (BenchmarkError, report.ReportError): + return { + "case_id": case["id"], + "variant": variant, + "surface": surface, + "status": "fail", + "driver_exit_code": None, + "reason_code": "sample_contract_error", + "fixture_oracle": {"available": False, "passed": None, "checks": [], "reason": "sample failed before oracle"}, + "summary": None, + } + finally: + _worktree_remove(repo, workspace) + + +def _comparison(direct: dict[str, Any], code_mode: dict[str, Any]) -> dict[str, Any] | None: + if direct["summary"] is None or code_mode["summary"] is None: + return None + return report.compare_reports(direct["summary"], code_mode["summary"]) + + +def _pair_order(pair_index: int) -> tuple[str, str]: + return VARIANTS if pair_index % 2 == 0 else tuple(reversed(VARIANTS)) + + +def run_benchmark( + *, + repo: Path, + base_revision: str, + driver_argv: list[str], + case_ids: list[str], + pairs: int, +) -> dict[str, Any]: + if pairs < 1: + raise BenchmarkError("pairs must be at least 1") + manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + selected = [_case(manifest, case_id) for case_id in case_ids] + resolved = _require_exact_revision(repo, base_revision) + if _head_revision(repo) != resolved: + raise BenchmarkError( + "base revision must equal the current checkout HEAD; check out the target commit before benchmarking" + ) + temp_owner = tempfile.TemporaryDirectory(prefix="webcodex-agent-loop-bench-") + root = Path(temp_owner.name) + for name in ("worktrees", "receipts", "annotations"): + (root / name).mkdir(parents=True, exist_ok=True) + + cases_out: list[dict[str, Any]] = [] + try: + for case in selected: + pair_outputs: list[dict[str, Any]] = [] + for pair_index in range(pairs): + samples: list[dict[str, Any]] = [] + order = _pair_order(pair_index) + for ordinal, variant in enumerate(order): + samples.append( + _run_sample( + repo, + root, + driver_argv, + case, + variant=variant, + pair_index=pair_index, + ordinal=ordinal, + base_revision=resolved, + ) + ) + by_variant = {sample["variant"]: sample for sample in samples} + pair_outputs.append( + { + "pair_index": pair_index, + "order": list(order), + "samples": samples, + "comparison": _comparison(by_variant["direct"], by_variant["code_mode"]), + } + ) + cases_out.append({"case_id": case["id"], "pairs": pair_outputs}) + + statuses = [ + sample["status"] + for case in cases_out + for pair in case["pairs"] + for sample in pair["samples"] + ] + return { + "schema_version": 1, + "kind": "agent_loop_benchmark", + "base_revision": resolved, + "pair_count": pairs, + "case_ids": case_ids, + "sample_count": len(statuses), + "status_counts": {status: statuses.count(status) for status in sorted(STATUS_VALUES)}, + "cases": cases_out, + "notes": [ + "each sample ran in a fresh detached Git worktree", + "pair order alternates direct/code_mode then code_mode/direct", + "driver-reported unsupported, partial, and failed samples remain in the result", + "comparisons reuse agent_loop_report exact base/case fingerprint/correctness gates", + "outer-call metrics remain proxies; exact model round trips stay unavailable without Host turn evidence", + ], + } + finally: + if temp_owner is not None: + temp_owner.cleanup() + + +def _human_summary(result: dict[str, Any]) -> str: + counts = result["status_counts"] + lines = [ + f"Agent Loop benchmark {result['base_revision'][:8]}", + f"cases={len(result['case_ids'])} pairs={result['pair_count']} samples={result['sample_count']}", + "status " + " ".join(f"{key}={counts[key]}" for key in sorted(counts)), + ] + for case in result["cases"]: + orders = ["/".join(pair["order"]) for pair in case["pairs"]] + comparable = sum( + 1 + for pair in case["pairs"] + if isinstance(pair.get("comparison"), dict) + and (pair["comparison"].get("throughput_compatibility") or {}).get("comparable") is True + ) + lines.append(f"{case['case_id']}: orders={','.join(orders)} comparable_pairs={comparable}/{len(orders)}") + return "\n".join(lines) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path(".")) + parser.add_argument("--base-revision", required=True) + parser.add_argument("--driver", required=True, help="Host driver command, parsed with shlex; no shell is invoked") + parser.add_argument("--case-id", action="append", dest="case_ids") + parser.add_argument("--pairs", type=int, default=2) + parser.add_argument("--output", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + driver_argv = shlex.split(args.driver) + if not driver_argv: + raise BenchmarkError("--driver must name an executable") + result = run_benchmark( + repo=args.repo.resolve(), + base_revision=args.base_revision, + driver_argv=driver_argv, + case_ids=args.case_ids or list(DEFAULT_CASES), + pairs=args.pairs, + ) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(_stable_json(result), encoding="utf-8") + print(_human_summary(result)) + if not args.output: + print(_stable_json(result), end="") + return 0 if result["status_counts"]["fail"] == 0 else 1 + except BenchmarkError as exc: + print(f"agent_loop_benchmark: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/agent_loop_cases.json b/scripts/agent_loop_cases.json index f59b67b8a..2ed688610 100644 --- a/scripts/agent_loop_cases.json +++ b/scripts/agent_loop_cases.json @@ -57,6 +57,51 @@ }, "code_mode_surface": "guarded_edit" }, + { + "id": "guarded_multi_file_edit", + "title": "Search, guarded multi-file edit, and post-edit inspection", + "target": "webcodex_repository", + "prompt": "Search for BENCH_TARGET_ALPHA and BENCH_TARGET_BETA, read only the two matched fixture files, then make exactly one guarded edit_project_files request containing both file changes: BENCH_TARGET_ALPHA=before to BENCH_TARGET_ALPHA=after and BENCH_TARGET_BETA=before to BENCH_TARGET_BETA=after using the read revisions you just observed. Post-read both files, inspect the resulting diff, and report the two changed paths. Preserve ALPHA_UNRELATED_SENTINEL=keep and BETA_UNRELATED_SENTINEL=keep exactly. Do not edit any other file.", + "correctness": { + "changed_files": [ + "tests/fixtures/agent-loop-multi-file/alpha.txt", + "tests/fixtures/agent-loop-multi-file/beta.txt" + ], + "one_guarded_edit_request_required": true, + "post_edit_read_required": true, + "fixture_oracle": { + "changed_files": [ + "tests/fixtures/agent-loop-multi-file/alpha.txt", + "tests/fixtures/agent-loop-multi-file/beta.txt" + ], + "files": { + "tests/fixtures/agent-loop-multi-file/alpha.txt": { + "required_text": [ + "BENCH_TARGET_ALPHA=after", + "ALPHA_UNRELATED_SENTINEL=keep" + ], + "forbidden_text": [ + "BENCH_TARGET_ALPHA=before" + ] + }, + "tests/fixtures/agent-loop-multi-file/beta.txt": { + "required_text": [ + "BENCH_TARGET_BETA=after", + "BETA_UNRELATED_SENTINEL=keep" + ], + "forbidden_text": [ + "BENCH_TARGET_BETA=before" + ] + } + } + } + }, + "validation": { + "required": false, + "expectation": "Fixture-side oracle checks the exact two-file diff and preserved unrelated sentinels; no code validation is required." + }, + "code_mode_surface": "guarded_edit" + }, { "id": "failed_validation_recovery", "title": "Failed validation recovery", diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py new file mode 100644 index 000000000..ca6ab0750 --- /dev/null +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from scripts import agent_loop_benchmark as benchmark +from scripts import agent_loop_report as report + + +class AgentLoopBenchmarkTests(unittest.TestCase): + def test_pair_order_alternates(self) -> None: + self.assertEqual(benchmark._pair_order(0), ("direct", "code_mode")) + self.assertEqual(benchmark._pair_order(1), ("code_mode", "direct")) + self.assertEqual(benchmark._pair_order(2), ("direct", "code_mode")) + + def test_fixture_oracle_checks_both_files_and_exact_diff(self) -> None: + manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + case = report._case_by_id(manifest, "guarded_multi_file_edit") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(["git", "init", "-b", "main"], cwd=root, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=root, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=root, check=True) + fixture = root / "tests/fixtures/agent-loop-multi-file" + fixture.mkdir(parents=True) + (fixture / "alpha.txt").write_text( + "BENCH_TARGET_ALPHA=before\nALPHA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=before\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-m", "fixture"], cwd=root, check=True, stdout=subprocess.DEVNULL) + (fixture / "alpha.txt").write_text( + "BENCH_TARGET_ALPHA=after\nALPHA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=after\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + result = benchmark._fixture_oracle(case, root) + self.assertTrue(result["available"]) + self.assertTrue(result["passed"]) + + subprocess.run(["git", "add", "."], cwd=root, check=True) + staged = benchmark._fixture_oracle(case, root) + self.assertTrue(staged["passed"]) + + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=after\nBETA_UNRELATED_SENTINEL=lost\n", + encoding="utf-8", + ) + result = benchmark._fixture_oracle(case, root) + self.assertFalse(result["passed"]) + + def test_unsupported_driver_samples_are_retained_and_worktrees_cleaned(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + + driver = root / "driver.py" + driver.write_text( + "import json, os\n" + "from pathlib import Path\n" + "Path(os.environ['WEBCODEX_BENCH_DRIVER_RESULT']).write_text(" + "json.dumps({'status':'unsupported','message':'fixture driver'}), encoding='utf-8')\n", + encoding="utf-8", + ) + result = benchmark.run_benchmark( + repo=repo, + base_revision=base, + driver_argv=[sys.executable, str(driver)], + case_ids=["readonly_review"], + pairs=2, + ) + self.assertEqual(result["sample_count"], 4) + self.assertEqual(result["status_counts"]["unsupported"], 4) + self.assertEqual(result["cases"][0]["pairs"][0]["order"], ["direct", "code_mode"]) + self.assertEqual(result["cases"][0]["pairs"][1]["order"], ["code_mode", "direct"]) + self.assertIsNone(result["cases"][0]["pairs"][0]["comparison"]) + + worktrees = subprocess.check_output(["git", "worktree", "list", "--porcelain"], cwd=repo, text=True) + self.assertEqual(worktrees.count("worktree "), 1) + + def test_base_revision_must_match_checkout_head(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("one\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "one"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + old = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + (repo / "README.md").write_text("two\n", encoding="utf-8") + subprocess.run(["git", "commit", "-am", "two"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + with self.assertRaisesRegex(benchmark.BenchmarkError, "current checkout HEAD"): + benchmark.run_benchmark( + repo=repo, + base_revision=old, + driver_argv=[sys.executable, "-c", "raise SystemExit(0)"], + case_ids=["readonly_review"], + pairs=1, + ) + + def test_cli_prints_human_summary_and_json_output(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + driver = root / "driver.py" + driver.write_text( + "import json, os\n" + "from pathlib import Path\n" + "Path(os.environ['WEBCODEX_BENCH_DRIVER_RESULT']).write_text(" + "json.dumps({'status':'unsupported'}), encoding='utf-8')\n", + encoding="utf-8", + ) + output = root / "result.json" + completed = subprocess.run( + [ + sys.executable, + str(Path(benchmark.__file__)), + "--repo", + str(repo), + "--base-revision", + base, + "--driver", + f"{sys.executable} {driver}", + "--case-id", + "readonly_review", + "--pairs", + "1", + "--output", + str(output), + ], + cwd=Path(__file__).resolve().parents[2], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertIn("Agent Loop benchmark", completed.stdout) + value = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(value["status_counts"]["unsupported"], 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_agent_loop_report.py b/scripts/tests/test_agent_loop_report.py index 1dc4c22c4..4f14e0005 100644 --- a/scripts/tests/test_agent_loop_report.py +++ b/scripts/tests/test_agent_loop_report.py @@ -1163,9 +1163,9 @@ def test_manifest_requires_fields_and_stable_serialization(self) -> None: serialized_once = report._stable_json(manifest) serialized_twice = report._stable_json(json.loads(serialized_once)) self.assertEqual(serialized_once, serialized_twice) - self.assertEqual(len(manifest["cases"]), 16) + self.assertEqual(len(manifest["cases"]), 17) typed_cases = [case for case in manifest["cases"] if "code_mode_surface" in case] - self.assertEqual(len(typed_cases), 16) + self.assertEqual(len(typed_cases), 17) self.assertEqual( {case["code_mode_surface"] for case in typed_cases}, {"read_only", "validation", "guarded_edit"}, diff --git a/tests/fixtures/agent-loop-multi-file/alpha.txt b/tests/fixtures/agent-loop-multi-file/alpha.txt new file mode 100644 index 000000000..92065b285 --- /dev/null +++ b/tests/fixtures/agent-loop-multi-file/alpha.txt @@ -0,0 +1,2 @@ +BENCH_TARGET_ALPHA=before +ALPHA_UNRELATED_SENTINEL=keep diff --git a/tests/fixtures/agent-loop-multi-file/beta.txt b/tests/fixtures/agent-loop-multi-file/beta.txt new file mode 100644 index 000000000..99a4954e4 --- /dev/null +++ b/tests/fixtures/agent-loop-multi-file/beta.txt @@ -0,0 +1,2 @@ +BENCH_TARGET_BETA=before +BETA_UNRELATED_SENTINEL=keep From 0aa539d61ed526164e22884cfb754fe22f9504f3 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:51:16 +0800 Subject: [PATCH 02/12] test(agent-loop): require persisted workflow evidence --- docs/experiments/agent-loop-baseline.md | 3 +- scripts/agent_loop_benchmark.py | 38 +++++++++++++++++++ scripts/tests/test_agent_loop_benchmark.py | 43 ++++++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/docs/experiments/agent-loop-baseline.md b/docs/experiments/agent-loop-baseline.md index d2c66a2ae..740f35fdb 100644 --- a/docs/experiments/agent-loop-baseline.md +++ b/docs/experiments/agent-loop-baseline.md @@ -272,7 +272,7 @@ only reproducibility mechanics: - alternating pair order (`direct/code_mode`, then `code_mode/direct`) to expose one-sided process/cache ordering effects; - bounded annotation and the existing report/correctness compatibility gates; -- fixture-side disk/diff oracles where the manifest provides one; +- fixture-side disk/diff oracles where the manifest provides one, plus persisted call-count evidence for the multi-file case's single guarded edit and search/read phases; - explicit `pass`, `fail`, `partial`, and `unsupported` samples. None are silently filtered from the aggregate; - cleanup of only the worktrees/temp state created by the orchestrator. @@ -303,6 +303,7 @@ The default lane is exactly `readonly_review`, `guarded_multi_file_edit`, and driver observes a genuine pending validation and follows the same Job to terminal; a synchronous completion or substitute pending process must be reported `partial`/`unsupported`, not upgraded to success. +The guarded multi-file case is likewise not a pass from final bytes alone: persisted Direct canonical-call or Code Mode composition evidence must show exactly one `edit_project_files` call, at least one admitted search call, and at least two `read_files` calls (pre-edit and post-edit phases). Current composition does not prove nested child ordering, so the benchmark does not claim stronger sequencing evidence than is persisted. This command coordinates real captures; it is not itself evidence that a model made fewer inference turns. Deterministic runtime/script acceptance and fixture oracles diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index d7c1a7c4a..f96f4174a 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -57,6 +57,7 @@ def _run_driver(argv: list[str], *, cwd: Path, env: dict[str, str]) -> subproces argv, cwd=cwd, env=env, + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, @@ -243,6 +244,35 @@ def _summary_for_receipt( ) +def _tool_count(summary: dict[str, Any], variant: str, tool_name: str) -> int | None: + if variant == "direct": + value = report._get_path(summary, f"canonical_calls.by_name.{tool_name}") + return value if isinstance(value, int) else 0 + if variant == "code_mode": + available = report._get_path(summary, "availability.code_mode_composition.available") + if available is not True: + return None + value = report._get_path(summary, f"composition.nested_tool_counts.{tool_name}") + return value if isinstance(value, int) else 0 + return None + + +def _guarded_multi_file_contract_proven(summary: dict[str, Any], variant: str) -> bool: + edit_count = _tool_count(summary, variant, "edit_project_files") + read_count = _tool_count(summary, variant, "read_files") + search_counts = [ + count + for name in ("search_project_texts", "search_and_read") + if (count := _tool_count(summary, variant, name)) is not None + ] + return ( + edit_count == 1 + and read_count is not None + and read_count >= 2 + and sum(search_counts) >= 1 + ) + + def _run_sample( repo: Path, root: Path, @@ -335,6 +365,14 @@ def _run_sample( and report._get_path(summary, "job_convergence.pending_handoff_count") in (None, 0) ): effective_status = "partial" + + if ( + case["id"] == "guarded_multi_file_edit" + and summary is not None + and effective_status == "pass" + and not _guarded_multi_file_contract_proven(summary, variant) + ): + effective_status = "partial" return { "case_id": case["id"], "variant": variant, diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index ca6ab0750..cc3aa7a67 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -17,6 +17,49 @@ def test_pair_order_alternates(self) -> None: self.assertEqual(benchmark._pair_order(1), ("code_mode", "direct")) self.assertEqual(benchmark._pair_order(2), ("direct", "code_mode")) + def test_driver_stdin_is_closed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "stdin.txt" + benchmark._run_driver( + [ + sys.executable, + "-c", + "from pathlib import Path; import sys; Path(sys.argv[1]).write_text(sys.stdin.read())", + str(output), + ], + cwd=Path(tmp), + env={}, + ) + self.assertEqual(output.read_text(encoding="utf-8"), "") + + def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(self) -> None: + direct = { + "canonical_calls": { + "by_name": { + "edit_project_files": 1, + "search_project_texts": 1, + "read_files": 2, + } + } + } + self.assertTrue(benchmark._guarded_multi_file_contract_proven(direct, "direct")) + direct["canonical_calls"]["by_name"]["edit_project_files"] = 2 + self.assertFalse(benchmark._guarded_multi_file_contract_proven(direct, "direct")) + + code_mode = { + "availability": {"code_mode_composition": {"available": True}}, + "composition": { + "nested_tool_counts": { + "edit_project_files": 1, + "search_project_texts": 1, + "read_files": 2, + } + }, + } + self.assertTrue(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) + code_mode["composition"]["nested_tool_counts"]["read_files"] = 1 + self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) + def test_fixture_oracle_checks_both_files_and_exact_diff(self) -> None: manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) case = report._case_by_id(manifest, "guarded_multi_file_edit") From 08b93ed1007420fe773231f5154387c14402539e Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:58:00 +0800 Subject: [PATCH 03/12] test(agent-loop): close benchmark failure paths --- scripts/agent_loop_benchmark.py | 66 +++++++++++++++------ scripts/agent_loop_cases.json | 16 +---- scripts/tests/test_agent_loop_benchmark.py | 68 ++++++++++++++++++++++ 3 files changed, 119 insertions(+), 31 deletions(-) diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index f96f4174a..2c7838251 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -30,6 +30,7 @@ DEFAULT_CASES = ("readonly_review", "guarded_multi_file_edit", "long_validation_handoff") VARIANTS = ("direct", "code_mode") MAX_DRIVER_RECEIPT_BYTES = 64 * 1024 +MAX_DRIVER_SECS = 30 * 60 class BenchmarkError(ValueError): @@ -60,8 +61,11 @@ def _run_driver(argv: list[str], *, cwd: Path, env: dict[str, str]) -> subproces stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=MAX_DRIVER_SECS, check=False, ) + except subprocess.TimeoutExpired as exc: + raise BenchmarkError("benchmark Host driver exceeded the 30 minute limit") from exc except OSError as exc: raise BenchmarkError("could not start benchmark Host driver") from exc @@ -110,7 +114,6 @@ def _worktree_remove(repo: Path, path: Path) -> None: _run(["git", "worktree", "remove", "--force", str(path)], cwd=repo) if path.exists(): shutil.rmtree(path, ignore_errors=True) - _run(["git", "worktree", "prune"], cwd=repo) def _workspace_status(workspace: Path) -> str: @@ -133,19 +136,27 @@ def _fixture_oracle(case: dict[str, Any], workspace: Path) -> dict[str, Any]: except OSError: checks.append({"path": rel_path, "passed": False, "reason": "file unavailable"}) continue - required = expected.get("required_text") or [] - forbidden = expected.get("forbidden_text") or [] - passed = all(item in text for item in required) and all(item not in text for item in forbidden) + expected_text = expected.get("expected_text") + if isinstance(expected_text, str): + passed = text == expected_text + else: + required = expected.get("required_text") or [] + forbidden = expected.get("forbidden_text") or [] + passed = all(item in text for item in required) and all(item not in text for item in forbidden) checks.append({"path": rel_path, "passed": passed, "reason": None if passed else "text oracle mismatch"}) expected_changed = sorted(spec.get("changed_files") or []) if expected_changed: - completed = _run(["git", "diff", "--name-only", "HEAD", "--"], cwd=workspace) - changed = sorted(line for line in completed.stdout.splitlines() if line) + tracked = _run(["git", "diff", "--name-only", "HEAD", "--"], cwd=workspace) + untracked = _run(["git", "ls-files", "--others", "--exclude-standard"], cwd=workspace) + changed = sorted( + set(line for line in tracked.stdout.splitlines() if line) + | set(line for line in untracked.stdout.splitlines() if line) + ) checks.append( { "kind": "changed_files", - "passed": completed.returncode == 0 and changed == expected_changed, + "passed": tracked.returncode == 0 and untracked.returncode == 0 and changed == expected_changed, "expected": expected_changed, "actual": changed, } @@ -165,12 +176,14 @@ def _load_driver_receipt(path: Path) -> dict[str, Any]: value = json.loads(path.read_text(encoding="utf-8")) except OSError as exc: raise BenchmarkError("driver did not create its result receipt") from exc + except UnicodeDecodeError as exc: + raise BenchmarkError("driver result must be UTF-8 JSON") from exc except json.JSONDecodeError as exc: raise BenchmarkError(f"driver result is not valid JSON: {exc}") from exc if not isinstance(value, dict): raise BenchmarkError("driver result must be a JSON object") status = value.get("status") - if status not in STATUS_VALUES: + if not isinstance(status, str) or status not in STATUS_VALUES: raise BenchmarkError("driver result status must be pass, fail, partial, or unsupported") return value @@ -221,6 +234,7 @@ def _summary_for_receipt( base_revision: str, receipt: dict[str, Any], annotation_path: Path, + case_manifest: Path, ) -> dict[str, Any] | None: if receipt["status"] == "unsupported": return None @@ -235,7 +249,7 @@ def _summary_for_receipt( trace_root=Path(trace_root) if isinstance(trace_root, str) and trace_root else None, audit_db=Path(audit_db), workflow_session_id=workflow_session_id, - case_manifest=report.DEFAULT_CASE_MANIFEST, + case_manifest=case_manifest, case_id=case["id"], variant=variant, surface=surface, @@ -283,15 +297,19 @@ def _run_sample( pair_index: int, ordinal: int, base_revision: str, + case_manifest: Path, ) -> dict[str, Any]: label = f"{case['id']}-p{pair_index + 1}-{ordinal + 1}-{variant}" workspace = root / "worktrees" / label receipt_path = root / "receipts" / f"{label}.json" annotation_path = root / "annotations" / f"{label}.json" - surface = _surface(case, variant) - _worktree_add(repo, workspace, base_revision) + surface: str | None = None + worktree_added = False started = time.time_ns() // 1_000_000 try: + surface = _surface(case, variant) + _worktree_add(repo, workspace, base_revision) + worktree_added = True if _workspace_status(workspace): raise BenchmarkError("fresh benchmark worktree is not clean") env = dict(os.environ) @@ -317,6 +335,8 @@ def _run_sample( "status": "fail", "driver_exit_code": completed.returncode, "reason_code": "driver_receipt_missing", + "fixture_oracle": {"available": False, "passed": None, "checks": [], "reason": "driver receipt missing"}, + "summary": None, } receipt = _load_driver_receipt(receipt_path) @@ -339,11 +359,18 @@ def _run_sample( base_revision=base_revision, receipt=receipt, annotation_path=annotation_path, + case_manifest=case_manifest, ) effective_status = receipt["status"] - if completed.returncode != 0 and effective_status == "pass": + if completed.returncode != 0 and effective_status in ("pass", "partial"): + effective_status = "fail" + if oracle["available"] and oracle["passed"] is not True and effective_status in ("pass", "partial"): effective_status = "fail" - if oracle["available"] and oracle["passed"] is not True and effective_status == "pass": + if ( + (case.get("correctness") or {}).get("workspace_must_remain_clean") is True + and _workspace_status(workspace) + and effective_status in ("pass", "partial") + ): effective_status = "fail" if summary is not None and effective_status == "pass": correctness = summary.get("correctness") or {} @@ -394,11 +421,14 @@ def _run_sample( "summary": None, } finally: - _worktree_remove(repo, workspace) + if worktree_added: + _worktree_remove(repo, workspace) def _comparison(direct: dict[str, Any], code_mode: dict[str, Any]) -> dict[str, Any] | None: - if direct["summary"] is None or code_mode["summary"] is None: + if direct.get("status") != "pass" or code_mode.get("status") != "pass": + return None + if direct.get("summary") is None or code_mode.get("summary") is None: return None return report.compare_reports(direct["summary"], code_mode["summary"]) @@ -417,13 +447,14 @@ def run_benchmark( ) -> dict[str, Any]: if pairs < 1: raise BenchmarkError("pairs must be at least 1") - manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) - selected = [_case(manifest, case_id) for case_id in case_ids] resolved = _require_exact_revision(repo, base_revision) if _head_revision(repo) != resolved: raise BenchmarkError( "base revision must equal the current checkout HEAD; check out the target commit before benchmarking" ) + case_manifest = repo / "scripts" / "agent_loop_cases.json" + manifest = report.load_case_manifest(case_manifest) + selected = [_case(manifest, case_id) for case_id in case_ids] temp_owner = tempfile.TemporaryDirectory(prefix="webcodex-agent-loop-bench-") root = Path(temp_owner.name) for name in ("worktrees", "receipts", "annotations"): @@ -447,6 +478,7 @@ def run_benchmark( pair_index=pair_index, ordinal=ordinal, base_revision=resolved, + case_manifest=case_manifest, ) ) by_variant = {sample["variant"]: sample for sample in samples} diff --git a/scripts/agent_loop_cases.json b/scripts/agent_loop_cases.json index 2ed688610..181356075 100644 --- a/scripts/agent_loop_cases.json +++ b/scripts/agent_loop_cases.json @@ -76,22 +76,10 @@ ], "files": { "tests/fixtures/agent-loop-multi-file/alpha.txt": { - "required_text": [ - "BENCH_TARGET_ALPHA=after", - "ALPHA_UNRELATED_SENTINEL=keep" - ], - "forbidden_text": [ - "BENCH_TARGET_ALPHA=before" - ] + "expected_text": "BENCH_TARGET_ALPHA=after\nALPHA_UNRELATED_SENTINEL=keep\n" }, "tests/fixtures/agent-loop-multi-file/beta.txt": { - "required_text": [ - "BENCH_TARGET_BETA=after", - "BETA_UNRELATED_SENTINEL=keep" - ], - "forbidden_text": [ - "BENCH_TARGET_BETA=before" - ] + "expected_text": "BENCH_TARGET_BETA=after\nBETA_UNRELATED_SENTINEL=keep\n" } } } diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index cc3aa7a67..50646ca41 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -12,6 +12,14 @@ class AgentLoopBenchmarkTests(unittest.TestCase): + def _install_manifest(self, repo: Path) -> None: + target = repo / "scripts/agent_loop_cases.json" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + report.DEFAULT_CASE_MANIFEST.read_text(encoding="utf-8"), + encoding="utf-8", + ) + def test_pair_order_alternates(self) -> None: self.assertEqual(benchmark._pair_order(0), ("direct", "code_mode")) self.assertEqual(benchmark._pair_order(1), ("code_mode", "direct")) @@ -103,6 +111,64 @@ def test_fixture_oracle_checks_both_files_and_exact_diff(self) -> None: result = benchmark._fixture_oracle(case, root) self.assertFalse(result["passed"]) + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=after\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + (root / "unexpected.txt").write_text("unexpected\n", encoding="utf-8") + result = benchmark._fixture_oracle(case, root) + self.assertFalse(result["passed"]) + changed_check = next(check for check in result["checks"] if check.get("kind") == "changed_files") + self.assertIn("unexpected.txt", changed_check["actual"]) + + def test_malformed_receipt_status_is_a_contract_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + receipt = Path(tmp) / "receipt.json" + receipt.write_text('{"status": []}', encoding="utf-8") + with self.assertRaisesRegex(benchmark.BenchmarkError, "status must be"): + benchmark._load_driver_receipt(receipt) + + def test_comparison_requires_two_effective_pass_samples(self) -> None: + summary = {"benchmark": {}} + self.assertIsNone( + benchmark._comparison( + {"status": "partial", "summary": summary}, + {"status": "pass", "summary": summary}, + ) + ) + self.assertIsNone( + benchmark._comparison( + {"status": "pass", "summary": None}, + {"status": "pass", "summary": summary}, + ) + ) + + def test_missing_receipts_are_retained_as_failed_samples(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + result = benchmark.run_benchmark( + repo=repo, + base_revision=base, + driver_argv=[sys.executable, "-c", "pass"], + case_ids=["readonly_review"], + pairs=1, + ) + self.assertEqual(result["status_counts"]["fail"], 2) + for sample in result["cases"][0]["pairs"][0]["samples"]: + self.assertEqual(sample["reason_code"], "driver_receipt_missing") + self.assertIsNone(sample["summary"]) + self.assertIsNone(result["cases"][0]["pairs"][0]["comparison"]) + def test_unsupported_driver_samples_are_retained_and_worktrees_cleaned(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -112,6 +178,7 @@ def test_unsupported_driver_samples_are_retained_and_worktrees_cleaned(self) -> subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) subprocess.run(["git", "add", "."], cwd=repo, check=True) subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() @@ -172,6 +239,7 @@ def test_cli_prints_human_summary_and_json_output(self) -> None: subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) subprocess.run(["git", "add", "."], cwd=repo, check=True) subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() From 7e2dd99a96ba3c40a4c0f6b704a5272f820919da Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:06:03 +0800 Subject: [PATCH 04/12] test(agent-loop): fail closed on invalid fixture text --- scripts/agent_loop_benchmark.py | 3 +++ scripts/tests/test_agent_loop_benchmark.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index 2c7838251..295dc679a 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -136,6 +136,9 @@ def _fixture_oracle(case: dict[str, Any], workspace: Path) -> dict[str, Any]: except OSError: checks.append({"path": rel_path, "passed": False, "reason": "file unavailable"}) continue + except UnicodeDecodeError: + checks.append({"path": rel_path, "passed": False, "reason": "file is not UTF-8"}) + continue expected_text = expected.get("expected_text") if isinstance(expected_text, str): passed = text == expected_text diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index 50646ca41..494d60035 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -121,6 +121,13 @@ def test_fixture_oracle_checks_both_files_and_exact_diff(self) -> None: changed_check = next(check for check in result["checks"] if check.get("kind") == "changed_files") self.assertIn("unexpected.txt", changed_check["actual"]) + (root / "unexpected.txt").unlink() + (fixture / "beta.txt").write_bytes(b"\xff\xfe") + result = benchmark._fixture_oracle(case, root) + self.assertFalse(result["passed"]) + beta_check = next(check for check in result["checks"] if check.get("path", "").endswith("beta.txt")) + self.assertEqual(beta_check["reason"], "file is not UTF-8") + def test_malformed_receipt_status_is_a_contract_error(self) -> None: with tempfile.TemporaryDirectory() as tmp: receipt = Path(tmp) / "receipt.json" From e3a901eb2d3fffa4cda5413d6ace288387867662 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:37:49 +0800 Subject: [PATCH 05/12] test(agent-loop): preserve unrelated worktrees --- scripts/agent_loop_benchmark.py | 5 ++- scripts/tests/test_agent_loop_benchmark.py | 43 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index 295dc679a..2ed87e2a9 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -133,6 +133,9 @@ def _fixture_oracle(case: dict[str, Any], workspace: Path) -> dict[str, Any]: target = workspace / rel_path try: text = target.read_text(encoding="utf-8") + except UnicodeDecodeError: + checks.append({"path": rel_path, "passed": False, "reason": "file is not UTF-8"}) + continue except OSError: checks.append({"path": rel_path, "passed": False, "reason": "file unavailable"}) continue @@ -573,7 +576,7 @@ def main(argv: list[str] | None = None) -> int: if not args.output: print(_stable_json(result), end="") return 0 if result["status_counts"]["fail"] == 0 else 1 - except BenchmarkError as exc: + except (BenchmarkError, report.ReportError) as exc: print(f"agent_loop_benchmark: {exc}", file=sys.stderr) return 2 diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index 494d60035..36ca56f8a 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -214,6 +214,48 @@ def test_unsupported_driver_samples_are_retained_and_worktrees_cleaned(self) -> worktrees = subprocess.check_output(["git", "worktree", "list", "--porcelain"], cwd=repo, text=True) self.assertEqual(worktrees.count("worktree "), 1) + def test_cleanup_does_not_prune_unrelated_worktree(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + + unrelated = root / "unrelated" + subprocess.run(["git", "worktree", "add", "--detach", str(unrelated), base], cwd=repo, check=True, stdout=subprocess.DEVNULL) + driver = root / "driver.py" + driver.write_text( + "import json, os\n" + "from pathlib import Path\n" + "Path(os.environ['WEBCODEX_BENCH_DRIVER_RESULT']).write_text(" + "json.dumps({'status':'unsupported'}), encoding='utf-8')\n", + encoding="utf-8", + ) + benchmark.run_benchmark( + repo=repo, + base_revision=base, + driver_argv=[sys.executable, str(driver)], + case_ids=["readonly_review"], + pairs=1, + ) + status = subprocess.run( + ["git", "status", "--porcelain"], + cwd=unrelated, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + self.assertEqual(status.returncode, 0, status.stderr) + subprocess.run(["git", "worktree", "remove", "--force", str(unrelated)], cwd=repo, check=True) + def test_base_revision_must_match_checkout_head(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -223,6 +265,7 @@ def test_base_revision_must_match_checkout_head(self) -> None: subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) (repo / "README.md").write_text("one\n", encoding="utf-8") + self._install_manifest(repo) subprocess.run(["git", "add", "."], cwd=repo, check=True) subprocess.run(["git", "commit", "-m", "one"], cwd=repo, check=True, stdout=subprocess.DEVNULL) old = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() From 181fb58955cd2f2314e5e14f13eac7f74482aed1 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:44:25 +0800 Subject: [PATCH 06/12] test(agent-loop): pin manifest to benchmark base --- scripts/agent_loop_benchmark.py | 17 ++++++++-- scripts/tests/test_agent_loop_benchmark.py | 39 ++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index 2ed87e2a9..f2ce7625a 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -90,6 +90,16 @@ def _head_revision(repo: Path) -> str: return head +def _read_manifest_at_revision(repo: Path, revision: str) -> str: + completed = _run( + ["git", "show", f"{revision}:scripts/agent_loop_cases.json"], + cwd=repo, + ) + if completed.returncode != 0: + raise BenchmarkError("could not read benchmark case manifest at base revision") + return completed.stdout + + def _case(manifest: dict[str, Any], case_id: str) -> dict[str, Any]: return report._case_by_id(manifest, case_id) @@ -458,13 +468,14 @@ def run_benchmark( raise BenchmarkError( "base revision must equal the current checkout HEAD; check out the target commit before benchmarking" ) - case_manifest = repo / "scripts" / "agent_loop_cases.json" - manifest = report.load_case_manifest(case_manifest) - selected = [_case(manifest, case_id) for case_id in case_ids] temp_owner = tempfile.TemporaryDirectory(prefix="webcodex-agent-loop-bench-") root = Path(temp_owner.name) for name in ("worktrees", "receipts", "annotations"): (root / name).mkdir(parents=True, exist_ok=True) + case_manifest = root / "agent_loop_cases.json" + case_manifest.write_text(_read_manifest_at_revision(repo, resolved), encoding="utf-8") + manifest = report.load_case_manifest(case_manifest) + selected = [_case(manifest, case_id) for case_id in case_ids] cases_out: list[dict[str, Any]] = [] try: diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index 36ca56f8a..b7892b2b6 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -256,6 +256,45 @@ def test_cleanup_does_not_prune_unrelated_worktree(self) -> None: self.assertEqual(status.returncode, 0, status.stderr) subprocess.run(["git", "worktree", "remove", "--force", str(unrelated)], cwd=repo, check=True) + def test_case_manifest_is_pinned_to_base_revision(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + + manifest_path = repo / "scripts/agent_loop_cases.json" + dirty = json.loads(manifest_path.read_text(encoding="utf-8")) + dirty["cases"][0]["prompt"] = "dirty prompt must not be observed" + manifest_path.write_text(json.dumps(dirty), encoding="utf-8") + + observed = root / "observed.txt" + driver = root / "driver.py" + driver.write_text( + "import os, sys\n" + "from pathlib import Path\n" + "Path(sys.argv[1]).write_text(os.environ['WEBCODEX_BENCH_PROMPT'], encoding='utf-8')\n" + "Path(os.environ['WEBCODEX_BENCH_DRIVER_RESULT']).write_text('{\"status\":\"unsupported\"}', encoding='utf-8')\n", + encoding="utf-8", + ) + benchmark.run_benchmark( + repo=repo, + base_revision=base, + driver_argv=[sys.executable, str(driver), str(observed)], + case_ids=["readonly_review"], + pairs=1, + ) + pinned = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + expected = report._case_by_id(pinned, "readonly_review")["prompt"] + self.assertEqual(observed.read_text(encoding="utf-8"), expected) + def test_base_revision_must_match_checkout_head(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From 5bcb54582ddd5bc2d18ad245d82fe746e483640f Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:46:12 +0800 Subject: [PATCH 07/12] test(agent-loop): bound fixture oracle reads --- scripts/agent_loop_benchmark.py | 8 +++++++- scripts/tests/test_agent_loop_benchmark.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index f2ce7625a..97b75651e 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -30,6 +30,7 @@ DEFAULT_CASES = ("readonly_review", "guarded_multi_file_edit", "long_validation_handoff") VARIANTS = ("direct", "code_mode") MAX_DRIVER_RECEIPT_BYTES = 64 * 1024 +MAX_FIXTURE_ORACLE_BYTES = 1024 * 1024 MAX_DRIVER_SECS = 30 * 60 @@ -142,7 +143,12 @@ def _fixture_oracle(case: dict[str, Any], workspace: Path) -> dict[str, Any]: for rel_path, expected in sorted((spec.get("files") or {}).items()): target = workspace / rel_path try: - text = target.read_text(encoding="utf-8") + with target.open("rb") as handle: + raw = handle.read(MAX_FIXTURE_ORACLE_BYTES + 1) + if len(raw) > MAX_FIXTURE_ORACLE_BYTES: + checks.append({"path": rel_path, "passed": False, "reason": "file exceeds oracle byte limit"}) + continue + text = raw.decode("utf-8") except UnicodeDecodeError: checks.append({"path": rel_path, "passed": False, "reason": "file is not UTF-8"}) continue diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index b7892b2b6..e63873a4d 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -128,6 +128,12 @@ def test_fixture_oracle_checks_both_files_and_exact_diff(self) -> None: beta_check = next(check for check in result["checks"] if check.get("path", "").endswith("beta.txt")) self.assertEqual(beta_check["reason"], "file is not UTF-8") + (fixture / "beta.txt").write_bytes(b"x" * (benchmark.MAX_FIXTURE_ORACLE_BYTES + 1)) + result = benchmark._fixture_oracle(case, root) + self.assertFalse(result["passed"]) + beta_check = next(check for check in result["checks"] if check.get("path", "").endswith("beta.txt")) + self.assertEqual(beta_check["reason"], "file exceeds oracle byte limit") + def test_malformed_receipt_status_is_a_contract_error(self) -> None: with tempfile.TemporaryDirectory() as tmp: receipt = Path(tmp) / "receipt.json" From d3635cadfc5ba9e8fa60bdc392b14fc5b6318cb3 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:57:19 +0800 Subject: [PATCH 08/12] test(agent-loop): close benchmark evidence boundaries --- docs/experiments/agent-loop-baseline.md | 10 +- scripts/agent_loop_benchmark.py | 118 +++++++-- scripts/tests/test_agent_loop_benchmark.py | 279 +++++++++++++++++++++ 3 files changed, 384 insertions(+), 23 deletions(-) diff --git a/docs/experiments/agent-loop-baseline.md b/docs/experiments/agent-loop-baseline.md index 740f35fdb..c43189ca7 100644 --- a/docs/experiments/agent-loop-baseline.md +++ b/docs/experiments/agent-loop-baseline.md @@ -294,9 +294,13 @@ For each invocation the driver receives `WEBCODEX_BENCH_WORKSPACE`, `WEBCODEX_BENCH_DRIVER_RESULT`. It writes one bounded JSON receipt to the last path. The receipt is capped at 64 KiB; driver stdout/stderr is not retained by the orchestrator. `status` is required and is one of `pass/fail/partial/unsupported`. Non-unsupported samples also provide the authoritative ActionAudit database path -and Workflow Session id; optional trace root, repair-turn annotation, task timing, -and correctness verdicts use the existing reporter semantics. The durable benchmark -result does not persist runtime Project, Window, tunnel, or Host-private ids. +and Workflow Session id; optional trace root, repair-turn annotation, and task timing +use the existing reporter semantics. Correctness verdicts are optional only when an +independent fixture oracle can establish the task result. For non-oracle cases, a +driver-reported pass needs `correctness.task_verdict="pass"`; validation-required +cases also need the corresponding passing validation verdict, otherwise the sample +remains partial. The durable benchmark result does not persist runtime Project, +Window, tunnel, or Host-private ids. The default lane is exactly `readonly_review`, `guarded_multi_file_edit`, and `long_validation_handoff`, matching #822. The handoff case is valid only when the diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index 97b75651e..4ae82324a 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -12,6 +12,7 @@ import argparse import json import os +import signal import shutil import shlex import subprocess @@ -53,23 +54,68 @@ def _run(argv: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> su check=False, ) +def _terminate_driver_tree(process: subprocess.Popen[bytes]) -> None: + if os.name == "nt": + subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + return + + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + if process.poll() is None: + process.kill() + process.wait() + + def _run_driver(argv: list[str], *, cwd: Path, env: dict[str, str]) -> subprocess.CompletedProcess[bytes]: + kwargs: dict[str, Any] = {} + if os.name == "nt": + kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP + else: + kwargs["start_new_session"] = True + try: - return subprocess.run( + process = subprocess.Popen( argv, cwd=cwd, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - timeout=MAX_DRIVER_SECS, - check=False, + **kwargs, ) - except subprocess.TimeoutExpired as exc: - raise BenchmarkError("benchmark Host driver exceeded the 30 minute limit") from exc except OSError as exc: raise BenchmarkError("could not start benchmark Host driver") from exc + try: + returncode = process.wait(timeout=MAX_DRIVER_SECS) + except subprocess.TimeoutExpired as exc: + _terminate_driver_tree(process) + raise BenchmarkError("benchmark Host driver exceeded the 30 minute limit") from exc + if os.name != "nt": + _terminate_driver_tree(process) + return subprocess.CompletedProcess(argv, returncode) + def _require_exact_revision(repo: Path, revision: str) -> str: completed = _run(["git", "rev-parse", "--verify", f"{revision}^{{commit}}"], cwd=repo) @@ -81,6 +127,13 @@ def _require_exact_revision(repo: Path, revision: str) -> str: return resolved +def _git_root(path: Path) -> Path: + completed = _run(["git", "rev-parse", "--show-toplevel"], cwd=path) + if completed.returncode != 0: + raise BenchmarkError("benchmark repo must be inside a Git worktree") + return Path(completed.stdout.strip()).resolve() + + def _head_revision(repo: Path) -> str: completed = _run(["git", "rev-parse", "HEAD"], cwd=repo) if completed.returncode != 0: @@ -134,7 +187,11 @@ def _workspace_status(workspace: Path) -> str: return completed.stdout -def _fixture_oracle(case: dict[str, Any], workspace: Path) -> dict[str, Any]: +def _fixture_oracle( + case: dict[str, Any], + workspace: Path, + base_revision: str = "HEAD", +) -> dict[str, Any]: spec = (case.get("correctness") or {}).get("fixture_oracle") if not isinstance(spec, dict): return {"available": False, "passed": None, "checks": [], "reason": "case has no fixture_oracle"} @@ -155,9 +212,6 @@ def _fixture_oracle(case: dict[str, Any], workspace: Path) -> dict[str, Any]: except OSError: checks.append({"path": rel_path, "passed": False, "reason": "file unavailable"}) continue - except UnicodeDecodeError: - checks.append({"path": rel_path, "passed": False, "reason": "file is not UTF-8"}) - continue expected_text = expected.get("expected_text") if isinstance(expected_text, str): passed = text == expected_text @@ -169,7 +223,7 @@ def _fixture_oracle(case: dict[str, Any], workspace: Path) -> dict[str, Any]: expected_changed = sorted(spec.get("changed_files") or []) if expected_changed: - tracked = _run(["git", "diff", "--name-only", "HEAD", "--"], cwd=workspace) + tracked = _run(["git", "diff", "--name-only", base_revision, "--"], cwd=workspace) untracked = _run(["git", "ls-files", "--others", "--exclude-standard"], cwd=workspace) changed = sorted( set(line for line in tracked.stdout.splitlines() if line) @@ -325,9 +379,11 @@ def _run_sample( workspace = root / "worktrees" / label receipt_path = root / "receipts" / f"{label}.json" annotation_path = root / "annotations" / f"{label}.json" + receipt_path.unlink(missing_ok=True) + annotation_path.unlink(missing_ok=True) surface: str | None = None worktree_added = False - started = time.time_ns() // 1_000_000 + completed: subprocess.CompletedProcess[bytes] | None = None try: surface = _surface(case, variant) _worktree_add(repo, workspace, base_revision) @@ -347,6 +403,7 @@ def _run_sample( "WEBCODEX_BENCH_DRIVER_RESULT": str(receipt_path), } ) + started = time.time_ns() // 1_000_000 completed = _run_driver(driver_argv, cwd=workspace, env=env) ended = time.time_ns() // 1_000_000 if not receipt_path.exists(): @@ -364,7 +421,8 @@ def _run_sample( receipt = _load_driver_receipt(receipt_path) if receipt.get("task_timing") is None: receipt["task_timing"] = {"started_at_ms": started, "ended_at_ms": ended} - oracle = _fixture_oracle(case, workspace) + head_changed = _head_revision(workspace) != base_revision + oracle = _fixture_oracle(case, workspace, base_revision) annotation = _annotation( case, variant=variant, @@ -384,6 +442,10 @@ def _run_sample( case_manifest=case_manifest, ) effective_status = receipt["status"] + reason_code: str | None = None + if head_changed: + effective_status = "fail" + reason_code = "workspace_head_changed" if completed.returncode != 0 and effective_status in ("pass", "partial"): effective_status = "fail" if oracle["available"] and oracle["passed"] is not True and effective_status in ("pass", "partial"): @@ -428,6 +490,7 @@ def _run_sample( "surface": surface, "status": effective_status, "driver_exit_code": completed.returncode, + **({"reason_code": reason_code} if reason_code is not None else {}), "fixture_oracle": oracle, "summary": summary, } @@ -437,7 +500,7 @@ def _run_sample( "variant": variant, "surface": surface, "status": "fail", - "driver_exit_code": None, + "driver_exit_code": completed.returncode if completed is not None else None, "reason_code": "sample_contract_error", "fixture_oracle": {"available": False, "passed": None, "checks": [], "reason": "sample failed before oracle"}, "summary": None, @@ -469,19 +532,34 @@ def run_benchmark( ) -> dict[str, Any]: if pairs < 1: raise BenchmarkError("pairs must be at least 1") + if len(set(case_ids)) != len(case_ids): + raise BenchmarkError("duplicate case ids are not allowed in one benchmark run") + repo = _git_root(repo) resolved = _require_exact_revision(repo, base_revision) if _head_revision(repo) != resolved: raise BenchmarkError( "base revision must equal the current checkout HEAD; check out the target commit before benchmarking" ) temp_owner = tempfile.TemporaryDirectory(prefix="webcodex-agent-loop-bench-") - root = Path(temp_owner.name) - for name in ("worktrees", "receipts", "annotations"): - (root / name).mkdir(parents=True, exist_ok=True) - case_manifest = root / "agent_loop_cases.json" - case_manifest.write_text(_read_manifest_at_revision(repo, resolved), encoding="utf-8") - manifest = report.load_case_manifest(case_manifest) - selected = [_case(manifest, case_id) for case_id in case_ids] + try: + root = Path(temp_owner.name) + for name in ("worktrees", "receipts", "annotations"): + (root / name).mkdir(parents=True, exist_ok=True) + case_manifest = root / "agent_loop_cases.json" + case_manifest.write_text(_read_manifest_at_revision(repo, resolved), encoding="utf-8") + manifest = report.load_case_manifest(case_manifest) + selected = [_case(manifest, case_id) for case_id in case_ids] + unsupported_targets = sorted( + {str(case.get("target")) for case in selected if case.get("target") != "webcodex_repository"} + ) + if unsupported_targets: + raise BenchmarkError( + "paired worktree runner supports only target=webcodex_repository; " + f"unsupported target(s): {', '.join(unsupported_targets)}" + ) + except BaseException: + temp_owner.cleanup() + raise cases_out: list[dict[str, Any]] = [] try: diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index e63873a4d..122a93153 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -1,9 +1,11 @@ from __future__ import annotations import json +import os import subprocess import sys import tempfile +import time import unittest from pathlib import Path @@ -325,6 +327,283 @@ def test_base_revision_must_match_checkout_head(self) -> None: pairs=1, ) + def test_fixture_oracle_uses_recorded_base_when_head_advances(self) -> None: + manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + case = report._case_by_id(manifest, "guarded_multi_file_edit") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(["git", "init", "-b", "main"], cwd=root, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=root, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=root, check=True) + fixture = root / "tests/fixtures/agent-loop-multi-file" + fixture.mkdir(parents=True) + (fixture / "alpha.txt").write_text( + "BENCH_TARGET_ALPHA=before\nALPHA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=before\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-m", "base"], cwd=root, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + (fixture / "alpha.txt").write_text( + "BENCH_TARGET_ALPHA=after\nALPHA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=after\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-m", "driver commit"], cwd=root, check=True, stdout=subprocess.DEVNULL) + oracle = benchmark._fixture_oracle(case, root, base) + changed = next(check for check in oracle["checks"] if check.get("kind") == "changed_files") + self.assertTrue(changed["passed"]) + self.assertEqual(changed["actual"], sorted(case["correctness"]["changed_files"])) + self.assertNotEqual(subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip(), base) + + @unittest.skipIf(os.name == "nt", "POSIX process-group regression") + def test_driver_timeout_terminates_descendant_process_group(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + marker = root / "descendant-wrote" + driver = root / "driver.py" + driver.write_text( + "import subprocess, sys, time\n" + "subprocess.Popen([sys.executable, '-c', " + "'import pathlib,time,sys; time.sleep(1); pathlib.Path(sys.argv[1]).write_text(\\\"late\\\")', " + "sys.argv[1]])\n" + "time.sleep(10)\n", + encoding="utf-8", + ) + old_timeout = benchmark.MAX_DRIVER_SECS + benchmark.MAX_DRIVER_SECS = 0.1 + try: + with self.assertRaisesRegex(benchmark.BenchmarkError, "exceeded"): + benchmark._run_driver( + [sys.executable, str(driver), str(marker)], + cwd=root, + env=dict(os.environ), + ) + finally: + benchmark.MAX_DRIVER_SECS = old_timeout + time.sleep(1.2) + self.assertFalse(marker.exists()) + + def test_duplicate_case_ids_are_rejected_before_driver_execution(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + + marker = root / "driver-ran" + driver = root / "driver.py" + driver.write_text( + "from pathlib import Path; import sys; Path(sys.argv[1]).write_text('ran')\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(benchmark.BenchmarkError, "duplicate case ids"): + benchmark.run_benchmark( + repo=repo, + base_revision=base, + driver_argv=[sys.executable, str(driver), str(marker)], + case_ids=["readonly_review", "readonly_review"], + pairs=1, + ) + self.assertFalse(marker.exists()) + + def test_run_sample_clears_stale_receipt_before_driver(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + manifest = report.load_case_manifest(repo / "scripts/agent_loop_cases.json") + case = report._case_by_id(manifest, "readonly_review") + + bench_root = root / "bench" + for name in ("worktrees", "receipts", "annotations"): + (bench_root / name).mkdir(parents=True, exist_ok=True) + stale = bench_root / "receipts" / "readonly_review-p1-1-direct.json" + stale.write_text('{"status":"unsupported"}', encoding="utf-8") + sample = benchmark._run_sample( + repo, + bench_root, + [sys.executable, "-c", "pass"], + case, + variant="direct", + pair_index=0, + ordinal=0, + base_revision=base, + case_manifest=repo / "scripts/agent_loop_cases.json", + ) + self.assertEqual(sample["status"], "fail") + self.assertEqual(sample["reason_code"], "driver_receipt_missing") + + def test_contract_error_preserves_driver_exit_code(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + manifest = report.load_case_manifest(repo / "scripts/agent_loop_cases.json") + case = report._case_by_id(manifest, "readonly_review") + bench_root = root / "bench" + for name in ("worktrees", "receipts", "annotations"): + (bench_root / name).mkdir(parents=True, exist_ok=True) + driver = root / "driver.py" + driver.write_text( + "import os\n" + "from pathlib import Path\n" + "Path(os.environ['WEBCODEX_BENCH_DRIVER_RESULT']).write_text('not-json', encoding='utf-8')\n" + "raise SystemExit(7)\n", + encoding="utf-8", + ) + sample = benchmark._run_sample( + repo, + bench_root, + [sys.executable, str(driver)], + case, + variant="direct", + pair_index=0, + ordinal=0, + base_revision=base, + case_manifest=repo / "scripts/agent_loop_cases.json", + ) + self.assertEqual(sample["status"], "fail") + self.assertEqual(sample["reason_code"], "sample_contract_error") + self.assertEqual(sample["driver_exit_code"], 7) + + def test_fallback_task_timing_starts_after_worktree_setup(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + manifest_path = repo / "scripts/agent_loop_cases.json" + manifest = report.load_case_manifest(manifest_path) + case = report._case_by_id(manifest, "readonly_review") + bench_root = root / "bench" + for name in ("worktrees", "receipts", "annotations"): + (bench_root / name).mkdir(parents=True, exist_ok=True) + driver = root / "driver.py" + driver.write_text( + "import os\n" + "from pathlib import Path\n" + "Path(os.environ['WEBCODEX_BENCH_DRIVER_RESULT']).write_text('{\"status\":\"unsupported\"}', encoding='utf-8')\n", + encoding="utf-8", + ) + + setup_done_ms: list[int] = [] + original = benchmark._worktree_add + + def recorded_add(repo_arg: Path, path_arg: Path, revision_arg: str) -> None: + original(repo_arg, path_arg, revision_arg) + setup_done_ms.append(time.time_ns() // 1_000_000) + + benchmark._worktree_add = recorded_add + try: + benchmark._run_sample( + repo, + bench_root, + [sys.executable, str(driver)], + case, + variant="direct", + pair_index=0, + ordinal=0, + base_revision=base, + case_manifest=manifest_path, + ) + finally: + benchmark._worktree_add = original + + annotation = json.loads( + (bench_root / "annotations" / "readonly_review-p1-1-direct.json").read_text(encoding="utf-8") + ) + self.assertGreaterEqual(annotation["task_timing"]["started_at_ms"], setup_done_ms[0]) + + def test_non_repository_case_target_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + with self.assertRaisesRegex(benchmark.BenchmarkError, "supports only target=webcodex_repository"): + benchmark.run_benchmark( + repo=repo, + base_revision=base, + driver_argv=[sys.executable, "-c", "pass"], + case_ids=["focused_edit_validation"], + pairs=1, + ) + + def test_repo_argument_is_canonicalized_from_subdirectory(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + driver = root / "driver.py" + driver.write_text( + "import os\n" + "from pathlib import Path\n" + "Path(os.environ['WEBCODEX_BENCH_DRIVER_RESULT']).write_text('{\"status\":\"unsupported\"}', encoding='utf-8')\n", + encoding="utf-8", + ) + result = benchmark.run_benchmark( + repo=repo / "scripts", + base_revision=base, + driver_argv=[sys.executable, str(driver)], + case_ids=["readonly_review"], + pairs=1, + ) + self.assertEqual(result["status_counts"]["unsupported"], 2) + def test_cli_prints_human_summary_and_json_output(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From bcae85593993666600cd88a34b4d4f22ab283930 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:16:59 +0800 Subject: [PATCH 09/12] test(agent-loop): fail closed on incomplete benchmark evidence --- .gitattributes | 2 + docs/experiments/agent-loop-baseline.md | 11 +- scripts/agent_loop_benchmark.py | 223 ++++++++++++++++---- scripts/tests/test_agent_loop_benchmark.py | 234 +++++++++++++++++++++ 4 files changed, 426 insertions(+), 44 deletions(-) diff --git a/.gitattributes b/.gitattributes index aa5d24639..993f9179d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,7 @@ *.sh text eol=lf *.toml text eol=lf +# Benchmark fixture bytes are part of the independent oracle. +tests/fixtures/agent-loop-multi-file/*.txt text eol=lf # Frontend freshness checks compare these generated assets byte-for-byte. frontend/dist/** text eol=lf diff --git a/docs/experiments/agent-loop-baseline.md b/docs/experiments/agent-loop-baseline.md index c43189ca7..eaea232f4 100644 --- a/docs/experiments/agent-loop-baseline.md +++ b/docs/experiments/agent-loop-baseline.md @@ -275,7 +275,9 @@ only reproducibility mechanics: - fixture-side disk/diff oracles where the manifest provides one, plus persisted call-count evidence for the multi-file case's single guarded edit and search/read phases; - explicit `pass`, `fail`, `partial`, and `unsupported` samples. None are silently filtered from the aggregate; +- non-unsupported `pass` requires non-empty canonical ActionAudit evidence; Code Mode also requires complete persisted composition evidence; missing capture is `partial`, never fabricated success; - cleanup of only the worktrees/temp state created by the orchestrator. +- Host-driver process ownership currently uses POSIX process groups; Windows runs are retained explicitly as `unsupported` until equivalent descendant ownership can be guaranteed. Example: @@ -292,7 +294,7 @@ For each invocation the driver receives `WEBCODEX_BENCH_WORKSPACE`, `WEBCODEX_BENCH_SURFACE`, `WEBCODEX_BENCH_BASE_REVISION`, `WEBCODEX_BENCH_CASE_FINGERPRINT`, `WEBCODEX_BENCH_PROMPT`, and `WEBCODEX_BENCH_DRIVER_RESULT`. It writes one bounded JSON receipt to the last -path. The receipt is capped at 64 KiB; driver stdout/stderr is not retained by the orchestrator. `status` is required and is one of `pass/fail/partial/unsupported`. +path. The receipt is capped at 64 KiB; driver stdout/stderr is not retained by the orchestrator. `status` is required and is one of `pass/fail/partial/unsupported`. Relative `audit_db` and `trace_root` paths are resolved against the fresh sample workspace; absolute paths remain absolute. Non-unsupported samples also provide the authoritative ActionAudit database path and Workflow Session id; optional trace root, repair-turn annotation, and task timing use the existing reporter semantics. Correctness verdicts are optional only when an @@ -304,9 +306,10 @@ Window, tunnel, or Host-private ids. The default lane is exactly `readonly_review`, `guarded_multi_file_edit`, and `long_validation_handoff`, matching #822. The handoff case is valid only when the -driver observes a genuine pending validation and follows the same Job to terminal; -a synchronous completion or substitute pending process must be reported -`partial`/`unsupported`, not upgraded to success. +report proves exactly one genuine pending validation, a correlated follow-up for +that relation, and a same-relation terminal observation with authoritative terminal +time. A synchronous completion, an uncorrelated/substitute process, or pending +without terminal evidence remains `partial`/`unsupported`, never upgraded to success. The guarded multi-file case is likewise not a pass from final bytes alone: persisted Direct canonical-call or Code Mode composition evidence must show exactly one `edit_project_files` call, at least one admitted search call, and at least two `read_files` calls (pre-edit and post-edit phases). Current composition does not prove nested child ordering, so the benchmark does not claim stronger sequencing evidence than is persisted. This command coordinates real captures; it is not itself evidence that a model made diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index 4ae82324a..eecfaeec1 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -13,6 +13,7 @@ import json import os import signal +import stat import shutil import shlex import subprocess @@ -32,6 +33,7 @@ VARIANTS = ("direct", "code_mode") MAX_DRIVER_RECEIPT_BYTES = 64 * 1024 MAX_FIXTURE_ORACLE_BYTES = 1024 * 1024 +MAX_CASE_MANIFEST_BYTES = 1024 * 1024 MAX_DRIVER_SECS = 30 * 60 @@ -39,6 +41,14 @@ class BenchmarkError(ValueError): pass +class DriverTimeoutError(BenchmarkError): + pass + + +class DriverPlatformUnsupported(BenchmarkError): + pass + + def _stable_json(value: Any) -> str: return json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n" @@ -55,21 +65,6 @@ def _run(argv: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> su ) def _terminate_driver_tree(process: subprocess.Popen[bytes]) -> None: - if os.name == "nt": - subprocess.run( - ["taskkill", "/PID", str(process.pid), "/T", "/F"], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - check=False, - ) - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - process.wait() - return - try: os.killpg(process.pid, signal.SIGTERM) except ProcessLookupError: @@ -88,11 +83,10 @@ def _terminate_driver_tree(process: subprocess.Popen[bytes]) -> None: def _run_driver(argv: list[str], *, cwd: Path, env: dict[str, str]) -> subprocess.CompletedProcess[bytes]: - kwargs: dict[str, Any] = {} if os.name == "nt": - kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP - else: - kwargs["start_new_session"] = True + raise DriverPlatformUnsupported( + "paired benchmark Host driver requires POSIX process-group ownership" + ) try: process = subprocess.Popen( @@ -102,7 +96,7 @@ def _run_driver(argv: list[str], *, cwd: Path, env: dict[str, str]) -> subproces stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - **kwargs, + start_new_session=True, ) except OSError as exc: raise BenchmarkError("could not start benchmark Host driver") from exc @@ -111,9 +105,11 @@ def _run_driver(argv: list[str], *, cwd: Path, env: dict[str, str]) -> subproces returncode = process.wait(timeout=MAX_DRIVER_SECS) except subprocess.TimeoutExpired as exc: _terminate_driver_tree(process) - raise BenchmarkError("benchmark Host driver exceeded the 30 minute limit") from exc - if os.name != "nt": + raise DriverTimeoutError("benchmark Host driver exceeded the 30 minute limit") from exc + except BaseException: _terminate_driver_tree(process) + raise + _terminate_driver_tree(process) return subprocess.CompletedProcess(argv, returncode) @@ -145,13 +141,30 @@ def _head_revision(repo: Path) -> str: def _read_manifest_at_revision(repo: Path, revision: str) -> str: - completed = _run( - ["git", "show", f"{revision}:scripts/agent_loop_cases.json"], + object_name = f"{revision}:scripts/agent_loop_cases.json" + size_result = _run(["git", "cat-file", "-s", object_name], cwd=repo) + if size_result.returncode != 0: + raise BenchmarkError("could not read benchmark case manifest at base revision") + try: + size = int(size_result.stdout.strip()) + except ValueError as exc: + raise BenchmarkError("benchmark case manifest size is invalid") from exc + if size < 0 or size > MAX_CASE_MANIFEST_BYTES: + raise BenchmarkError("benchmark case manifest exceeds the 1 MiB limit") + + completed = subprocess.run( + ["git", "show", object_name], cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, ) - if completed.returncode != 0: + if completed.returncode != 0 or len(completed.stdout) != size: raise BenchmarkError("could not read benchmark case manifest at base revision") - return completed.stdout + try: + return completed.stdout.decode("utf-8") + except UnicodeDecodeError as exc: + raise BenchmarkError("benchmark case manifest must be UTF-8 JSON") from exc def _case(manifest: dict[str, Any], case_id: str) -> dict[str, Any]: @@ -200,18 +213,44 @@ def _fixture_oracle( for rel_path, expected in sorted((spec.get("files") or {}).items()): target = workspace / rel_path try: - with target.open("rb") as handle: - raw = handle.read(MAX_FIXTURE_ORACLE_BYTES + 1) - if len(raw) > MAX_FIXTURE_ORACLE_BYTES: - checks.append({"path": rel_path, "passed": False, "reason": "file exceeds oracle byte limit"}) + before = target.lstat() + except OSError: + checks.append({"path": rel_path, "passed": False, "reason": "file unavailable"}) + continue + if not stat.S_ISREG(before.st_mode): + checks.append({"path": rel_path, "passed": False, "reason": "file is not regular"}) + continue + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + try: + fd = os.open(target, flags) + except OSError: + checks.append({"path": rel_path, "passed": False, "reason": "file unavailable"}) + continue + try: + opened = os.fstat(fd) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_dev != before.st_dev + or opened.st_ino != before.st_ino + ): + checks.append({"path": rel_path, "passed": False, "reason": "file identity changed"}) continue + with os.fdopen(fd, "rb", closefd=True) as handle: + fd = -1 + raw = handle.read(MAX_FIXTURE_ORACLE_BYTES + 1) + finally: + if fd >= 0: + os.close(fd) + if len(raw) > MAX_FIXTURE_ORACLE_BYTES: + checks.append({"path": rel_path, "passed": False, "reason": "file exceeds oracle byte limit"}) + continue + try: text = raw.decode("utf-8") except UnicodeDecodeError: checks.append({"path": rel_path, "passed": False, "reason": "file is not UTF-8"}) continue - except OSError: - checks.append({"path": rel_path, "passed": False, "reason": "file unavailable"}) - continue expected_text = expected.get("expected_text") if isinstance(expected_text, str): passed = text == expected_text @@ -244,14 +283,39 @@ def _fixture_oracle( def _load_driver_receipt(path: Path) -> dict[str, Any]: try: - if path.stat().st_size > MAX_DRIVER_RECEIPT_BYTES: - raise BenchmarkError("driver result exceeds the 64 KiB receipt limit") + before = path.lstat() except OSError as exc: raise BenchmarkError("driver did not create its result receipt") from exc + if not stat.S_ISREG(before.st_mode): + raise BenchmarkError("driver result must be a regular file") + if before.st_size > MAX_DRIVER_RECEIPT_BYTES: + raise BenchmarkError("driver result exceeds the 64 KiB receipt limit") + + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW try: - value = json.loads(path.read_text(encoding="utf-8")) + fd = os.open(path, flags) except OSError as exc: - raise BenchmarkError("driver did not create its result receipt") from exc + raise BenchmarkError("driver result must be a readable regular file") from exc + try: + opened = os.fstat(fd) + if ( + not stat.S_ISREG(opened.st_mode) + or opened.st_dev != before.st_dev + or opened.st_ino != before.st_ino + ): + raise BenchmarkError("driver result changed identity before read") + with os.fdopen(fd, "rb", closefd=True) as handle: + fd = -1 + raw = handle.read(MAX_DRIVER_RECEIPT_BYTES + 1) + finally: + if fd >= 0: + os.close(fd) + if len(raw) > MAX_DRIVER_RECEIPT_BYTES: + raise BenchmarkError("driver result exceeds the 64 KiB receipt limit") + try: + value = json.loads(raw.decode("utf-8")) except UnicodeDecodeError as exc: raise BenchmarkError("driver result must be UTF-8 JSON") from exc except json.JSONDecodeError as exc: @@ -311,6 +375,7 @@ def _summary_for_receipt( receipt: dict[str, Any], annotation_path: Path, case_manifest: Path, + workspace: Path, ) -> dict[str, Any] | None: if receipt["status"] == "unsupported": return None @@ -321,9 +386,18 @@ def _summary_for_receipt( raise BenchmarkError("non-unsupported driver result requires audit_db") if not isinstance(workflow_session_id, str) or not workflow_session_id: raise BenchmarkError("non-unsupported driver result requires workflow_session_id") + + audit_path = Path(audit_db) + if not audit_path.is_absolute(): + audit_path = workspace / audit_path + trace_path: Path | None = None + if isinstance(trace_root, str) and trace_root: + trace_path = Path(trace_root) + if not trace_path.is_absolute(): + trace_path = workspace / trace_path return report.summarize( - trace_root=Path(trace_root) if isinstance(trace_root, str) and trace_root else None, - audit_db=Path(audit_db), + trace_root=trace_path, + audit_db=audit_path, workflow_session_id=workflow_session_id, case_manifest=case_manifest, case_id=case["id"], @@ -347,6 +421,45 @@ def _tool_count(summary: dict[str, Any], variant: str, tool_name: str) -> int | return None +def _runtime_evidence_proven(summary: dict[str, Any], variant: str) -> bool: + outer_total = report._get_path(summary, "outer_calls.total") + if not isinstance(outer_total, int) or isinstance(outer_total, bool) or outer_total <= 0: + return False + if variant == "direct": + canonical_total = report._get_path(summary, "canonical_calls.total") + return ( + isinstance(canonical_total, int) + and not isinstance(canonical_total, bool) + and canonical_total > 0 + ) + if variant == "code_mode": + outer_code_mode = report._get_path(summary, "composition.outer_code_mode_calls") + return ( + report._get_path(summary, "availability.code_mode_composition.available") is True + and isinstance(outer_code_mode, int) + and not isinstance(outer_code_mode, bool) + and outer_code_mode > 0 + ) + return False + + +def _long_handoff_terminal_proven(summary: dict[str, Any]) -> bool: + pending = report._get_path(summary, "job_convergence.pending_handoff_count") + known_followups = report._get_path(summary, "job_convergence.pending_followup_known_count") + terminal_samples = report._get_path(summary, "job_convergence.pending_to_terminal_ms.samples") + return ( + isinstance(pending, int) + and not isinstance(pending, bool) + and pending == 1 + and isinstance(known_followups, int) + and not isinstance(known_followups, bool) + and known_followups >= 1 + and isinstance(terminal_samples, int) + and not isinstance(terminal_samples, bool) + and terminal_samples >= 1 + ) + + def _guarded_multi_file_contract_proven(summary: dict[str, Any], variant: str) -> bool: edit_count = _tool_count(summary, variant, "edit_project_files") read_count = _tool_count(summary, variant, "read_files") @@ -440,6 +553,7 @@ def _run_sample( receipt=receipt, annotation_path=annotation_path, case_manifest=case_manifest, + workspace=workspace, ) effective_status = receipt["status"] reason_code: str | None = None @@ -469,11 +583,18 @@ def _run_sample( elif not case["validation"]["required"] and validation_verdict not in ("pass", "not_required"): effective_status = "partial" + if ( + summary is not None + and effective_status == "pass" + and not _runtime_evidence_proven(summary, variant) + ): + effective_status = "partial" + if ( case["id"] == "long_validation_handoff" and summary is not None and effective_status == "pass" - and report._get_path(summary, "job_convergence.pending_handoff_count") in (None, 0) + and not _long_handoff_terminal_proven(summary) ): effective_status = "partial" @@ -494,6 +615,28 @@ def _run_sample( "fixture_oracle": oracle, "summary": summary, } + except DriverPlatformUnsupported: + return { + "case_id": case["id"], + "variant": variant, + "surface": surface, + "status": "unsupported", + "driver_exit_code": None, + "reason_code": "driver_platform_unsupported", + "fixture_oracle": {"available": False, "passed": None, "checks": [], "reason": "driver platform unsupported"}, + "summary": None, + } + except DriverTimeoutError: + return { + "case_id": case["id"], + "variant": variant, + "surface": surface, + "status": "fail", + "driver_exit_code": None, + "reason_code": "driver_timeout", + "fixture_oracle": {"available": False, "passed": None, "checks": [], "reason": "driver timed out"}, + "summary": None, + } except (BenchmarkError, report.ReportError): return { "case_id": case["id"], diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index 122a93153..400c8c163 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -8,6 +8,7 @@ import time import unittest from pathlib import Path +from unittest import mock from scripts import agent_loop_benchmark as benchmark from scripts import agent_loop_report as report @@ -27,6 +28,7 @@ def test_pair_order_alternates(self) -> None: self.assertEqual(benchmark._pair_order(1), ("code_mode", "direct")) self.assertEqual(benchmark._pair_order(2), ("direct", "code_mode")) + @unittest.skipIf(os.name == "nt", "paired Host driver is explicitly POSIX-only") def test_driver_stdin_is_closed(self) -> None: with tempfile.TemporaryDirectory() as tmp: output = Path(tmp) / "stdin.txt" @@ -42,6 +44,68 @@ def test_driver_stdin_is_closed(self) -> None: ) self.assertEqual(output.read_text(encoding="utf-8"), "") + @unittest.skipIf(os.name == "nt", "paired Host driver is explicitly POSIX-only") + def test_driver_interruption_terminates_owned_process_tree(self) -> None: + process = mock.Mock() + process.pid = 12345 + process.wait.side_effect = KeyboardInterrupt + with ( + mock.patch.object(benchmark.subprocess, "Popen", return_value=process), + mock.patch.object(benchmark, "_terminate_driver_tree") as terminate, + ): + with self.assertRaises(KeyboardInterrupt): + benchmark._run_driver( + [sys.executable, "-c", "pass"], + cwd=Path("."), + env={}, + ) + terminate.assert_called_once_with(process) + + def test_windows_host_driver_fails_closed_as_unsupported(self) -> None: + with mock.patch.object(benchmark.os, "name", "nt"): + with self.assertRaisesRegex( + benchmark.DriverPlatformUnsupported, + "POSIX process-group ownership", + ): + benchmark._run_driver( + [sys.executable, "-c", "pass"], + cwd=Path("."), + env={}, + ) + + def test_runtime_evidence_gate_fails_closed(self) -> None: + direct = { + "outer_calls": {"total": 1}, + "canonical_calls": {"total": 1}, + } + self.assertTrue(benchmark._runtime_evidence_proven(direct, "direct")) + direct["outer_calls"]["total"] = 0 + self.assertFalse(benchmark._runtime_evidence_proven(direct, "direct")) + + code_mode = { + "outer_calls": {"total": 1}, + "availability": {"code_mode_composition": {"available": True}}, + "composition": {"outer_code_mode_calls": 1}, + } + self.assertTrue(benchmark._runtime_evidence_proven(code_mode, "code_mode")) + code_mode["availability"]["code_mode_composition"]["available"] = False + self.assertFalse(benchmark._runtime_evidence_proven(code_mode, "code_mode")) + + def test_long_handoff_requires_correlated_terminal_evidence(self) -> None: + summary = { + "job_convergence": { + "pending_handoff_count": 1, + "pending_followup_known_count": 1, + "pending_to_terminal_ms": {"samples": 1}, + } + } + self.assertTrue(benchmark._long_handoff_terminal_proven(summary)) + summary["job_convergence"]["pending_to_terminal_ms"]["samples"] = 0 + self.assertFalse(benchmark._long_handoff_terminal_proven(summary)) + summary["job_convergence"]["pending_to_terminal_ms"]["samples"] = 1 + summary["job_convergence"]["pending_handoff_count"] = 2 + self.assertFalse(benchmark._long_handoff_terminal_proven(summary)) + def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(self) -> None: direct = { "canonical_calls": { @@ -136,6 +200,58 @@ def test_fixture_oracle_checks_both_files_and_exact_diff(self) -> None: beta_check = next(check for check in result["checks"] if check.get("path", "").endswith("beta.txt")) self.assertEqual(beta_check["reason"], "file exceeds oracle byte limit") + @unittest.skipIf(os.name == "nt", "FIFO is POSIX-only") + def test_fixture_oracle_rejects_fifo_without_blocking(self) -> None: + manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + case = report._case_by_id(manifest, "guarded_multi_file_edit") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(["git", "init", "-b", "main"], cwd=root, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=root, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=root, check=True) + fixture = root / "tests/fixtures/agent-loop-multi-file" + fixture.mkdir(parents=True) + (fixture / "alpha.txt").write_text( + "BENCH_TARGET_ALPHA=after\nALPHA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + beta = fixture / "beta.txt" + beta.write_text( + "BENCH_TARGET_BETA=before\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-m", "fixture"], cwd=root, check=True, stdout=subprocess.DEVNULL) + beta.unlink() + os.mkfifo(beta) + started = time.monotonic() + result = benchmark._fixture_oracle(case, root) + self.assertLess(time.monotonic() - started, 0.5) + self.assertFalse(result["passed"]) + beta_check = next(check for check in result["checks"] if check.get("path", "").endswith("beta.txt")) + self.assertEqual(beta_check["reason"], "file is not regular") + + def test_multi_file_fixture_is_pinned_to_lf_checkout(self) -> None: + repo = Path(__file__).resolve().parents[2] + completed = subprocess.run( + [ + "git", + "check-attr", + "eol", + "--", + "tests/fixtures/agent-loop-multi-file/alpha.txt", + "tests/fixtures/agent-loop-multi-file/beta.txt", + ], + cwd=repo, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + lines = [line for line in completed.stdout.splitlines() if line] + self.assertEqual(len(lines), 2) + self.assertTrue(all(line.endswith(": lf") for line in lines)) + def test_malformed_receipt_status_is_a_contract_error(self) -> None: with tempfile.TemporaryDirectory() as tmp: receipt = Path(tmp) / "receipt.json" @@ -143,6 +259,29 @@ def test_malformed_receipt_status_is_a_contract_error(self) -> None: with self.assertRaisesRegex(benchmark.BenchmarkError, "status must be"): benchmark._load_driver_receipt(receipt) + @unittest.skipIf(os.name == "nt", "FIFO is POSIX-only") + def test_driver_receipt_rejects_fifo_without_blocking(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + receipt = Path(tmp) / "receipt.json" + os.mkfifo(receipt) + started = time.monotonic() + with self.assertRaisesRegex(benchmark.BenchmarkError, "regular file"): + benchmark._load_driver_receipt(receipt) + self.assertLess(time.monotonic() - started, 0.5) + + def test_driver_receipt_rejects_symlink(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + target = root / "target.json" + target.write_text('{"status":"pass"}', encoding="utf-8") + receipt = root / "receipt.json" + try: + receipt.symlink_to(target) + except OSError: + self.skipTest("symlinks unavailable") + with self.assertRaisesRegex(benchmark.BenchmarkError, "regular file"): + benchmark._load_driver_receipt(receipt) + def test_comparison_requires_two_effective_pass_samples(self) -> None: summary = {"benchmark": {}} self.assertIsNone( @@ -158,6 +297,39 @@ def test_comparison_requires_two_effective_pass_samples(self) -> None: ) ) + def test_relative_receipt_artifacts_are_anchored_to_workspace(self) -> None: + manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + case = report._case_by_id(manifest, "readonly_review") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + annotation = root / "annotation.json" + case_manifest = root / "manifest.json" + case_manifest.write_text( + report.DEFAULT_CASE_MANIFEST.read_text(encoding="utf-8"), + encoding="utf-8", + ) + receipt = { + "status": "pass", + "audit_db": ".webcodex/action_audit.db", + "trace_root": ".webcodex/traces", + "workflow_session_id": "wc_sess_test", + } + with mock.patch.object(report, "summarize", return_value={"ok": True}) as summarize: + value = benchmark._summary_for_receipt( + case, + variant="direct", + surface="direct", + base_revision="0" * 40, + receipt=receipt, + annotation_path=annotation, + case_manifest=case_manifest, + workspace=root, + ) + self.assertEqual(value, {"ok": True}) + kwargs = summarize.call_args.kwargs + self.assertEqual(kwargs["audit_db"], root / ".webcodex/action_audit.db") + self.assertEqual(kwargs["trace_root"], root / ".webcodex/traces") + def test_missing_receipts_are_retained_as_failed_samples(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -264,6 +436,28 @@ def test_cleanup_does_not_prune_unrelated_worktree(self) -> None: self.assertEqual(status.returncode, 0, status.stderr) subprocess.run(["git", "worktree", "remove", "--force", str(unrelated)], cwd=repo, check=True) + def test_manifest_blob_is_bounded_and_utf8(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(["git", "init", "-b", "main"], cwd=root, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=root, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=root, check=True) + manifest = root / "scripts/agent_loop_cases.json" + manifest.parent.mkdir(parents=True) + manifest.write_bytes(b"\xff\xfe") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-m", "bad utf8"], cwd=root, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + with self.assertRaisesRegex(benchmark.BenchmarkError, "UTF-8"): + benchmark._read_manifest_at_revision(root, base) + + manifest.write_bytes(b"x" * (benchmark.MAX_CASE_MANIFEST_BYTES + 1)) + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-m", "too large"], cwd=root, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + with self.assertRaisesRegex(benchmark.BenchmarkError, "1 MiB"): + benchmark._read_manifest_at_revision(root, base) + def test_case_manifest_is_pinned_to_base_revision(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -457,6 +651,46 @@ def test_run_sample_clears_stale_receipt_before_driver(self) -> None: self.assertEqual(sample["status"], "fail") self.assertEqual(sample["reason_code"], "driver_receipt_missing") + def test_driver_timeout_has_distinct_reason_code(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + manifest_path = repo / "scripts/agent_loop_cases.json" + manifest = report.load_case_manifest(manifest_path) + case = report._case_by_id(manifest, "readonly_review") + bench_root = root / "bench" + for name in ("worktrees", "receipts", "annotations"): + (bench_root / name).mkdir(parents=True, exist_ok=True) + + with mock.patch.object( + benchmark, + "_run_driver", + side_effect=benchmark.DriverTimeoutError("timeout"), + ): + sample = benchmark._run_sample( + repo, + bench_root, + [sys.executable, "-c", "pass"], + case, + variant="direct", + pair_index=0, + ordinal=0, + base_revision=base, + case_manifest=manifest_path, + ) + self.assertEqual(sample["status"], "fail") + self.assertEqual(sample["reason_code"], "driver_timeout") + self.assertIsNone(sample["driver_exit_code"]) + def test_contract_error_preserves_driver_exit_code(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From afc3c61d73345d0fc566e1dde599df22a9501ff5 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:43:52 +0800 Subject: [PATCH 10/12] test(agent-loop): harden benchmark evidence gates --- docs/experiments/agent-loop-baseline.md | 8 +- scripts/agent_loop_benchmark.py | 174 ++++++++++++++++++--- scripts/tests/test_agent_loop_benchmark.py | 123 ++++++++++++++- 3 files changed, 272 insertions(+), 33 deletions(-) diff --git a/docs/experiments/agent-loop-baseline.md b/docs/experiments/agent-loop-baseline.md index eaea232f4..ad6e54588 100644 --- a/docs/experiments/agent-loop-baseline.md +++ b/docs/experiments/agent-loop-baseline.md @@ -272,11 +272,11 @@ only reproducibility mechanics: - alternating pair order (`direct/code_mode`, then `code_mode/direct`) to expose one-sided process/cache ordering effects; - bounded annotation and the existing report/correctness compatibility gates; -- fixture-side disk/diff oracles where the manifest provides one, plus persisted call-count evidence for the multi-file case's single guarded edit and search/read phases; +- fixture-side disk/diff oracles where the manifest provides one; changed-path capture is bounded to 64 KiB / 512 paths, and the multi-file case additionally requires persisted evidence for one successful guarded edit plus its search/read phases; - explicit `pass`, `fail`, `partial`, and `unsupported` samples. None are silently filtered from the aggregate; -- non-unsupported `pass` requires non-empty canonical ActionAudit evidence; Code Mode also requires complete persisted composition evidence; missing capture is `partial`, never fabricated success; -- cleanup of only the worktrees/temp state created by the orchestrator. +- non-unsupported `pass` requires non-empty canonical ActionAudit evidence and the audited execution lane must match the requested variant/surface (`direct`, `execute_code_mode`, `execute_effectful_code_mode`, or `execute_mutating_code_mode` as applicable); Code Mode also requires complete persisted composition evidence; missing or mismatched capture is `partial`, never fabricated success; +- cleanup of only the worktrees/temp state created by the orchestrator, including targeted rollback of an owned worktree registration when checkout setup fails; no global `git worktree prune` is used; - Host-driver process ownership currently uses POSIX process groups; Windows runs are retained explicitly as `unsupported` until equivalent descendant ownership can be guaranteed. Example: @@ -310,7 +310,7 @@ report proves exactly one genuine pending validation, a correlated follow-up for that relation, and a same-relation terminal observation with authoritative terminal time. A synchronous completion, an uncorrelated/substitute process, or pending without terminal evidence remains `partial`/`unsupported`, never upgraded to success. -The guarded multi-file case is likewise not a pass from final bytes alone: persisted Direct canonical-call or Code Mode composition evidence must show exactly one `edit_project_files` call, at least one admitted search call, and at least two `read_files` calls (pre-edit and post-edit phases). Current composition does not prove nested child ordering, so the benchmark does not claim stronger sequencing evidence than is persisted. +The guarded multi-file case is likewise not a pass from final bytes alone: persisted Direct canonical-call or Code Mode composition evidence must show exactly one `edit_project_files` call, at least one admitted search call, and at least two `read_files` calls (pre-edit and post-edit phases). The outer capture must contain no failed/unknown calls, and Code Mode must report zero nested failures, so a failed guarded edit followed by a shell/process fallback cannot be promoted to a pass by the final-byte oracle. Current composition does not prove nested child ordering, so the benchmark does not claim stronger sequencing evidence than is persisted. This command coordinates real captures; it is not itself evidence that a model made fewer inference turns. Deterministic runtime/script acceptance and fixture oracles diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index eecfaeec1..733f74393 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -34,6 +34,13 @@ MAX_DRIVER_RECEIPT_BYTES = 64 * 1024 MAX_FIXTURE_ORACLE_BYTES = 1024 * 1024 MAX_CASE_MANIFEST_BYTES = 1024 * 1024 +MAX_GIT_PATH_BYTES = 64 * 1024 +MAX_GIT_PATHS = 512 +CODE_MODE_TOOL_BY_SURFACE = { + "read_only": "execute_code_mode", + "validation": "execute_effectful_code_mode", + "guarded_edit": "execute_mutating_code_mode", +} MAX_DRIVER_SECS = 30 * 60 @@ -183,21 +190,87 @@ def _surface(case: dict[str, Any], variant: str) -> str: def _worktree_add(repo: Path, path: Path, base_revision: str) -> None: completed = _run(["git", "worktree", "add", "--detach", str(path), base_revision], cwd=repo) - if completed.returncode != 0: - raise BenchmarkError(completed.stderr.strip() or "git worktree add failed") + if completed.returncode == 0: + return + + add_error = completed.stderr.strip() or "git worktree add failed" + try: + _worktree_remove(repo, path) + except BenchmarkError as cleanup_error: + raise BenchmarkError(f"{add_error}; owned worktree rollback failed: {cleanup_error}") from cleanup_error + raise BenchmarkError(add_error) def _worktree_remove(repo: Path, path: Path) -> None: - _run(["git", "worktree", "remove", "--force", str(path)], cwd=repo) + completed = _run(["git", "worktree", "remove", "--force", str(path)], cwd=repo) + if completed.returncode == 0: + return + if path.exists(): shutil.rmtree(path, ignore_errors=True) + retry = _run(["git", "worktree", "remove", "--force", str(path)], cwd=repo) + if retry.returncode == 0: + return + if not path.exists() and "is not a working tree" in retry.stderr: + return + raise BenchmarkError(retry.stderr.strip() or "failed to remove owned benchmark worktree") -def _workspace_status(workspace: Path) -> str: - completed = _run(["git", "status", "--porcelain", "--untracked-files=all"], cwd=workspace) - if completed.returncode != 0: - raise BenchmarkError(completed.stderr.strip() or "git status failed") - return completed.stdout +def _workspace_status(workspace: Path) -> bool: + try: + process = subprocess.Popen( + ["git", "status", "--porcelain", "--untracked-files=all"], + cwd=workspace, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except OSError as exc: + raise BenchmarkError("could not start git status") from exc + + assert process.stdout is not None + with process.stdout: + first = process.stdout.read(1) + if first and process.poll() is None: + process.kill() + returncode = process.wait() + if first: + return True + if returncode != 0: + raise BenchmarkError("git status failed") + return False + + +def _bounded_git_paths( + argv: list[str], + *, + cwd: Path, +) -> tuple[bool, bool, list[bytes]]: + try: + process = subprocess.Popen( + ["git", *argv], + cwd=cwd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except OSError as exc: + raise BenchmarkError("could not start bounded Git path query") from exc + + assert process.stdout is not None + with process.stdout: + raw = process.stdout.read(MAX_GIT_PATH_BYTES + 1) + byte_overflow = len(raw) > MAX_GIT_PATH_BYTES + if byte_overflow and process.poll() is None: + process.kill() + returncode = process.wait() + if byte_overflow: + return returncode == 0, True, [] + + paths = [item for item in raw.split(b"\0") if item] + if len(paths) > MAX_GIT_PATHS: + return returncode == 0, True, [] + return returncode == 0, False, paths def _fixture_oracle( @@ -262,18 +335,28 @@ def _fixture_oracle( expected_changed = sorted(spec.get("changed_files") or []) if expected_changed: - tracked = _run(["git", "diff", "--name-only", base_revision, "--"], cwd=workspace) - untracked = _run(["git", "ls-files", "--others", "--exclude-standard"], cwd=workspace) - changed = sorted( - set(line for line in tracked.stdout.splitlines() if line) - | set(line for line in untracked.stdout.splitlines() if line) + tracked_ok, tracked_overflow, tracked_paths = _bounded_git_paths( + ["diff", "--name-only", "-z", base_revision, "--"], + cwd=workspace, + ) + untracked_ok, untracked_overflow, untracked_paths = _bounded_git_paths( + ["ls-files", "-z", "--others", "--exclude-standard"], + cwd=workspace, ) + overflow = tracked_overflow or untracked_overflow + changed_raw = sorted(set(tracked_paths) | set(untracked_paths)) if not overflow else [] + expected_raw = sorted(item.encode("utf-8") for item in expected_changed) + changed = [ + item.decode("utf-8", errors="backslashreplace") + for item in changed_raw + ] checks.append( { "kind": "changed_files", - "passed": tracked.returncode == 0 and untracked.returncode == 0 and changed == expected_changed, + "passed": tracked_ok and untracked_ok and not overflow and changed_raw == expected_raw, "expected": expected_changed, - "actual": changed, + "actual": None if overflow else changed, + "reason": "changed path listing exceeds oracle limit" if overflow else None, } ) @@ -421,24 +504,50 @@ def _tool_count(summary: dict[str, Any], variant: str, tool_name: str) -> int | return None -def _runtime_evidence_proven(summary: dict[str, Any], variant: str) -> bool: +def _runtime_evidence_proven( + summary: dict[str, Any], + variant: str, + surface: str | None = None, +) -> bool: outer_total = report._get_path(summary, "outer_calls.total") if not isinstance(outer_total, int) or isinstance(outer_total, bool) or outer_total <= 0: return False + + composition_available = ( + report._get_path(summary, "availability.code_mode_composition.available") is True + ) if variant == "direct": canonical_total = report._get_path(summary, "canonical_calls.total") return ( - isinstance(canonical_total, int) + composition_available + and isinstance(canonical_total, int) and not isinstance(canonical_total, bool) and canonical_total > 0 ) + if variant == "code_mode": + expected_tool = CODE_MODE_TOOL_BY_SURFACE.get(surface or "") outer_code_mode = report._get_path(summary, "composition.outer_code_mode_calls") + outer_by_name = report._get_path(summary, "tools.outer_by_name") + if ( + not composition_available + or expected_tool is None + or not isinstance(outer_code_mode, int) + or isinstance(outer_code_mode, bool) + or outer_code_mode <= 0 + or not isinstance(outer_by_name, dict) + ): + return False + code_mode_counts = { + name: count + for name in report.CODE_MODE_TOOLS + if isinstance((count := outer_by_name.get(name, 0)), int) + and not isinstance(count, bool) + and count > 0 + } return ( - report._get_path(summary, "availability.code_mode_composition.available") is True - and isinstance(outer_code_mode, int) - and not isinstance(outer_code_mode, bool) - and outer_code_mode > 0 + code_mode_counts.get(expected_tool) == outer_code_mode + and sum(code_mode_counts.values()) == outer_code_mode ) return False @@ -468,6 +577,27 @@ def _guarded_multi_file_contract_proven(summary: dict[str, Any], variant: str) - for name in ("search_project_texts", "search_and_read") if (count := _tool_count(summary, variant, name)) is not None ] + outer_failed = report._get_path(summary, "outer_calls.failed") + outer_unknown = report._get_path(summary, "outer_calls.timeout_or_unknown") + if ( + not isinstance(outer_failed, int) + or isinstance(outer_failed, bool) + or outer_failed != 0 + or not isinstance(outer_unknown, int) + or isinstance(outer_unknown, bool) + or outer_unknown != 0 + ): + return False + + if variant == "code_mode": + child_failed = report._get_path(summary, "child_calls.failed") + if ( + not isinstance(child_failed, int) + or isinstance(child_failed, bool) + or child_failed != 0 + ): + return False + return ( edit_count == 1 and read_count is not None @@ -586,7 +716,7 @@ def _run_sample( if ( summary is not None and effective_status == "pass" - and not _runtime_evidence_proven(summary, variant) + and not _runtime_evidence_proven(summary, variant, surface) ): effective_status = "partial" diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index 400c8c163..7051b1ed7 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -77,19 +77,30 @@ def test_runtime_evidence_gate_fails_closed(self) -> None: direct = { "outer_calls": {"total": 1}, "canonical_calls": {"total": 1}, + "availability": {"code_mode_composition": {"available": True}}, + "tools": {"outer_by_name": {"read_files": 1}}, } - self.assertTrue(benchmark._runtime_evidence_proven(direct, "direct")) - direct["outer_calls"]["total"] = 0 - self.assertFalse(benchmark._runtime_evidence_proven(direct, "direct")) + self.assertTrue(benchmark._runtime_evidence_proven(direct, "direct", "direct")) + direct["availability"]["code_mode_composition"]["available"] = False + self.assertFalse(benchmark._runtime_evidence_proven(direct, "direct", "direct")) code_mode = { "outer_calls": {"total": 1}, "availability": {"code_mode_composition": {"available": True}}, "composition": {"outer_code_mode_calls": 1}, + "tools": {"outer_by_name": {"execute_mutating_code_mode": 1}}, } - self.assertTrue(benchmark._runtime_evidence_proven(code_mode, "code_mode")) - code_mode["availability"]["code_mode_composition"]["available"] = False - self.assertFalse(benchmark._runtime_evidence_proven(code_mode, "code_mode")) + self.assertTrue( + benchmark._runtime_evidence_proven(code_mode, "code_mode", "guarded_edit") + ) + self.assertFalse( + benchmark._runtime_evidence_proven(code_mode, "code_mode", "read_only") + ) + code_mode["tools"]["outer_by_name"]["execute_code_mode"] = 1 + code_mode["composition"]["outer_code_mode_calls"] = 2 + self.assertFalse( + benchmark._runtime_evidence_proven(code_mode, "code_mode", "guarded_edit") + ) def test_long_handoff_requires_correlated_terminal_evidence(self) -> None: summary = { @@ -114,9 +125,13 @@ def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(sel "search_project_texts": 1, "read_files": 2, } - } + }, + "outer_calls": {"failed": 0, "timeout_or_unknown": 0}, } self.assertTrue(benchmark._guarded_multi_file_contract_proven(direct, "direct")) + direct["outer_calls"]["failed"] = 1 + self.assertFalse(benchmark._guarded_multi_file_contract_proven(direct, "direct")) + direct["outer_calls"]["failed"] = 0 direct["canonical_calls"]["by_name"]["edit_project_files"] = 2 self.assertFalse(benchmark._guarded_multi_file_contract_proven(direct, "direct")) @@ -129,8 +144,13 @@ def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(sel "read_files": 2, } }, + "outer_calls": {"failed": 0, "timeout_or_unknown": 0}, + "child_calls": {"failed": 0}, } self.assertTrue(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) + code_mode["child_calls"]["failed"] = 1 + self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) + code_mode["child_calls"]["failed"] = 0 code_mode["composition"]["nested_tool_counts"]["read_files"] = 1 self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) @@ -330,6 +350,48 @@ def test_relative_receipt_artifacts_are_anchored_to_workspace(self) -> None: self.assertEqual(kwargs["audit_db"], root / ".webcodex/action_audit.db") self.assertEqual(kwargs["trace_root"], root / ".webcodex/traces") + def test_fixture_oracle_bounds_changed_path_listing(self) -> None: + manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + case = report._case_by_id(manifest, "guarded_multi_file_edit") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(["git", "init", "-b", "main"], cwd=root, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=root, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=root, check=True) + fixture = root / "tests/fixtures/agent-loop-multi-file" + fixture.mkdir(parents=True) + (fixture / "alpha.txt").write_text( + "BENCH_TARGET_ALPHA=before\nALPHA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=before\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-m", "fixture"], cwd=root, check=True, stdout=subprocess.DEVNULL) + (fixture / "alpha.txt").write_text( + "BENCH_TARGET_ALPHA=after\nALPHA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=after\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + overflow = root / "overflow" + overflow.mkdir() + for index in range(benchmark.MAX_GIT_PATHS + 1): + (overflow / f"path-{index:04d}.txt").write_text("x", encoding="utf-8") + + result = benchmark._fixture_oracle(case, root) + changed = next( + check for check in result["checks"] if check.get("kind") == "changed_files" + ) + self.assertFalse(result["passed"]) + self.assertFalse(changed["passed"]) + self.assertEqual(changed["reason"], "changed path listing exceeds oracle limit") + self.assertIsNone(changed["actual"]) + def test_missing_receipts_are_retained_as_failed_samples(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -394,6 +456,53 @@ def test_unsupported_driver_samples_are_retained_and_worktrees_cleaned(self) -> worktrees = subprocess.check_output(["git", "worktree", "list", "--porcelain"], cwd=repo, text=True) self.assertEqual(worktrees.count("worktree "), 1) + @unittest.skipIf(os.name == "nt", "post-checkout hook regression is POSIX-only") + def test_failed_worktree_checkout_rolls_back_owned_registration(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + repo = root / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + subprocess.run(["git", "config", "user.email", "bench@test.local"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Bench"], cwd=repo, check=True) + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + self._install_manifest(repo) + subprocess.run(["git", "add", "."], cwd=repo, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=repo, check=True, stdout=subprocess.DEVNULL) + base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip() + + unrelated = root / "unrelated" + subprocess.run( + ["git", "worktree", "add", "--detach", str(unrelated), base], + cwd=repo, + check=True, + stdout=subprocess.DEVNULL, + ) + hook = repo / ".git/hooks/post-checkout" + hook.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + hook.chmod(0o755) + + result = benchmark.run_benchmark( + repo=repo, + base_revision=base, + driver_argv=[sys.executable, "-c", "pass"], + case_ids=["readonly_review"], + pairs=1, + ) + self.assertEqual(result["status_counts"]["fail"], 2) + worktrees = subprocess.check_output( + ["git", "worktree", "list", "--porcelain"], + cwd=repo, + text=True, + ) + self.assertIn(str(unrelated), worktrees) + self.assertNotIn("webcodex-agent-loop-bench-", worktrees) + subprocess.run( + ["git", "worktree", "remove", "--force", str(unrelated)], + cwd=repo, + check=True, + ) + def test_cleanup_does_not_prune_unrelated_worktree(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From b5726da057c07f1c5d8843d9c0cc69ac1c7608cc Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:06:24 +0800 Subject: [PATCH 11/12] test(agent-loop): close remaining evidence gaps --- docs/experiments/agent-loop-baseline.md | 22 ++-- scripts/agent_loop_benchmark.py | 118 +++++++++++++++++--- scripts/agent_loop_report.py | 67 ++++++++++- scripts/tests/test_agent_loop_benchmark.py | 124 +++++++++++++++++++-- scripts/tests/test_agent_loop_report.py | 59 ++++++++++ 5 files changed, 356 insertions(+), 34 deletions(-) diff --git a/docs/experiments/agent-loop-baseline.md b/docs/experiments/agent-loop-baseline.md index ad6e54588..23c8764b5 100644 --- a/docs/experiments/agent-loop-baseline.md +++ b/docs/experiments/agent-loop-baseline.md @@ -302,15 +302,23 @@ independent fixture oracle can establish the task result. For non-oracle cases, driver-reported pass needs `correctness.task_verdict="pass"`; validation-required cases also need the corresponding passing validation verdict, otherwise the sample remains partial. The durable benchmark result does not persist runtime Project, -Window, tunnel, or Host-private ids. +Window, tunnel, or Host-private ids. For the Code Mode validation-handoff +case only, the temporary driver receipt also carries bounded +`job_identity_evidence` with the exact child handoff Job id and terminal-observe +Job id. The orchestrator compares those values and immediately discards them with +the temporary run state; neither id is copied into the annotation, sample result, or +durable benchmark JSON. The default lane is exactly `readonly_review`, `guarded_multi_file_edit`, and -`long_validation_handoff`, matching #822. The handoff case is valid only when the -report proves exactly one genuine pending validation, a correlated follow-up for -that relation, and a same-relation terminal observation with authoritative terminal -time. A synchronous completion, an uncorrelated/substitute process, or pending -without terminal evidence remains `partial`/`unsupported`, never upgraded to success. -The guarded multi-file case is likewise not a pass from final bytes alone: persisted Direct canonical-call or Code Mode composition evidence must show exactly one `edit_project_files` call, at least one admitted search call, and at least two `read_files` calls (pre-edit and post-edit phases). The outer capture must contain no failed/unknown calls, and Code Mode must report zero nested failures, so a failed guarded edit followed by a shell/process fallback cannot be promoted to a pass by the final-byte oracle. Current composition does not prove nested child ordering, so the benchmark does not claim stronger sequencing evidence than is persisted. +`long_validation_handoff`, matching #822. The handoff case requires exactly one +`cargo_test` launch. Direct must prove one canonical pending relation, a correlated +follow-up, and same-relation terminal timing from ActionAudit. Code Mode must prove +one validator child and one Job handoff in persisted composition, then match the +exact transient child Job id to the terminal `observe_jobs` Job id from the Host +receipt. A synchronous completion, redispatch, uncorrelated/substitute process, or +pending without terminal evidence remains `partial`/`unsupported`, never upgraded +to success. +The guarded multi-file case is likewise not a pass from final bytes alone: persisted Direct canonical-call or Code Mode composition evidence must show exactly one `edit_project_files` call, at least one admitted search call, and at least two `read_files` calls (pre-edit and post-edit phases). Direct additionally requires the existing authoritative ModelErgonomics edit outcome to be exactly `applied`. Code Mode composition does not persist per-child edit outcomes, so it instead requires exactly one consequential known-result child, zero Job handoffs/outcome-unknown/nested failures, and the fixture-side exact-byte oracle; a dry-run edit plus shell/process/write fallback therefore cannot be promoted to a pass. Current composition does not prove nested child ordering, so the benchmark does not claim stronger sequencing evidence than is persisted. This command coordinates real captures; it is not itself evidence that a model made fewer inference turns. Deterministic runtime/script acceptance and fixture oracles diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index 733f74393..adeefc04d 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -34,6 +34,7 @@ MAX_DRIVER_RECEIPT_BYTES = 64 * 1024 MAX_FIXTURE_ORACLE_BYTES = 1024 * 1024 MAX_CASE_MANIFEST_BYTES = 1024 * 1024 +MAX_JOB_ID_BYTES = 256 MAX_GIT_PATH_BYTES = 64 * 1024 MAX_GIT_PATHS = 512 CODE_MODE_TOOL_BY_SURFACE = { @@ -398,11 +399,15 @@ def _load_driver_receipt(path: Path) -> dict[str, Any]: if len(raw) > MAX_DRIVER_RECEIPT_BYTES: raise BenchmarkError("driver result exceeds the 64 KiB receipt limit") try: - value = json.loads(raw.decode("utf-8")) + text = raw.decode("utf-8") except UnicodeDecodeError as exc: raise BenchmarkError("driver result must be UTF-8 JSON") from exc + try: + value = json.loads(text) except json.JSONDecodeError as exc: raise BenchmarkError(f"driver result is not valid JSON: {exc}") from exc + except (RecursionError, ValueError) as exc: + raise BenchmarkError("driver result is not valid bounded JSON") from exc if not isinstance(value, dict): raise BenchmarkError("driver result must be a JSON object") status = value.get("status") @@ -433,6 +438,13 @@ def _annotation( if case["validation"]["required"] is False and correctness.get("validation_verdict") is None: correctness["validation_verdict"] = "not_required" + if correctness: + correctness.setdefault("task_verdict", "unavailable") + correctness.setdefault( + "validation_verdict", + "unavailable" if case["validation"]["required"] else "not_required", + ) + value: dict[str, Any] = { "schema_version": report.RUN_ANNOTATION_SCHEMA_VERSION, "case_id": case["id"], @@ -504,6 +516,36 @@ def _tool_count(summary: dict[str, Any], variant: str, tool_name: str) -> int | return None +def _bounded_job_id(value: Any) -> str | None: + if not isinstance(value, str) or not value: + return None + try: + encoded = value.encode("utf-8") + except UnicodeEncodeError: + return None + if len(encoded) > MAX_JOB_ID_BYTES: + return None + return value + + +def _job_identity_evidence_proven(receipt: dict[str, Any]) -> bool: + evidence = receipt.get("job_identity_evidence") + if not isinstance(evidence, dict) or set(evidence) != { + "handoff_job_id", + "terminal_job_id", + "terminal", + }: + return False + handoff = _bounded_job_id(evidence.get("handoff_job_id")) + terminal = _bounded_job_id(evidence.get("terminal_job_id")) + return ( + handoff is not None + and terminal is not None + and handoff == terminal + and evidence.get("terminal") is True + ) + + def _runtime_evidence_proven( summary: dict[str, Any], variant: str, @@ -552,21 +594,46 @@ def _runtime_evidence_proven( return False -def _long_handoff_terminal_proven(summary: dict[str, Any]) -> bool: - pending = report._get_path(summary, "job_convergence.pending_handoff_count") - known_followups = report._get_path(summary, "job_convergence.pending_followup_known_count") - terminal_samples = report._get_path(summary, "job_convergence.pending_to_terminal_ms.samples") - return ( - isinstance(pending, int) - and not isinstance(pending, bool) - and pending == 1 - and isinstance(known_followups, int) - and not isinstance(known_followups, bool) - and known_followups >= 1 - and isinstance(terminal_samples, int) - and not isinstance(terminal_samples, bool) - and terminal_samples >= 1 - ) +def _long_handoff_terminal_proven( + summary: dict[str, Any], + variant: str, + receipt: dict[str, Any], +) -> bool: + if _tool_count(summary, variant, "cargo_test") != 1: + return False + + if variant == "direct": + pending = report._get_path(summary, "job_convergence.pending_handoff_count") + known_followups = report._get_path(summary, "job_convergence.pending_followup_known_count") + terminal_samples = report._get_path(summary, "job_convergence.pending_to_terminal_ms.samples") + return ( + isinstance(pending, int) + and not isinstance(pending, bool) + and pending == 1 + and isinstance(known_followups, int) + and not isinstance(known_followups, bool) + and known_followups >= 1 + and isinstance(terminal_samples, int) + and not isinstance(terminal_samples, bool) + and terminal_samples >= 1 + ) + + if variant == "code_mode": + job_handoffs = report._get_path(summary, "composition.job_handoffs.total") + consequential = report._get_path(summary, "composition.consequential_calls.total") + outcome_unknown = report._get_path(summary, "composition.outcome_unknown.total") + outer_observes = report._get_path(summary, "tools.outer_by_name.observe_jobs") + return ( + job_handoffs == 1 + and consequential == 1 + and outcome_unknown == 0 + and isinstance(outer_observes, int) + and not isinstance(outer_observes, bool) + and outer_observes >= 1 + and _job_identity_evidence_proven(receipt) + ) + + return False def _guarded_multi_file_contract_proven(summary: dict[str, Any], variant: str) -> bool: @@ -589,12 +656,29 @@ def _guarded_multi_file_contract_proven(summary: dict[str, Any], variant: str) - ): return False + if variant == "direct": + if report._get_path(summary, "availability.edit_outcomes.available") is not True: + return False + if report._get_path( + summary, + "edit_outcomes.by_tool.edit_project_files.applied", + ) != 1: + return False + if variant == "code_mode": child_failed = report._get_path(summary, "child_calls.failed") + consequential = report._get_path(summary, "composition.consequential_calls.total") + known_results = report._get_path(summary, "composition.known_results.total") + job_handoffs = report._get_path(summary, "composition.job_handoffs.total") + outcome_unknown = report._get_path(summary, "composition.outcome_unknown.total") if ( not isinstance(child_failed, int) or isinstance(child_failed, bool) or child_failed != 0 + or consequential != 1 + or known_results != 1 + or job_handoffs != 0 + or outcome_unknown != 0 ): return False @@ -724,7 +808,7 @@ def _run_sample( case["id"] == "long_validation_handoff" and summary is not None and effective_status == "pass" - and not _long_handoff_terminal_proven(summary) + and not _long_handoff_terminal_proven(summary, variant, receipt) ): effective_status = "partial" diff --git a/scripts/agent_loop_report.py b/scripts/agent_loop_report.py index a88f3f5a4..ed3e68da5 100644 --- a/scripts/agent_loop_report.py +++ b/scripts/agent_loop_report.py @@ -71,6 +71,16 @@ "run_process_bash_lc_to_login_run_shell", )) INPUT_NORMALIZATION_SCHEMA_VERSION = 13 +EDIT_OUTCOME_SCHEMA_VERSION = 13 +EDIT_OUTCOMES = frozenset(( + "applied", + "dry_run_would_change", + "dry_run_no_change", + "no_change", + "conflict", + "uncertain", + "rejected", +)) class ReportError(ValueError): @@ -151,11 +161,17 @@ def validate_case_manifest(value: Any) -> dict[str, Any]: def load_case_manifest(path: Path) -> dict[str, Any]: try: - value = json.loads(path.read_text(encoding="utf-8")) + text = path.read_text(encoding="utf-8") except OSError as exc: raise ReportError(f"could not read case manifest: {path}") from exc + except UnicodeDecodeError as exc: + raise ReportError(f"case manifest must be UTF-8: {path}") from exc + try: + value = json.loads(text) except json.JSONDecodeError as exc: raise ReportError(f"case manifest is not valid JSON: {path}:{exc.lineno}") from exc + except (RecursionError, ValueError) as exc: + raise ReportError(f"case manifest is not valid bounded JSON: {path}") from exc return validate_case_manifest(value) @@ -1513,11 +1529,31 @@ def _summarize_audit( failure_kinds: Counter[str] = Counter() recovery_guidance: Counter[str] = Counter() error_kinds: Counter[str] = Counter() + edit_outcomes: Counter[str] = Counter() + edit_outcomes_by_tool: dict[str, Counter[str]] = {} + unrecognized_edit_outcomes = 0 for _, value in present_telemetries: for field, counter in (("failure_kind", failure_kinds), ("recovery_kind", recovery_guidance), ("error_kind", error_kinds)): item = value.get(field) if isinstance(item, str) and item: counter[item] += 1 + outcome = value.get("edit_outcome") + if outcome is None: + continue + schema_version = value.get("schema_version") + if ( + not isinstance(schema_version, int) + or isinstance(schema_version, bool) + or schema_version < EDIT_OUTCOME_SCHEMA_VERSION + or not isinstance(outcome, str) + or outcome not in EDIT_OUTCOMES + ): + unrecognized_edit_outcomes += 1 + continue + edit_outcomes[outcome] += 1 + tool_name = value.get("tool_name") + if isinstance(tool_name, str) and tool_name: + edit_outcomes_by_tool.setdefault(tool_name, Counter())[outcome] += 1 gaps, overlap_count, missing_serial = _window_timing(outer, continuity_events) host_short_chain, host_short_chain_availability = _summarize_host_short_chains( @@ -1553,6 +1589,15 @@ def _summarize_audit( "observed_outer_runtime_records": canonical_observed, "by_name": dict(sorted(canonical_by_name.items())), }, + "edit_outcomes": { + "observed": sum(edit_outcomes.values()), + "unrecognized": unrecognized_edit_outcomes, + "by_outcome": dict(sorted(edit_outcomes.items())), + "by_tool": { + tool: dict(sorted(counts.items())) + for tool, counts in sorted(edit_outcomes_by_tool.items()) + }, + }, "composition": composition, "host_short_chain": host_short_chain, "child_calls": { @@ -1599,6 +1644,16 @@ def _summarize_audit( "tool_runtime_timing": {"available": runtime_duration["total"] is not None, "reason": None if runtime_duration["total"] is not None else "one or more outer calls lack ModelErgonomics runtime duration evidence"}, "window_timing": {"available": missing_serial == 0, "reason": None if missing_serial == 0 else "one or more canonical serial transitions lack the predecessor timestamps needed for a gap"}, "canonical_calls": {"available": canonical_total is not None, "reason": canonical_reason}, + "edit_outcomes": { + "available": bool(edit_outcomes) and unrecognized_edit_outcomes == 0, + "reason": ( + None + if edit_outcomes and unrecognized_edit_outcomes == 0 + else "no complete authoritative ModelErgonomics edit outcome evidence" + if not edit_outcomes + else "one or more edit outcome labels are unknown or malformed" + ), + }, "code_mode_composition": composition_availability, "host_short_chain": host_short_chain_availability, "host_same_model_turn_identity": { @@ -1663,6 +1718,12 @@ def _summarize_trace_only(trace_events: list[dict[str, Any]], variant: str | Non "outer_calls": {"total": len(handlers), "meaningful": None, "successful": statuses.get("success", 0), "failed": statuses.get("failed", 0), "timeout_or_unknown": statuses.get("unknown", 0), "by_status": dict(sorted(statuses.items()))}, "tools": {"outer_by_name": dict(sorted(tools.items()))}, "canonical_calls": {"total": None, "observed_outer_runtime_records": None, "by_name": {}}, + "edit_outcomes": { + "observed": 0, + "unrecognized": 0, + "by_outcome": {}, + "by_tool": {}, + }, "composition": composition, "host_short_chain": { "serial_transitions": None, @@ -1685,6 +1746,10 @@ def _summarize_trace_only(trace_events: list[dict[str, Any]], variant: str | Non "failures": {"error_kind_by_name": {}, "failure_kind_by_name": {}, "recovery_guidance_by_kind": {}, "child_failure_kind_by_name": None, "resolved_recoveries": None}, "availability": { "action_audit": {"available": False, "reason": "no ActionAudit DB evidence was provided"}, + "edit_outcomes": { + "available": False, + "reason": "trace metadata does not persist authoritative edit outcome evidence", + }, "model_round_trips": { "available": False, "reason": "events.jsonl does not persist model response/turn identity", diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index 7051b1ed7..2f49de507 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -102,20 +102,52 @@ def test_runtime_evidence_gate_fails_closed(self) -> None: benchmark._runtime_evidence_proven(code_mode, "code_mode", "guarded_edit") ) - def test_long_handoff_requires_correlated_terminal_evidence(self) -> None: - summary = { + def test_long_handoff_requires_one_launch_and_exact_terminal_identity(self) -> None: + direct = { + "canonical_calls": {"by_name": {"cargo_test": 1}}, "job_convergence": { "pending_handoff_count": 1, "pending_followup_known_count": 1, "pending_to_terminal_ms": {"samples": 1}, + }, + } + self.assertTrue( + benchmark._long_handoff_terminal_proven(direct, "direct", {}) + ) + direct["canonical_calls"]["by_name"]["cargo_test"] = 2 + self.assertFalse( + benchmark._long_handoff_terminal_proven(direct, "direct", {}) + ) + + code_mode = { + "availability": {"code_mode_composition": {"available": True}}, + "composition": { + "nested_tool_counts": {"cargo_test": 1}, + "job_handoffs": {"total": 1}, + "consequential_calls": {"total": 1}, + "outcome_unknown": {"total": 0}, + }, + "tools": {"outer_by_name": {"execute_effectful_code_mode": 1, "observe_jobs": 1}}, + } + receipt = { + "job_identity_evidence": { + "handoff_job_id": "job-1", + "terminal_job_id": "job-1", + "terminal": True, } } - self.assertTrue(benchmark._long_handoff_terminal_proven(summary)) - summary["job_convergence"]["pending_to_terminal_ms"]["samples"] = 0 - self.assertFalse(benchmark._long_handoff_terminal_proven(summary)) - summary["job_convergence"]["pending_to_terminal_ms"]["samples"] = 1 - summary["job_convergence"]["pending_handoff_count"] = 2 - self.assertFalse(benchmark._long_handoff_terminal_proven(summary)) + self.assertTrue( + benchmark._long_handoff_terminal_proven(code_mode, "code_mode", receipt) + ) + receipt["job_identity_evidence"]["terminal_job_id"] = "job-2" + self.assertFalse( + benchmark._long_handoff_terminal_proven(code_mode, "code_mode", receipt) + ) + receipt["job_identity_evidence"]["terminal_job_id"] = "job-1" + code_mode["composition"]["nested_tool_counts"]["cargo_test"] = 2 + self.assertFalse( + benchmark._long_handoff_terminal_proven(code_mode, "code_mode", receipt) + ) def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(self) -> None: direct = { @@ -127,11 +159,20 @@ def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(sel } }, "outer_calls": {"failed": 0, "timeout_or_unknown": 0}, + "edit_outcomes": { + "by_tool": {"edit_project_files": {"applied": 1}}, + }, + "availability": {"edit_outcomes": {"available": True}}, } self.assertTrue(benchmark._guarded_multi_file_contract_proven(direct, "direct")) direct["outer_calls"]["failed"] = 1 self.assertFalse(benchmark._guarded_multi_file_contract_proven(direct, "direct")) direct["outer_calls"]["failed"] = 0 + direct["edit_outcomes"]["by_tool"]["edit_project_files"] = { + "dry_run_would_change": 1 + } + self.assertFalse(benchmark._guarded_multi_file_contract_proven(direct, "direct")) + direct["edit_outcomes"]["by_tool"]["edit_project_files"] = {"applied": 1} direct["canonical_calls"]["by_name"]["edit_project_files"] = 2 self.assertFalse(benchmark._guarded_multi_file_contract_proven(direct, "direct")) @@ -142,7 +183,11 @@ def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(sel "edit_project_files": 1, "search_project_texts": 1, "read_files": 2, - } + }, + "consequential_calls": {"total": 1}, + "known_results": {"total": 1}, + "job_handoffs": {"total": 0}, + "outcome_unknown": {"total": 0}, }, "outer_calls": {"failed": 0, "timeout_or_unknown": 0}, "child_calls": {"failed": 0}, @@ -151,6 +196,9 @@ def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(sel code_mode["child_calls"]["failed"] = 1 self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) code_mode["child_calls"]["failed"] = 0 + code_mode["composition"]["consequential_calls"]["total"] = 2 + self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) + code_mode["composition"]["consequential_calls"]["total"] = 1 code_mode["composition"]["nested_tool_counts"]["read_files"] = 1 self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) @@ -279,6 +327,34 @@ def test_malformed_receipt_status_is_a_contract_error(self) -> None: with self.assertRaisesRegex(benchmark.BenchmarkError, "status must be"): benchmark._load_driver_receipt(receipt) + def test_driver_receipt_pathological_json_is_a_contract_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name, raw in ( + ("deep.json", '{"status":"unsupported","extra":' + "[" * 10000 + "0" + "]" * 10000 + "}"), + ("huge-int.json", '{"status":"unsupported","extra":' + "1" * 10000 + "}"), + ): + receipt = root / name + receipt.write_text(raw, encoding="utf-8") + with self.assertRaisesRegex(benchmark.BenchmarkError, "bounded JSON"): + benchmark._load_driver_receipt(receipt) + + def test_job_identity_evidence_is_exact_bounded_and_private(self) -> None: + receipt = { + "job_identity_evidence": { + "handoff_job_id": "private-job-1", + "terminal_job_id": "private-job-1", + "terminal": True, + } + } + self.assertTrue(benchmark._job_identity_evidence_proven(receipt)) + receipt["job_identity_evidence"]["terminal_job_id"] = "other-job" + self.assertFalse(benchmark._job_identity_evidence_proven(receipt)) + receipt["job_identity_evidence"]["terminal_job_id"] = "x" * ( + benchmark.MAX_JOB_ID_BYTES + 1 + ) + self.assertFalse(benchmark._job_identity_evidence_proven(receipt)) + @unittest.skipIf(os.name == "nt", "FIFO is POSIX-only") def test_driver_receipt_rejects_fifo_without_blocking(self) -> None: with tempfile.TemporaryDirectory() as tmp: @@ -317,6 +393,36 @@ def test_comparison_requires_two_effective_pass_samples(self) -> None: ) ) + def test_annotation_normalizes_omitted_correctness_verdicts(self) -> None: + manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + readonly = report._case_by_id(manifest, "readonly_review") + annotation = benchmark._annotation( + readonly, + variant="direct", + surface="direct", + base_revision="a" * 40, + receipt={"correctness": {"task_verdict": "pass"}}, + oracle={"available": False, "passed": None, "checks": [], "reason": None}, + ) + self.assertEqual( + annotation["correctness"], + {"task_verdict": "pass", "validation_verdict": "not_required"}, + ) + + validation = report._case_by_id(manifest, "long_validation_handoff") + annotation = benchmark._annotation( + validation, + variant="direct", + surface="direct", + base_revision="a" * 40, + receipt={"correctness": {"task_verdict": "pass"}}, + oracle={"available": False, "passed": None, "checks": [], "reason": None}, + ) + self.assertEqual( + annotation["correctness"], + {"task_verdict": "pass", "validation_verdict": "unavailable"}, + ) + def test_relative_receipt_artifacts_are_anchored_to_workspace(self) -> None: manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) case = report._case_by_id(manifest, "readonly_review") diff --git a/scripts/tests/test_agent_loop_report.py b/scripts/tests/test_agent_loop_report.py index 4f14e0005..6bf05641e 100644 --- a/scripts/tests/test_agent_loop_report.py +++ b/scripts/tests/test_agent_loop_report.py @@ -98,6 +98,7 @@ def insert_event( readiness: dict[str, object] | None = None, job_convergence: dict[str, object] | None = None, error_kind: str | None = None, + edit_outcome: object | None = None, normalization_code: object | None = None, schema_version: int = 12, ) -> None: @@ -118,6 +119,8 @@ def insert_event( ) if normalization_code is not None: telemetry["input_normalization_code"] = normalization_code + if edit_outcome is not None: + telemetry["edit_outcome"] = edit_outcome if readiness is not None: telemetry["readiness"] = readiness if job_convergence is not None: @@ -586,6 +589,46 @@ def test_failed_call_and_result_bytes_are_aggregated_from_telemetry(self) -> Non ) self.assertIsNone(result["failures"]["resolved_recoveries"]) + def test_edit_outcomes_are_aggregated_from_authoritative_telemetry(self) -> None: + self.insert_event( + "applied", + tool="edit_project_files", + schema_version=13, + edit_outcome="applied", + ) + self.insert_event( + "dry-run", + tool="edit_project_files", + schema_version=13, + edit_outcome="dry_run_would_change", + started=150, + handed=170, + transition="serial", + ) + + result = self.summarize() + self.assertEqual(result["edit_outcomes"]["observed"], 2) + self.assertEqual(result["edit_outcomes"]["unrecognized"], 0) + self.assertEqual( + result["edit_outcomes"]["by_tool"]["edit_project_files"], + {"applied": 1, "dry_run_would_change": 1}, + ) + self.assertTrue(result["availability"]["edit_outcomes"]["available"]) + + self.insert_event( + "unknown", + tool="edit_project_files", + schema_version=13, + edit_outcome="PRIVATE_UNKNOWN", + started=200, + handed=220, + transition="serial", + ) + result = self.summarize() + self.assertEqual(result["edit_outcomes"]["unrecognized"], 1) + self.assertFalse(result["availability"]["edit_outcomes"]["available"]) + self.assertNotIn("PRIVATE_UNKNOWN", json.dumps(result)) + def test_missing_optional_result_bytes_never_become_zero(self) -> None: self.insert_event("known", result_bytes=90) self.insert_event( @@ -1158,6 +1201,22 @@ def test_manifest_validation_rejects_duplicate_case_ids(self) -> None: with self.assertRaisesRegex(report.ReportError, "duplicate case id"): report.validate_case_manifest(manifest) + def test_manifest_loader_fail_closes_on_pathological_json(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + deep = root / "deep.json" + deep.write_text( + '{"schema_version":1,"cases":' + "[" * 10000 + "0" + "]" * 10000 + "}", + encoding="utf-8", + ) + with self.assertRaisesRegex(report.ReportError, "bounded JSON"): + report.load_case_manifest(deep) + + huge_int = root / "huge-int.json" + huge_int.write_text("{" + '"schema_version":' + "1" * 10000 + "}", encoding="utf-8") + with self.assertRaisesRegex(report.ReportError, "bounded JSON"): + report.load_case_manifest(huge_int) + def test_manifest_requires_fields_and_stable_serialization(self) -> None: manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) serialized_once = report._stable_json(manifest) From b543a416d9e09ba0d5486219a8bc9bd583bf638e Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:02:21 +0800 Subject: [PATCH 12/12] fix(agent-loop): bind benchmark evidence to canonical runtime facts --- docs/experiments/agent-loop-baseline.md | 36 ++-- scripts/agent_loop_benchmark.py | 169 +++++++++------ scripts/agent_loop_report.py | 129 +++++++++-- scripts/tests/test_agent_loop_benchmark.py | 156 ++++++++++---- scripts/tests/test_agent_loop_report.py | 167 ++++++++++++++- src/mcp_tests/model_ergonomics.rs | 4 + .../tests/window_activity.rs | 45 +++- .../job_convergence.rs | 69 +++++- src/tool_runtime/orchestration_host.rs | 16 ++ src/tool_runtime/runtime_metrics.rs | 2 + .../tests/job_attention/telemetry.rs | 201 ++++++++++++++++++ .../window_activity_projection.rs | 9 + 12 files changed, 858 insertions(+), 145 deletions(-) diff --git a/docs/experiments/agent-loop-baseline.md b/docs/experiments/agent-loop-baseline.md index 23c8764b5..db5738f62 100644 --- a/docs/experiments/agent-loop-baseline.md +++ b/docs/experiments/agent-loop-baseline.md @@ -275,7 +275,7 @@ only reproducibility mechanics: - fixture-side disk/diff oracles where the manifest provides one; changed-path capture is bounded to 64 KiB / 512 paths, and the multi-file case additionally requires persisted evidence for one successful guarded edit plus its search/read phases; - explicit `pass`, `fail`, `partial`, and `unsupported` samples. None are silently filtered from the aggregate; -- non-unsupported `pass` requires non-empty canonical ActionAudit evidence and the audited execution lane must match the requested variant/surface (`direct`, `execute_code_mode`, `execute_effectful_code_mode`, or `execute_mutating_code_mode` as applicable); Code Mode also requires complete persisted composition evidence; missing or mismatched capture is `partial`, never fabricated success; +- non-unsupported `pass` requires non-empty canonical ActionAudit evidence and the audited execution lane must match the requested variant/surface (`direct`, `execute_code_mode`, `execute_effectful_code_mode`, or `execute_mutating_code_mode` as applicable); the read-only Code Mode lane additionally permits exactly one required `work_on_project` bootstrap and at most one `read_tool_manifest` contract-discovery call, but no Direct read/search fallback; Code Mode also requires complete persisted composition evidence; missing or mismatched capture is `partial`, never fabricated success; - cleanup of only the worktrees/temp state created by the orchestrator, including targeted rollback of an owned worktree registration when checkout setup fails; no global `git worktree prune` is used; - Host-driver process ownership currently uses POSIX process groups; Windows runs are retained explicitly as `unsupported` until equivalent descendant ownership can be guaranteed. @@ -294,7 +294,7 @@ For each invocation the driver receives `WEBCODEX_BENCH_WORKSPACE`, `WEBCODEX_BENCH_SURFACE`, `WEBCODEX_BENCH_BASE_REVISION`, `WEBCODEX_BENCH_CASE_FINGERPRINT`, `WEBCODEX_BENCH_PROMPT`, and `WEBCODEX_BENCH_DRIVER_RESULT`. It writes one bounded JSON receipt to the last -path. The receipt is capped at 64 KiB; driver stdout/stderr is not retained by the orchestrator. `status` is required and is one of `pass/fail/partial/unsupported`. Relative `audit_db` and `trace_root` paths are resolved against the fresh sample workspace; absolute paths remain absolute. +path. The receipt is capped at 64 KiB; malformed/decode-pathological JSON fails closed as a retained failed sample, and driver stdout/stderr is not retained by the orchestrator. `status` is required and is one of `pass/fail/partial/unsupported`. Relative `audit_db` and `trace_root` paths are resolved against the fresh sample workspace; absolute paths remain absolute. Non-unsupported samples also provide the authoritative ActionAudit database path and Workflow Session id; optional trace root, repair-turn annotation, and task timing use the existing reporter semantics. Correctness verdicts are optional only when an @@ -302,23 +302,26 @@ independent fixture oracle can establish the task result. For non-oracle cases, driver-reported pass needs `correctness.task_verdict="pass"`; validation-required cases also need the corresponding passing validation verdict, otherwise the sample remains partial. The durable benchmark result does not persist runtime Project, -Window, tunnel, or Host-private ids. For the Code Mode validation-handoff -case only, the temporary driver receipt also carries bounded -`job_identity_evidence` with the exact child handoff Job id and terminal-observe -Job id. The orchestrator compares those values and immediately discards them with -the temporary run state; neither id is copied into the annotation, sample result, or -durable benchmark JSON. +Window, tunnel, Job, or Host-private ids. Job identity is correlated only through +the existing payload-safe salted ActionAudit relation; the benchmark driver does not +supply or certify a second Job-identity receipt. The default lane is exactly `readonly_review`, `guarded_multi_file_edit`, and `long_validation_handoff`, matching #822. The handoff case requires exactly one -`cargo_test` launch. Direct must prove one canonical pending relation, a correlated -follow-up, and same-relation terminal timing from ActionAudit. Code Mode must prove -one validator child and one Job handoff in persisted composition, then match the -exact transient child Job id to the terminal `observe_jobs` Job id from the Host -receipt. A synchronous completion, redispatch, uncorrelated/substitute process, or -pending without terminal evidence remains `partial`/`unsupported`, never upgraded -to success. -The guarded multi-file case is likewise not a pass from final bytes alone: persisted Direct canonical-call or Code Mode composition evidence must show exactly one `edit_project_files` call, at least one admitted search call, and at least two `read_files` calls (pre-edit and post-edit phases). Direct additionally requires the existing authoritative ModelErgonomics edit outcome to be exactly `applied`. Code Mode composition does not persist per-child edit outcomes, so it instead requires exactly one consequential known-result child, zero Job handoffs/outcome-unknown/nested failures, and the fixture-side exact-byte oracle; a dry-run edit plus shell/process/write fallback therefore cannot be promoted to a pass. Current composition does not prove nested child ordering, so the benchmark does not claim stronger sequencing evidence than is persisted. +`cargo_test` launch. Direct and Code Mode both require one canonical pending +relation, a correlated follow-up, and same-relation terminal timing from ActionAudit. +For Code Mode, the canonical ModelErgonomics producer derives that pending relation +from the nested `effect_receipt.children` Job handoff. The reporter joins that salted +relation internally and emits only payload-safe counts/timing grouped by the pending +origin tool. Continuity context may support the relation/timing join, but benchmark +success additionally requires the terminal evidence row itself to belong to the +selected ActionAudit run; for Code Mode that selected chain must originate from +`execute_effectful_code_mode`. Persisted composition must independently show exactly +one consequential validator handoff and no known or unknown substitute result. A +synchronous completion, redispatch, unrelated outer Job, uncorrelated/substitute +process, or pending without terminal evidence remains `partial`/`unsupported`, +never upgraded to success. +The guarded multi-file case is likewise not a pass from final bytes alone: persisted Direct canonical-call or Code Mode composition evidence must show exactly one `edit_project_files` call, at least one admitted search call, and at least two `read_files` calls (pre-edit and post-edit phases). Direct additionally requires the existing authoritative ModelErgonomics edit outcome to be exactly `applied`. Code Mode requires exactly one consequential known-result child, zero Job handoffs/outcome-unknown/nested failures, and the aggregate canonical mutation evidence to show one `state_changed=true` mutation and zero known no-change mutations, in addition to the fixture-side exact-byte oracle. A dry-run/no-op edit plus shell/process/write fallback therefore cannot be promoted to a pass. Current composition does not prove nested child ordering, so the benchmark does not claim stronger sequencing evidence than is persisted. This command coordinates real captures; it is not itself evidence that a model made fewer inference turns. Deterministic runtime/script acceptance and fixture oracles @@ -375,6 +378,7 @@ trace-only input cannot apply `--workflow-session-id`. Start each benchmark run with its fresh `work_on_project` bootstrap; that call links its own ActionAudit row through the canonical `WorkOnProject` relation. +For `readonly_review`, the benchmark gate admits the one bootstrap plus at most one `read_tool_manifest` discovery call when the selected Code Mode callable contract is not already retained. That contract-discovery call is not Direct repository inspection; any outer `read_files`, search, or other fallback still fails the lane check closed. Once the bootstrap returns the exact run Session id, every subsequent model-facing outer call in the run must pass that id as `recording_session_id`. A business `session_id` does not substitute for ActionAudit recorder provenance. Code Mode diff --git a/scripts/agent_loop_benchmark.py b/scripts/agent_loop_benchmark.py index adeefc04d..405ddaf86 100644 --- a/scripts/agent_loop_benchmark.py +++ b/scripts/agent_loop_benchmark.py @@ -34,7 +34,6 @@ MAX_DRIVER_RECEIPT_BYTES = 64 * 1024 MAX_FIXTURE_ORACLE_BYTES = 1024 * 1024 MAX_CASE_MANIFEST_BYTES = 1024 * 1024 -MAX_JOB_ID_BYTES = 256 MAX_GIT_PATH_BYTES = 64 * 1024 MAX_GIT_PATHS = 512 CODE_MODE_TOOL_BY_SURFACE = { @@ -357,7 +356,13 @@ def _fixture_oracle( "passed": tracked_ok and untracked_ok and not overflow and changed_raw == expected_raw, "expected": expected_changed, "actual": None if overflow else changed, - "reason": "changed path listing exceeds oracle limit" if overflow else None, + "reason": ( + "changed path listing exceeds oracle limit" + if overflow + else "changed path query failed" + if not tracked_ok or not untracked_ok + else None + ), } ) @@ -516,36 +521,6 @@ def _tool_count(summary: dict[str, Any], variant: str, tool_name: str) -> int | return None -def _bounded_job_id(value: Any) -> str | None: - if not isinstance(value, str) or not value: - return None - try: - encoded = value.encode("utf-8") - except UnicodeEncodeError: - return None - if len(encoded) > MAX_JOB_ID_BYTES: - return None - return value - - -def _job_identity_evidence_proven(receipt: dict[str, Any]) -> bool: - evidence = receipt.get("job_identity_evidence") - if not isinstance(evidence, dict) or set(evidence) != { - "handoff_job_id", - "terminal_job_id", - "terminal", - }: - return False - handoff = _bounded_job_id(evidence.get("handoff_job_id")) - terminal = _bounded_job_id(evidence.get("terminal_job_id")) - return ( - handoff is not None - and terminal is not None - and handoff == terminal - and evidence.get("terminal") is True - ) - - def _runtime_evidence_proven( summary: dict[str, Any], variant: str, @@ -587,52 +562,104 @@ def _runtime_evidence_proven( and not isinstance(count, bool) and count > 0 } - return ( - code_mode_counts.get(expected_tool) == outer_code_mode - and sum(code_mode_counts.values()) == outer_code_mode - ) + if ( + code_mode_counts.get(expected_tool) != outer_code_mode + or sum(code_mode_counts.values()) != outer_code_mode + ): + return False + if surface == "read_only": + if any( + not isinstance(count, int) + or isinstance(count, bool) + or count < 0 + for count in outer_by_name.values() + ): + return False + bootstrap_count = outer_by_name.get("work_on_project", 0) + manifest_count = outer_by_name.get("read_tool_manifest", 0) + allowed = {expected_tool, "work_on_project", "read_tool_manifest"} + unexpected = { + name: count + for name, count in outer_by_name.items() + if name not in allowed and count > 0 + } + return ( + bootstrap_count == 1 + and manifest_count in (0, 1) + and not unexpected + and outer_total + == outer_code_mode + bootstrap_count + manifest_count + ) + return True return False -def _long_handoff_terminal_proven( - summary: dict[str, Any], - variant: str, - receipt: dict[str, Any], -) -> bool: +def _long_handoff_terminal_proven(summary: dict[str, Any], variant: str) -> bool: if _tool_count(summary, variant, "cargo_test") != 1: return False + pending = report._get_path(summary, "job_convergence.pending_handoff_count") + known_followups = report._get_path(summary, "job_convergence.pending_followup_known_count") + terminal_samples = report._get_path(summary, "job_convergence.pending_to_terminal_ms.samples") + if ( + not isinstance(pending, int) + or isinstance(pending, bool) + or pending != 1 + or not isinstance(known_followups, int) + or isinstance(known_followups, bool) + or known_followups < 1 + or not isinstance(terminal_samples, int) + or isinstance(terminal_samples, bool) + or terminal_samples < 1 + ): + return False + if variant == "direct": - pending = report._get_path(summary, "job_convergence.pending_handoff_count") - known_followups = report._get_path(summary, "job_convergence.pending_followup_known_count") - terminal_samples = report._get_path(summary, "job_convergence.pending_to_terminal_ms.samples") + origin = "cargo_test" return ( - isinstance(pending, int) - and not isinstance(pending, bool) - and pending == 1 - and isinstance(known_followups, int) - and not isinstance(known_followups, bool) - and known_followups >= 1 - and isinstance(terminal_samples, int) - and not isinstance(terminal_samples, bool) - and terminal_samples >= 1 + report._get_path( + summary, f"job_convergence.pending_handoff_by_origin_tool.{origin}" + ) + == 1 + and report._get_path( + summary, f"job_convergence.pending_followup_known_by_origin_tool.{origin}" + ) + == 1 + and report._get_path( + summary, + f"job_convergence.pending_to_terminal_ms_by_origin_tool.{origin}.samples", + ) + == 1 + and report._get_path( + summary, f"job_convergence.selected_terminal_by_origin_tool.{origin}" + ) + == 1 ) - if variant == "code_mode": - job_handoffs = report._get_path(summary, "composition.job_handoffs.total") - consequential = report._get_path(summary, "composition.consequential_calls.total") - outcome_unknown = report._get_path(summary, "composition.outcome_unknown.total") - outer_observes = report._get_path(summary, "tools.outer_by_name.observe_jobs") + origin = "execute_effectful_code_mode" return ( - job_handoffs == 1 - and consequential == 1 - and outcome_unknown == 0 - and isinstance(outer_observes, int) - and not isinstance(outer_observes, bool) - and outer_observes >= 1 - and _job_identity_evidence_proven(receipt) + report._get_path(summary, "composition.job_handoffs.total") == 1 + and report._get_path(summary, "composition.consequential_calls.total") == 1 + and report._get_path(summary, "composition.known_results.total") == 0 + and report._get_path(summary, "composition.outcome_unknown.total") == 0 + and report._get_path( + summary, f"job_convergence.pending_handoff_by_origin_tool.{origin}" + ) + == 1 + and report._get_path( + summary, f"job_convergence.pending_followup_known_by_origin_tool.{origin}" + ) + == 1 + and report._get_path( + summary, + f"job_convergence.pending_to_terminal_ms_by_origin_tool.{origin}.samples", + ) + == 1 + and report._get_path( + summary, f"job_convergence.selected_terminal_by_origin_tool.{origin}" + ) + == 1 ) - return False @@ -671,6 +698,12 @@ def _guarded_multi_file_contract_proven(summary: dict[str, Any], variant: str) - known_results = report._get_path(summary, "composition.known_results.total") job_handoffs = report._get_path(summary, "composition.job_handoffs.total") outcome_unknown = report._get_path(summary, "composition.outcome_unknown.total") + mutation_changed = report._get_path( + summary, "composition.mutation_state_changed.total" + ) + mutation_no_change = report._get_path( + summary, "composition.mutation_no_change.total" + ) if ( not isinstance(child_failed, int) or isinstance(child_failed, bool) @@ -679,6 +712,8 @@ def _guarded_multi_file_contract_proven(summary: dict[str, Any], variant: str) - or known_results != 1 or job_handoffs != 0 or outcome_unknown != 0 + or mutation_changed != 1 + or mutation_no_change != 0 ): return False @@ -808,7 +843,7 @@ def _run_sample( case["id"] == "long_validation_handoff" and summary is not None and effective_status == "pass" - and not _long_handoff_terminal_proven(summary, variant, receipt) + and not _long_handoff_terminal_proven(summary, variant) ): effective_status = "partial" diff --git a/scripts/agent_loop_report.py b/scripts/agent_loop_report.py index ed3e68da5..d02a1a13b 100644 --- a/scripts/agent_loop_report.py +++ b/scripts/agent_loop_report.py @@ -60,7 +60,11 @@ ) # Additive fields may be absent from historical ActionAudit rows. They get their # own availability/missing accounting and never invalidate the core composition. -CODE_MODE_COMPOSITION_OPTIONAL_NUMERIC_FIELDS = ("input_bytes",) +CODE_MODE_COMPOSITION_OPTIONAL_NUMERIC_FIELDS = ( + "input_bytes", + "mutation_state_changed", + "mutation_no_change", +) # Closed v13 persisted wire vocabulary for this offline consumer, not alias rules. # Runtime/schema spellings are owned by ToolInputNormalizationCode in tool-contracts. @@ -846,6 +850,13 @@ def _summarize_job_convergence(selected: list[dict[str, Any]], context: list[dic failure_observed: set[str] = set() timed: set[str] = set() pending_observed: set[str] = set() + pending_origin_by_relation: dict[str, str | None] = {} + pending_by_origin_tool: Counter[str] = Counter() + followup_by_origin_tool: Counter[str] = Counter() + timings_by_origin_tool: dict[str, list[int]] = {} + timing_missing_by_origin_tool: Counter[str] = Counter() + selected_terminal_by_origin_tool: Counter[str] = Counter() + selected_terminal_relations: set[str] = set() def facts(row: dict[str, Any]) -> dict[str, Any]: value = (_telemetry(row) or {}).get("job_convergence") @@ -859,6 +870,19 @@ def events(row: dict[str, Any]) -> list[dict[str, Any]]: and _is_exact_sha256(event.get("relation")) and event.get("kind") in ("pending_handoff", "explicit_observe", "passive_terminal")] + def origin_tool(row: dict[str, Any]) -> str | None: + telemetry = _telemetry(row) + if not isinstance(telemetry, dict): + return None + tool = telemetry.get("tool_name") + if ( + isinstance(tool, str) + and re.fullmatch(r"[a-z][a-z0-9_]{0,127}", tool) is not None + and tool == row.get("operation") + ): + return tool + return None + def predecessor(row: dict[str, Any]) -> dict[str, Any] | None: nonlocal links_remaining if links_remaining <= 0: @@ -891,6 +915,28 @@ def history(row: dict[str, Any]): selected_relations = {event["relation"] for row in selected for event in events(row)} selected_traces = {row.get("server_trace_id") for row in selected} + for row in selected: + telemetry = _telemetry(row) or {} + version = telemetry.get("schema_version") + if ( + facts(row).get("correlation_complete") is not True + or not isinstance(version, int) + or isinstance(version, bool) + or version < 10 + ): + continue + tool = origin_tool(row) + for event in events(row): + if event["kind"] != "pending_handoff": + continue + relation_id = event["relation"] + if relation_id not in pending_origin_by_relation: + pending_origin_by_relation[relation_id] = tool + elif pending_origin_by_relation[relation_id] != tool: + pending_origin_by_relation[relation_id] = None + for tool in pending_origin_by_relation.values(): + if tool is not None: + pending_by_origin_tool[tool] += 1 measured_rows = [row for row in rows.values() if row.get("server_trace_id") in selected_traces or any(event["relation"] in selected_relations for event in events(row))] # Missing trace/timing still contributes observed invocation counts, but @@ -907,11 +953,24 @@ def history(row: dict[str, Any]): if event["kind"] == "pending_handoff" and event["relation"] in selected_relations: known_followups.add(event["relation"]) counts["pending_followup_known_count"] = len(known_followups) + for relation_id in known_followups: + tool = pending_origin_by_relation.get(relation_id) + if tool is not None: + followup_by_origin_tool[tool] += 1 for row in measured_rows: data = facts(row) - for name in ("pending_handoff_count", "passive_terminal_delivery_count", "passive_failure_delivery_count", "wait_for_job_terminal_count"): + for name, maximum in ( + ("pending_handoff_count", 32), + ("passive_terminal_delivery_count", 9), + ("passive_failure_delivery_count", 9), + ("wait_for_job_terminal_count", 1), + ): value = data.get(name, 0) - if isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 9: + if ( + isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value <= maximum + ): counts[name] += value if data.get("correlation_complete") is False: counts["uncorrelated_calls"] += 1 @@ -924,18 +983,17 @@ def history(row: dict[str, Any]): if relation_id not in pending_observed: counts["pending_followed_immediately_by_observe_count"] += 1 pending_observed.add(relation_id) - if relation_id in failure_observed: - continue - for previous in history(row): - matching = [old for old in events(previous) if old["relation"] == relation_id] - if any(old["kind"] == "passive_terminal" and old.get("failure") is True for old in matching): - counts["terminal_failure_followed_by_observe_count"] += 1 - failure_observed.add(relation_id) - if any(old.get("validation_failure") is True for old in matching if old["kind"] == "passive_terminal"): - counts["terminal_validation_failure_followed_by_observe_count"] += 1 - break - if any(old["kind"] == "pending_handoff" for old in matching): - break + if relation_id not in failure_observed: + for previous in history(row): + matching = [old for old in events(previous) if old["relation"] == relation_id] + if any(old["kind"] == "passive_terminal" and old.get("failure") is True for old in matching): + counts["terminal_failure_followed_by_observe_count"] += 1 + failure_observed.add(relation_id) + if any(old.get("validation_failure") is True for old in matching if old["kind"] == "passive_terminal"): + counts["terminal_validation_failure_followed_by_observe_count"] += 1 + break + if any(old["kind"] == "pending_handoff" for old in matching): + break elif event["kind"] == "passive_terminal": counts["passive_validation_failure_delivery_count"] += int(event.get("validation_failure") is True) observed = False @@ -956,15 +1014,52 @@ def history(row: dict[str, Any]): pending = previous break if event["kind"] == "passive_terminal" or event.get("terminal_observed_at_ms") is not None: + origin = pending_origin_by_relation.get(relation_id) + trace_id = row.get("server_trace_id") + if ( + pending is not None + and origin is not None + and isinstance(trace_id, str) + and trace_id in selected_traces + and relation_id not in selected_terminal_relations + ): + selected_terminal_relations.add(relation_id) + selected_terminal_by_origin_tool[origin] += 1 if relation_id not in timed: timed.add(relation_id) handed = pending.get("response_handed_at_ms") if pending else None terminal = event.get("terminal_observed_at_ms") if isinstance(handed, int) and isinstance(terminal, int) and terminal >= handed: - timings.append(terminal - handed) + elapsed = terminal - handed + timings.append(elapsed) + if origin is not None: + timings_by_origin_tool.setdefault(origin, []).append(elapsed) else: timing_missing += 1 - return {**counts, "pending_to_terminal_ms": _metric_distribution(timings, missing=timing_missing)} + if origin is not None: + timing_missing_by_origin_tool[origin] += 1 + origin_tools = sorted( + set(pending_by_origin_tool) + | set(followup_by_origin_tool) + | set(timings_by_origin_tool) + | set(timing_missing_by_origin_tool) + ) + return { + **counts, + "pending_handoff_by_origin_tool": dict(sorted(pending_by_origin_tool.items())), + "pending_followup_known_by_origin_tool": dict(sorted(followup_by_origin_tool.items())), + "selected_terminal_by_origin_tool": dict( + sorted(selected_terminal_by_origin_tool.items()) + ), + "pending_to_terminal_ms": _metric_distribution(timings, missing=timing_missing), + "pending_to_terminal_ms_by_origin_tool": { + tool: _metric_distribution( + timings_by_origin_tool.get(tool, []), + missing=timing_missing_by_origin_tool.get(tool, 0), + ) + for tool in origin_tools + }, + } diff --git a/scripts/tests/test_agent_loop_benchmark.py b/scripts/tests/test_agent_loop_benchmark.py index 2f49de507..2ccf324b8 100644 --- a/scripts/tests/test_agent_loop_benchmark.py +++ b/scripts/tests/test_agent_loop_benchmark.py @@ -96,28 +96,70 @@ def test_runtime_evidence_gate_fails_closed(self) -> None: self.assertFalse( benchmark._runtime_evidence_proven(code_mode, "code_mode", "read_only") ) + read_only = { + "outer_calls": {"total": 2}, + "availability": {"code_mode_composition": {"available": True}}, + "composition": {"outer_code_mode_calls": 1}, + "tools": { + "outer_by_name": { + "work_on_project": 1, + "execute_code_mode": 1, + } + }, + } + self.assertTrue( + benchmark._runtime_evidence_proven(read_only, "code_mode", "read_only") + ) + read_only["outer_calls"]["total"] = 3 + read_only["tools"]["outer_by_name"]["read_tool_manifest"] = 1 + self.assertTrue( + benchmark._runtime_evidence_proven(read_only, "code_mode", "read_only") + ) + read_only["tools"]["outer_by_name"]["read_tool_manifest"] = 2 + read_only["outer_calls"]["total"] = 4 + self.assertFalse( + benchmark._runtime_evidence_proven(read_only, "code_mode", "read_only") + ) + read_only["tools"]["outer_by_name"]["read_tool_manifest"] = 1 + read_only["outer_calls"]["total"] = 4 + read_only["tools"]["outer_by_name"]["read_files"] = 1 + self.assertFalse( + benchmark._runtime_evidence_proven(read_only, "code_mode", "read_only") + ) + del read_only["tools"]["outer_by_name"]["read_files"] + del read_only["tools"]["outer_by_name"]["read_tool_manifest"] + read_only["outer_calls"]["total"] = 1 + del read_only["tools"]["outer_by_name"]["work_on_project"] + self.assertFalse( + benchmark._runtime_evidence_proven(read_only, "code_mode", "read_only") + ) code_mode["tools"]["outer_by_name"]["execute_code_mode"] = 1 code_mode["composition"]["outer_code_mode_calls"] = 2 self.assertFalse( benchmark._runtime_evidence_proven(code_mode, "code_mode", "guarded_edit") ) - def test_long_handoff_requires_one_launch_and_exact_terminal_identity(self) -> None: + def test_long_handoff_requires_one_launch_and_exact_terminal_relation(self) -> None: direct = { "canonical_calls": {"by_name": {"cargo_test": 1}}, "job_convergence": { "pending_handoff_count": 1, "pending_followup_known_count": 1, "pending_to_terminal_ms": {"samples": 1}, + "pending_handoff_by_origin_tool": {"cargo_test": 1}, + "pending_followup_known_by_origin_tool": {"cargo_test": 1}, + "pending_to_terminal_ms_by_origin_tool": { + "cargo_test": {"samples": 1} + }, + "selected_terminal_by_origin_tool": {"cargo_test": 1}, }, } - self.assertTrue( - benchmark._long_handoff_terminal_proven(direct, "direct", {}) - ) + self.assertTrue(benchmark._long_handoff_terminal_proven(direct, "direct")) + direct["job_convergence"]["selected_terminal_by_origin_tool"] = {} + self.assertFalse(benchmark._long_handoff_terminal_proven(direct, "direct")) + direct["job_convergence"]["selected_terminal_by_origin_tool"] = {"cargo_test": 1} direct["canonical_calls"]["by_name"]["cargo_test"] = 2 - self.assertFalse( - benchmark._long_handoff_terminal_proven(direct, "direct", {}) - ) + self.assertFalse(benchmark._long_handoff_terminal_proven(direct, "direct")) code_mode = { "availability": {"code_mode_composition": {"available": True}}, @@ -125,29 +167,49 @@ def test_long_handoff_requires_one_launch_and_exact_terminal_identity(self) -> N "nested_tool_counts": {"cargo_test": 1}, "job_handoffs": {"total": 1}, "consequential_calls": {"total": 1}, + "known_results": {"total": 0}, "outcome_unknown": {"total": 0}, }, - "tools": {"outer_by_name": {"execute_effectful_code_mode": 1, "observe_jobs": 1}}, - } - receipt = { - "job_identity_evidence": { - "handoff_job_id": "job-1", - "terminal_job_id": "job-1", - "terminal": True, - } + "job_convergence": { + "pending_handoff_count": 1, + "pending_followup_known_count": 1, + "pending_to_terminal_ms": {"samples": 1}, + "pending_handoff_by_origin_tool": { + "execute_effectful_code_mode": 1 + }, + "pending_followup_known_by_origin_tool": { + "execute_effectful_code_mode": 1 + }, + "pending_to_terminal_ms_by_origin_tool": { + "execute_effectful_code_mode": {"samples": 1} + }, + "selected_terminal_by_origin_tool": { + "execute_effectful_code_mode": 1 + }, + }, } - self.assertTrue( - benchmark._long_handoff_terminal_proven(code_mode, "code_mode", receipt) - ) - receipt["job_identity_evidence"]["terminal_job_id"] = "job-2" + self.assertTrue(benchmark._long_handoff_terminal_proven(code_mode, "code_mode")) + code_mode["job_convergence"]["selected_terminal_by_origin_tool"] = {} self.assertFalse( - benchmark._long_handoff_terminal_proven(code_mode, "code_mode", receipt) + benchmark._long_handoff_terminal_proven(code_mode, "code_mode") ) - receipt["job_identity_evidence"]["terminal_job_id"] = "job-1" - code_mode["composition"]["nested_tool_counts"]["cargo_test"] = 2 + code_mode["job_convergence"]["selected_terminal_by_origin_tool"] = { + "execute_effectful_code_mode": 1 + } + code_mode["job_convergence"]["pending_handoff_by_origin_tool"] = { + "run_process": 1 + } self.assertFalse( - benchmark._long_handoff_terminal_proven(code_mode, "code_mode", receipt) + benchmark._long_handoff_terminal_proven(code_mode, "code_mode") ) + code_mode["job_convergence"]["pending_handoff_by_origin_tool"] = { + "execute_effectful_code_mode": 1 + } + code_mode["job_convergence"]["pending_to_terminal_ms"]["samples"] = 0 + self.assertFalse(benchmark._long_handoff_terminal_proven(code_mode, "code_mode")) + code_mode["job_convergence"]["pending_to_terminal_ms"]["samples"] = 1 + code_mode["composition"]["nested_tool_counts"]["cargo_test"] = 2 + self.assertFalse(benchmark._long_handoff_terminal_proven(code_mode, "code_mode")) def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(self) -> None: direct = { @@ -188,6 +250,8 @@ def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(sel "known_results": {"total": 1}, "job_handoffs": {"total": 0}, "outcome_unknown": {"total": 0}, + "mutation_state_changed": {"total": 1}, + "mutation_no_change": {"total": 0}, }, "outer_calls": {"failed": 0, "timeout_or_unknown": 0}, "child_calls": {"failed": 0}, @@ -199,6 +263,9 @@ def test_multi_file_contract_requires_one_edit_search_and_post_read_capacity(sel code_mode["composition"]["consequential_calls"]["total"] = 2 self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) code_mode["composition"]["consequential_calls"]["total"] = 1 + code_mode["composition"]["mutation_state_changed"]["total"] = 0 + self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) + code_mode["composition"]["mutation_state_changed"]["total"] = 1 code_mode["composition"]["nested_tool_counts"]["read_files"] = 1 self.assertFalse(benchmark._guarded_multi_file_contract_proven(code_mode, "code_mode")) @@ -268,6 +335,33 @@ def test_fixture_oracle_checks_both_files_and_exact_diff(self) -> None: beta_check = next(check for check in result["checks"] if check.get("path", "").endswith("beta.txt")) self.assertEqual(beta_check["reason"], "file exceeds oracle byte limit") + def test_fixture_oracle_reports_git_path_query_failure(self) -> None: + manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) + case = report._case_by_id(manifest, "guarded_multi_file_edit") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fixture = root / "tests/fixtures/agent-loop-multi-file" + fixture.mkdir(parents=True) + (fixture / "alpha.txt").write_text( + "BENCH_TARGET_ALPHA=after\nALPHA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + (fixture / "beta.txt").write_text( + "BENCH_TARGET_BETA=after\nBETA_UNRELATED_SENTINEL=keep\n", + encoding="utf-8", + ) + with mock.patch.object( + benchmark, + "_bounded_git_paths", + side_effect=[(False, False, []), (True, False, [])], + ): + result = benchmark._fixture_oracle(case, root) + changed = next( + check for check in result["checks"] if check.get("kind") == "changed_files" + ) + self.assertFalse(changed["passed"]) + self.assertEqual(changed["reason"], "changed path query failed") + @unittest.skipIf(os.name == "nt", "FIFO is POSIX-only") def test_fixture_oracle_rejects_fifo_without_blocking(self) -> None: manifest = report.load_case_manifest(report.DEFAULT_CASE_MANIFEST) @@ -339,22 +433,6 @@ def test_driver_receipt_pathological_json_is_a_contract_error(self) -> None: with self.assertRaisesRegex(benchmark.BenchmarkError, "bounded JSON"): benchmark._load_driver_receipt(receipt) - def test_job_identity_evidence_is_exact_bounded_and_private(self) -> None: - receipt = { - "job_identity_evidence": { - "handoff_job_id": "private-job-1", - "terminal_job_id": "private-job-1", - "terminal": True, - } - } - self.assertTrue(benchmark._job_identity_evidence_proven(receipt)) - receipt["job_identity_evidence"]["terminal_job_id"] = "other-job" - self.assertFalse(benchmark._job_identity_evidence_proven(receipt)) - receipt["job_identity_evidence"]["terminal_job_id"] = "x" * ( - benchmark.MAX_JOB_ID_BYTES + 1 - ) - self.assertFalse(benchmark._job_identity_evidence_proven(receipt)) - @unittest.skipIf(os.name == "nt", "FIFO is POSIX-only") def test_driver_receipt_rejects_fifo_without_blocking(self) -> None: with tempfile.TemporaryDirectory() as tmp: diff --git a/scripts/tests/test_agent_loop_report.py b/scripts/tests/test_agent_loop_report.py index 6bf05641e..177deea86 100644 --- a/scripts/tests/test_agent_loop_report.py +++ b/scripts/tests/test_agent_loop_report.py @@ -1067,7 +1067,11 @@ def test_code_mode_composition_aggregates_effect_and_projection_counters(self) - self.insert_event( "guarded_edit", tool="execute_mutating_code_mode", - composition=code_mode_composition(input_bytes=80), + composition=code_mode_composition( + input_bytes=80, + mutation_state_changed=1, + mutation_no_change=0, + ), ) self.insert_event( "validation", @@ -1086,6 +1090,8 @@ def test_code_mode_composition_aggregates_effect_and_projection_counters(self) - consequential_calls=1, known_results=0, job_handoffs=1, + mutation_state_changed=0, + mutation_no_change=0, ), ) result = self.summarize(variant="code_mode") @@ -1096,6 +1102,8 @@ def test_code_mode_composition_aggregates_effect_and_projection_counters(self) - self.assertEqual(composition["known_results"]["total"], 1) self.assertEqual(composition["job_handoffs"]["total"], 1) self.assertEqual(composition["outcome_unknown"]["total"], 0) + self.assertEqual(composition["mutation_state_changed"]["total"], 1) + self.assertEqual(composition["mutation_no_change"]["total"], 0) self.assertEqual(composition["duration_ms"]["total"], 35) self.assertEqual(composition["input_bytes"]["total"], 120) self.assertEqual(composition["returned_bytes"]["total"], 90) @@ -1119,6 +1127,12 @@ def test_historical_code_mode_composition_without_input_bytes_stays_valid(self) self.assertIsNone(input_bytes["total"]) self.assertEqual(input_bytes["observed_total"], 0) self.assertEqual(input_bytes["missing"], 1) + mutation_changed = result["composition"]["mutation_state_changed"] + mutation_no_change = result["composition"]["mutation_no_change"] + self.assertIsNone(mutation_changed["total"]) + self.assertEqual(mutation_changed["missing"], 1) + self.assertIsNone(mutation_no_change["total"]) + self.assertEqual(mutation_no_change["missing"], 1) def test_direct_variant_with_code_mode_call_fails_composition_closed(self) -> None: self.insert_event( @@ -1838,7 +1852,15 @@ def test_job_convergence_loads_exact_unlinked_successor_after_selected_tail(self class JobConvergenceTests(unittest.TestCase): @staticmethod - def row(trace, previous=None, kind=None, relation="a" * 64, failure=None, terminal=None): + def row( + trace, + previous=None, + kind=None, + relation="a" * 64, + failure=None, + terminal=None, + tool="run_process", + ): event = {"kind": kind, "relation": relation} if failure is not None: event["failure"] = failure @@ -1854,6 +1876,7 @@ def row(trace, previous=None, kind=None, relation="a" * 64, failure=None, termin } return { "event_id": str(trace), "server_trace_id": str(trace), + "operation": tool, "action_name": "toolsCall", "client_window_key": "window", "principal_correlation_kind": "user", "principal_correlation_id": "principal", "window_meaningful": True, "window_continuity_eligible": True, "response_streaming": False, @@ -1861,7 +1884,11 @@ def row(trace, previous=None, kind=None, relation="a" * 64, failure=None, termin "request_observed_at_ms": trace * 1000, "response_handed_at_ms": trace * 1000 + 100, "summary": {"previous_meaningful_call": str(previous) if previous else None, - "model_ergonomics": {"schema_version": 10, "job_convergence": facts}}, + "model_ergonomics": { + "schema_version": 10, + "tool_name": tool, + "job_convergence": facts, + }}, } def test_exact_immediate_observe_and_unrelated_relations(self): @@ -1869,7 +1896,29 @@ def test_exact_immediate_observe_and_unrelated_relations(self): observe = self.row(2, 1, "explicit_observe") result = report._summarize_job_convergence([pending, observe], []) self.assertEqual(result["pending_handoff_count"], 1) + many_handoffs = self.row(9) + many_handoffs["summary"]["model_ergonomics"]["job_convergence"].update( + { + "pending_handoff_count": 32, + "correlation_complete": False, + "events": [], + } + ) + self.assertEqual( + report._summarize_job_convergence([many_handoffs], [])[ + "pending_handoff_count" + ], + 32, + ) self.assertEqual(result["pending_followed_immediately_by_observe_count"], 1) + self.assertEqual( + result["pending_handoff_by_origin_tool"], + {"run_process": 1}, + ) + self.assertEqual( + result["pending_followup_known_by_origin_tool"], + {"run_process": 1}, + ) unrelated = self.row(2, 1, "explicit_observe", relation="b" * 64) self.assertEqual(report._summarize_job_convergence([pending, unrelated], [])["pending_followed_immediately_by_observe_count"], 0) intervening_read = self.row(2, 1) @@ -1887,6 +1936,10 @@ def test_passive_failure_then_fix_or_observe_and_authoritative_timing(self): self.assertEqual(result["passive_terminal_before_explicit_observe_count"], 1) self.assertEqual(result["terminal_failure_followed_by_observe_count"], 0) self.assertEqual(result["pending_to_terminal_ms"]["observed_total"], 900) + self.assertEqual( + result["pending_to_terminal_ms_by_origin_tool"]["run_process"]["observed_total"], + 900, + ) observe = self.row(3, 2, "explicit_observe") second_observe = self.row(4, 3, "explicit_observe") result = report._summarize_job_convergence([pending, terminal, observe, second_observe], []) @@ -1896,6 +1949,114 @@ def test_passive_failure_then_fix_or_observe_and_authoritative_timing(self): unrelated = self.row(3, 2, "explicit_observe", relation="b" * 64) self.assertEqual(report._summarize_job_convergence([pending, terminal, unrelated], [])["terminal_failure_followed_by_observe_count"], 0) + def test_pending_origin_tool_binds_terminal_relation_without_exporting_relation(self): + relation = "c" * 64 + pending = self.row( + 1, + kind="pending_handoff", + relation=relation, + tool="execute_effectful_code_mode", + ) + unrelated = self.row( + 2, + 1, + "pending_handoff", + relation="d" * 64, + tool="run_process", + ) + terminal = self.row( + 3, + 1, + "explicit_observe", + relation=relation, + terminal=3000, + tool="observe_jobs", + ) + result = report._summarize_job_convergence([pending, unrelated, terminal], []) + self.assertEqual( + result["pending_handoff_by_origin_tool"], + {"execute_effectful_code_mode": 1, "run_process": 1}, + ) + self.assertEqual( + result["pending_followup_known_by_origin_tool"], + {"execute_effectful_code_mode": 1}, + ) + self.assertEqual( + result["pending_to_terminal_ms_by_origin_tool"][ + "execute_effectful_code_mode" + ]["samples"], + 1, + ) + self.assertEqual( + result["selected_terminal_by_origin_tool"], + {"execute_effectful_code_mode": 1}, + ) + context_only = report._summarize_job_convergence( + [pending, unrelated], + [terminal], + ) + self.assertEqual( + context_only["pending_to_terminal_ms_by_origin_tool"][ + "execute_effectful_code_mode" + ]["samples"], + 1, + ) + self.assertEqual(context_only["selected_terminal_by_origin_tool"], {}) + selected_terminal = self.row( + 4, + 1, + "explicit_observe", + relation=relation, + terminal=4000, + tool="observe_jobs", + ) + context_first = report._summarize_job_convergence( + [pending, selected_terminal], + [terminal], + ) + self.assertEqual( + context_first["selected_terminal_by_origin_tool"], + {"execute_effectful_code_mode": 1}, + ) + passive_failure = self.row( + 5, + 1, + "passive_terminal", + relation=relation, + failure=True, + terminal=5000, + ) + context_observe = self.row( + 6, + 5, + "explicit_observe", + relation=relation, + tool="observe_jobs", + ) + selected_after_failure = self.row( + 7, + 6, + "explicit_observe", + relation=relation, + terminal=7000, + tool="observe_jobs", + ) + failure_context_first = report._summarize_job_convergence( + [pending, selected_after_failure], + [passive_failure, context_observe], + ) + self.assertEqual( + failure_context_first["terminal_failure_followed_by_observe_count"], + 1, + ) + self.assertEqual( + failure_context_first["selected_terminal_by_origin_tool"], + {"execute_effectful_code_mode": 1}, + ) + encoded = json.dumps(result) + self.assertNotIn(relation, encoded) + self.assertNotIn("d" * 64, encoded) + def test_prior_observe_does_not_count_as_passive_before_observe(self): rows = [self.row(1, kind="pending_handoff"), self.row(2, 1, "explicit_observe"), self.row(3, 2, "passive_terminal", failure=False, terminal=3000)] result = report._summarize_job_convergence(rows, []) diff --git a/src/mcp_tests/model_ergonomics.rs b/src/mcp_tests/model_ergonomics.rs index e0f3c0b98..3a951b200 100644 --- a/src/mcp_tests/model_ergonomics.rs +++ b/src/mcp_tests/model_ergonomics.rs @@ -625,6 +625,8 @@ async fn http_mcp_code_mode_persists_only_bounded_composition_telemetry() { assert_eq!(composition["known_results"], 0); assert_eq!(composition["job_handoffs"], 0); assert_eq!(composition["outcome_unknown"], 0); + assert_eq!(composition["mutation_state_changed"], 0); + assert_eq!(composition["mutation_no_change"], 0); assert_eq!(composition["max_in_flight"], 0); assert_eq!(composition["nested_tool_counts"], json!({})); assert!(composition["duration_ms"].is_u64()); @@ -648,6 +650,8 @@ async fn http_mcp_code_mode_persists_only_bounded_composition_telemetry() { "job_handoffs", "known_results", "max_in_flight", + "mutation_no_change", + "mutation_state_changed", "nested_calls", "nested_failures", "nested_raw_result_bytes_total", diff --git a/src/runtime_console_http/tests/window_activity.rs b/src/runtime_console_http/tests/window_activity.rs index 07750bc7e..09790abf4 100644 --- a/src/runtime_console_http/tests/window_activity.rs +++ b/src/runtime_console_http/tests/window_activity.rs @@ -48,7 +48,9 @@ async fn code_mode_composition_projects_on_one_outer_window_activity() { "consequential_calls": 1, "known_results": 1, "job_handoffs": 0, - "outcome_unknown": 0 + "outcome_unknown": 0, + "mutation_state_changed": 1, + "mutation_no_change": 0 } }), request_bytes: None, @@ -108,6 +110,47 @@ async fn code_mode_composition_projects_on_one_outer_window_activity() { assert_eq!(composition.known_results, 1); assert_eq!(composition.job_handoffs, 0); assert_eq!(composition.outcome_unknown, 0); + assert_eq!(composition.mutation_state_changed, Some(1)); + assert_eq!(composition.mutation_no_change, Some(0)); + + let legacy = json!({ + "nested_calls": 1, + "nested_successes": 1, + "nested_failures": 0, + "max_in_flight": 1, + "duration_ms": 1, + "slot_wait_ms": 0, + "returned_bytes": 1, + "nested_raw_result_bytes_total": 1, + "nested_tool_counts": {"read_files": 1}, + "consequential_calls": 0, + "known_results": 0, + "job_handoffs": 0, + "outcome_unknown": 0 + }); + let legacy = project_code_mode_composition(&legacy) + .expect("historical composition without mutation counters remains readable"); + assert_eq!(legacy.mutation_state_changed, None); + assert_eq!(legacy.mutation_no_change, None); + + let impossible_mutation_counts = json!({ + "nested_calls": 1, + "nested_successes": 1, + "nested_failures": 0, + "max_in_flight": 1, + "duration_ms": 1, + "slot_wait_ms": 0, + "returned_bytes": 1, + "nested_raw_result_bytes_total": 1, + "nested_tool_counts": {"read_files": 1}, + "consequential_calls": 0, + "known_results": 0, + "job_handoffs": 0, + "outcome_unknown": 0, + "mutation_state_changed": 1, + "mutation_no_change": 0 + }); + assert!(project_code_mode_composition(&impossible_mutation_counts).is_none()); let invalid = json!({ "nested_calls": 1, diff --git a/src/tool_runtime/model_ergonomics_telemetry/job_convergence.rs b/src/tool_runtime/model_ergonomics_telemetry/job_convergence.rs index 0b05690a7..c775e62f8 100644 --- a/src/tool_runtime/model_ergonomics_telemetry/job_convergence.rs +++ b/src/tool_runtime/model_ergonomics_telemetry/job_convergence.rs @@ -8,6 +8,10 @@ use serde::Serialize; use sha2::{Digest, Sha256}; const MAX_EVENTS: usize = webcodex_runner_registry::MAX_JOB_TELEMETRY_SNAPSHOTS; +#[cfg(feature = "experimental-code-mode")] +const MAX_CODE_MODE_RECEIPT_CHILDREN: usize = webcodex_code_mode::MAX_TOOL_CALLS; +#[cfg(not(feature = "experimental-code-mode"))] +const MAX_CODE_MODE_RECEIPT_CHILDREN: usize = 32; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] @@ -68,9 +72,67 @@ impl ToolRuntime { }; let mut selected = Vec::new(); let mut observation_complete = tool != "observe_jobs"; + let passive_eligible = + webcodex_tool_contracts::runtime_tool_supports_passive_job_attention(tool); + if matches!( + tool, + "execute_effectful_code_mode" | "execute_mutating_code_mode" + ) { + match result + .output + .pointer("/effect_receipt/children") + .and_then(serde_json::Value::as_array) + { + Some(children) if children.len() <= MAX_CODE_MODE_RECEIPT_CHILDREN => { + for child in children { + if child.get("outcome").and_then(serde_json::Value::as_str) + != Some("job_handoff") + { + continue; + } + let job_id = child.get("job_id").and_then(serde_json::Value::as_str); + let continuation = child.get("continuation"); + let continuation_job_id = continuation + .and_then(|value| value.get("arguments")) + .and_then(|value| value.get("items")) + .and_then(serde_json::Value::as_array) + .filter(|items| items.len() == 1) + .and_then(|items| items[0].get("job_id")) + .and_then(serde_json::Value::as_str); + let continuation_tool = continuation + .and_then(|value| value.get("tool")) + .and_then(serde_json::Value::as_str); + if let (Some(job_id), Some(continuation_job_id)) = + (job_id, continuation_job_id) + { + if continuation_tool == Some("observe_jobs") + && job_id == continuation_job_id + { + if !selected.iter().any(|(selected_id, kind, _, _)| { + *selected_id == job_id && *kind == JobEventKind::PendingHandoff + }) { + record.pending_handoff_count = + record.pending_handoff_count.saturating_add(1); + selected.push(( + job_id, + JobEventKind::PendingHandoff, + None, + None, + )); + } + } else { + observation_complete = false; + } + } else { + observation_complete = false; + } + } + } + Some(_) => observation_complete = false, + None => {} + } + } if result.success { - let passive_eligible = - webcodex_tool_contracts::runtime_tool_supports_passive_job_attention(tool); if let Some(id) = super::super::job_attention::pending_continuation_job_id(result) .filter(|_| passive_eligible) { @@ -130,6 +192,9 @@ impl ToolRuntime { if selected.is_empty() && record.wait_for_job_terminal_count == 0 && observation_complete { return None; } + if selected.len() > MAX_EVENTS { + observation_complete = false; + } selected.truncate(MAX_EVENTS); let (Some(auth), Some(window)) = (auth, window) else { return Some(record); diff --git a/src/tool_runtime/orchestration_host.rs b/src/tool_runtime/orchestration_host.rs index 871c34855..0d7d7c261 100644 --- a/src/tool_runtime/orchestration_host.rs +++ b/src/tool_runtime/orchestration_host.rs @@ -178,6 +178,10 @@ pub(crate) struct OrchestrationCompositionSummary { pub(crate) known_results: usize, pub(crate) job_handoffs: usize, pub(crate) outcome_unknown: usize, + /// Authoritative mutation effects from canonical child ToolResults. These + /// are aggregate observability facts only; they never expose paths or payloads. + pub(crate) mutation_state_changed: usize, + pub(crate) mutation_no_change: usize, } #[derive(Debug, Default)] @@ -241,6 +245,8 @@ impl OrchestrationCompositionAccumulator { known_results: 0, job_handoffs: 0, outcome_unknown: 0, + mutation_state_changed: 0, + mutation_no_change: 0, } } } @@ -679,6 +685,16 @@ impl CanonicalOrchestrationHost { summary.known_results = effects.known_results; summary.job_handoffs = effects.job_handoffs; summary.outcome_unknown = effects.outcome_unknown; + summary.mutation_state_changed = effects + .children + .iter() + .filter(|child| child.state_changed == Some(true)) + .count(); + summary.mutation_no_change = effects + .children + .iter() + .filter(|child| child.state_changed == Some(false)) + .count(); summary } diff --git a/src/tool_runtime/runtime_metrics.rs b/src/tool_runtime/runtime_metrics.rs index e83832f4b..8bc9f9f78 100644 --- a/src/tool_runtime/runtime_metrics.rs +++ b/src/tool_runtime/runtime_metrics.rs @@ -452,6 +452,8 @@ mod tests { known_results: 0, job_handoffs: 0, outcome_unknown: 0, + mutation_state_changed: 0, + mutation_no_change: 0, }; observe_code_mode_composition(&PanicMetrics, &observation); } diff --git a/src/tool_runtime/tests/job_attention/telemetry.rs b/src/tool_runtime/tests/job_attention/telemetry.rs index da853df0d..326493bba 100644 --- a/src/tool_runtime/tests/job_attention/telemetry.rs +++ b/src/tool_runtime/tests/job_attention/telemetry.rs @@ -40,6 +40,136 @@ async fn job_convergence_correlates_only_exact_authorized_relations_without_payl assert_eq!(facts.events[0].kind, JobEventKind::PendingHandoff); assert!(facts.events[0].terminal_observed_at_ms.is_none()); assert!(facts.correlation_complete); + let code_mode_pending = ToolResult::ok(json!({ + "effect_receipt": { + "children": [{ + "outcome": "job_handoff", + "job_id": job_id, + "continuation": super::super::super::jobs::observe_job_continuation(&job_id, None), + }] + } + })); + let code_mode_facts = runtime + .job_convergence_record( + "execute_effectful_code_mode", + &code_mode_pending, + &context, + Some(&auth), + Some(&window), + ) + .unwrap(); + assert_eq!(code_mode_facts.pending_handoff_count, 1); + assert_eq!(code_mode_facts.events.len(), 1); + assert_eq!(code_mode_facts.events[0].kind, JobEventKind::PendingHandoff); + assert_eq!(code_mode_facts.events[0].relation, facts.events[0].relation); + assert!(code_mode_facts.correlation_complete); + let mut mixed_children = vec![json!({"outcome": "known_result"}); 9]; + mixed_children.push(json!({ + "outcome": "job_handoff", + "job_id": job_id, + "continuation": super::super::super::jobs::observe_job_continuation(&job_id, None), + })); + let mixed_receipt = ToolResult::ok(json!({ + "effect_receipt": {"children": mixed_children} + })); + let mixed_facts = runtime + .job_convergence_record( + "execute_effectful_code_mode", + &mixed_receipt, + &context, + Some(&auth), + Some(&window), + ) + .unwrap(); + assert_eq!(mixed_facts.pending_handoff_count, 1); + assert_eq!(mixed_facts.events.len(), 1); + assert_eq!(mixed_facts.events[0].relation, facts.events[0].relation); + assert!( + mixed_facts.correlation_complete, + "non-handoff receipt children do not consume the telemetry event budget", + ); + let mut oversized_children = vec![json!({"outcome": "known_result"}); 32]; + oversized_children.push(json!({ + "outcome": "job_handoff", + "job_id": job_id, + "continuation": super::super::super::jobs::observe_job_continuation(&job_id, None), + })); + let oversized_receipt = ToolResult::ok(json!({ + "effect_receipt": {"children": oversized_children} + })); + let oversized_facts = runtime + .job_convergence_record( + "execute_effectful_code_mode", + &oversized_receipt, + &context, + Some(&auth), + Some(&window), + ) + .unwrap(); + assert!(oversized_facts.events.is_empty()); + assert!( + !oversized_facts.correlation_complete, + "receipt input beyond the canonical Code Mode child bound fails closed", + ); + assert!( + !serde_json::to_string(&code_mode_facts) + .unwrap() + .contains(&job_id), + "payload-safe convergence telemetry must not persist raw nested Job ids", + ); + + let failed_code_mode_pending = ToolResult::err_with_output( + "cell failed after handoff", + json!({ + "effect_receipt": { + "children": [{ + "outcome": "job_handoff", + "job_id": job_id, + "continuation": super::super::super::jobs::observe_job_continuation(&job_id, None), + }] + } + }), + ); + let failed_parent_facts = runtime + .job_convergence_record( + "execute_effectful_code_mode", + &failed_code_mode_pending, + &context, + Some(&auth), + Some(&window), + ) + .unwrap(); + assert_eq!(failed_parent_facts.pending_handoff_count, 1); + assert_eq!(failed_parent_facts.events.len(), 1); + assert_eq!( + failed_parent_facts.events[0].relation, facts.events[0].relation, + "canonical child handoff truth survives a failed Code Mode parent", + ); + assert!(failed_parent_facts.correlation_complete); + + let malformed_code_mode_pending = ToolResult::ok(json!({ + "effect_receipt": { + "children": [{ + "outcome": "job_handoff", + "job_id": job_id, + "continuation": { + "tool": "observe_jobs", + "arguments": {"items": [{"job_id": "different-job"}]} + } + }] + } + })); + let malformed = runtime + .job_convergence_record( + "execute_effectful_code_mode", + &malformed_code_mode_pending, + &context, + Some(&auth), + Some(&window), + ) + .unwrap(); + assert!(!malformed.correlation_complete); + assert!(malformed.events.is_empty()); let observe = ToolResult::ok( json!({"items": [{"job_id": job_id, "success": true, "output": {"stdout": "SECRET_OUTPUT", "stderr": "SECRET_ERROR"}}], "command": "SECRET_COMMAND", "source": "SECRET_SOURCE", "env": "SECRET_ENV"}), ); @@ -206,6 +336,77 @@ async fn job_convergence_correlates_only_exact_authorized_relations_without_payl .is_none()); } +#[cfg(feature = "experimental-code-mode")] +#[tokio::test] +async fn job_convergence_marks_combined_code_mode_handoff_and_attention_overflow_incomplete() { + let runtime = ToolRuntime::new_for_tests(); + let auth = shared_key_auth_context("telemetry-overflow-owner"); + let client = "telemetry-overflow"; + register_job_agent_for_auth(&runtime, client, "repo", &auth).await; + let project = format!("agent:{client}:repo"); + let session = runtime + .sessions + .start_session(Some(project.clone()), None) + .session_id; + let window = ClientWindow::for_test("telemetry-overflow-window"); + let context = ToolCallCorrelation { + resolved_project: Some(project), + business_session_id: Some(session.clone()), + ..Default::default() + }; + + assert!( + webcodex_tool_contracts::runtime_tool_supports_passive_job_attention( + "execute_effectful_code_mode" + ) + ); + let mut jobs = Vec::new(); + for _ in 0..10 { + jobs.push( + start_agent_runtime_job_in_session(&runtime, client, "repo", Some(&session), &auth) + .await, + ); + } + let children: Vec<_> = jobs[..9] + .iter() + .map(|job_id| { + json!({ + "outcome": "job_handoff", + "job_id": job_id, + "continuation": super::super::super::jobs::observe_job_continuation(job_id, None), + }) + }) + .collect(); + let result = ToolResult::ok(json!({ + "effect_receipt": {"children": children}, + "job_attention": { + "items": [{ + "job_id": jobs[9], + "state": "terminal", + "command_ok": true, + "validation": {"passed": true} + }] + } + })); + + let facts = runtime + .job_convergence_record( + "execute_effectful_code_mode", + &result, + &context, + Some(&auth), + Some(&window), + ) + .unwrap(); + assert_eq!(facts.pending_handoff_count, 9); + assert_eq!(facts.passive_terminal_delivery_count, 1); + assert_eq!(facts.events.len(), 9); + assert!( + !facts.correlation_complete, + "bounded event truncation must fail correlation completeness closed", + ); +} + #[tokio::test] async fn job_convergence_terminal_delivery_uses_existing_server_timestamp() { let runtime = ToolRuntime::new_for_tests(); diff --git a/src/tool_runtime/window_activity_projection.rs b/src/tool_runtime/window_activity_projection.rs index 58c9db0d6..9e5cf5127 100644 --- a/src/tool_runtime/window_activity_projection.rs +++ b/src/tool_runtime/window_activity_projection.rs @@ -90,6 +90,10 @@ pub(crate) struct RuntimeConsoleCodeModeComposition { pub(crate) known_results: usize, pub(crate) job_handoffs: usize, pub(crate) outcome_unknown: usize, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) mutation_state_changed: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) mutation_no_change: Option, } #[cfg(feature = "experimental-code-mode")] @@ -106,6 +110,10 @@ pub(crate) fn project_code_mode_composition( .known_results .checked_add(projection.job_handoffs) .and_then(|total| total.checked_add(projection.outcome_unknown))?; + let mutation_counted = projection + .mutation_state_changed + .unwrap_or(0) + .checked_add(projection.mutation_no_change.unwrap_or(0))?; if projection.nested_calls > 32 || projection.max_in_flight > 8 || projection @@ -115,6 +123,7 @@ pub(crate) fn project_code_mode_composition( || counted != projection.nested_calls || consequential_counted != projection.consequential_calls || projection.consequential_calls > projection.nested_calls + || mutation_counted > projection.consequential_calls || projection.nested_tool_counts.len() > 32 || projection .nested_tool_counts