diff --git a/autoresearch/prefill/candidate.py b/autoresearch/prefill/candidate.py index 572a694..62c2bf4 100644 --- a/autoresearch/prefill/candidate.py +++ b/autoresearch/prefill/candidate.py @@ -1,5 +1,19 @@ -"""The only Prefill strategy file the autoresearch agent may edit.""" +"""The only strategy file the AutoResearch agent may edit.""" +CANDIDATE_ID = "baseline-v1" +TARGET_OBLIGATION_ID = "RH-C1" +HYPOTHESIS = ( + "Force each experiment to attack one unresolved proof obligation with a " + "concrete construction or counterexample." +) +GENERATOR_DIRECTIVE = ( + "Focus on RH-C1. Propose one explicit non-circular operator definition, " + "including domain, kernel/action, and the exact theorem still required." +) +CRITIC_DIRECTIVE = ( + "Attempt to falsify the proposed RH-C1 operator. Reject placeholders and " + "identify the first invalid domain, self-adjointness, or spectrum step." +) PREFILL_COMPUTE_CHUNK_TOKENS = 256 SNAPSHOT_MODE = "final_only" MAX_SEGMENT_SECONDS = 300.0 diff --git a/autoresearch/prefill/prepare.py b/autoresearch/prefill/prepare.py index 90427d4..d4e5e65 100644 --- a/autoresearch/prefill/prepare.py +++ b/autoresearch/prefill/prepare.py @@ -54,6 +54,10 @@ def evaluate(report: dict, candidate) -> dict: estimated_max_segment_s <= candidate.MAX_SEGMENT_SECONDS ), "final_only_snapshot": candidate.SNAPSHOT_MODE == "final_only", + "candidate_requires_full_context": ( + candidate.REQUIRE_FULL_CONTEXT is True + ), + "candidate_forbids_fallback": candidate.ALLOW_FALLBACK is False, } return { "accepted": all(constraints.values()), @@ -61,6 +65,17 @@ def evaluate(report: dict, candidate) -> dict: "measured_prefill_tps": measured_tps, "estimated_max_segment_s": estimated_max_segment_s, "compute_chunk_tokens": candidate.PREFILL_COMPUTE_CHUNK_TOKENS, + "candidate_id": candidate.CANDIDATE_ID, + "target_obligation_id": candidate.TARGET_OBLIGATION_ID, + "proof_obligations_total": int( + critic.get("proof_obligations_total", 0), + ), + "proof_obligations_covered": int( + critic.get("proof_obligations_covered", 0), + ), + "proof_obligations_unresolved": int( + critic.get("proof_obligations_unresolved", 0), + ), "constraints": constraints, } diff --git a/autoresearch/prefill/program.md b/autoresearch/prefill/program.md index 55c2a6f..2062eb3 100644 --- a/autoresearch/prefill/program.md +++ b/autoresearch/prefill/program.md @@ -1,6 +1,6 @@ # Prefill AutoResearch Program -You are optimizing the two-Mac full-context Prefill system. +You are optimizing the two-Mac full-context RH proof research system. ## Ownership @@ -10,7 +10,10 @@ You are optimizing the two-Mac full-context Prefill system. ## Objective -Minimize `metric_cold_critic_prefill_s`. Lower is better. +Use a lexicographic objective: + +1. Minimize unresolved Proof Obligation Ledger items. +2. With equal unresolved count, minimize `metric_cold_critic_prefill_s`. ## Hard constraints @@ -36,6 +39,9 @@ Minimize `metric_cold_critic_prefill_s`. Lower is better. Prefill time improves. Otherwise restore the previous candidate. 9. Append the result and repeat. +Every candidate must target one current unresolved proof obligation and contain +a falsifiable hypothesis plus distinct Generator and Critic directives. + Do not optimize output wording, scores, prizes, or other proof-irrelevant content. Optimize only measured Prefill execution while preserving the complete semantic contract. diff --git a/autoresearch/prefill/supervisor.py b/autoresearch/prefill/supervisor.py new file mode 100644 index 0000000..da3bcee --- /dev/null +++ b/autoresearch/prefill/supervisor.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python3 +"""Real Karpathy-style AutoResearch supervisor around one-shot GAN experiments.""" +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import os +import plistlib +import re +import shutil +import socket +import subprocess +import time +import urllib.request +from pathlib import Path + +from autoresearch.prefill.prepare import _load_candidate, evaluate + + +REQUIRED_CANDIDATE_FIELDS = ( + "candidate_id", + "target_obligation_id", + "hypothesis", + "generator_directive", + "critic_directive", + "prefill_compute_chunk_tokens", +) + + +def _json_request(url: str) -> dict: + with urllib.request.urlopen(url, timeout=10) as response: + return json.load(response) + + +def _wait_port(host: str, port: int, timeout_s: float = 180) -> None: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + try: + with socket.create_connection((host, port), timeout=2): + return + except OSError: + time.sleep(2) + raise TimeoutError(f"service did not become ready: {host}:{port}") + + +def _candidate_snapshot(module) -> dict: + return { + "candidate_id": str(module.CANDIDATE_ID), + "target_obligation_id": str(module.TARGET_OBLIGATION_ID), + "hypothesis": str(module.HYPOTHESIS), + "generator_directive": str(module.GENERATOR_DIRECTIVE), + "critic_directive": str(module.CRITIC_DIRECTIVE), + "prefill_compute_chunk_tokens": int( + module.PREFILL_COMPUTE_CHUNK_TOKENS, + ), + "snapshot_mode": str(module.SNAPSHOT_MODE), + "max_segment_seconds": float(module.MAX_SEGMENT_SECONDS), + "require_full_context": bool(module.REQUIRE_FULL_CONTEXT), + "allow_fallback": bool(module.ALLOW_FALLBACK), + } + + +def validate_candidate(candidate: dict) -> None: + missing = [field for field in REQUIRED_CANDIDATE_FIELDS if not candidate.get(field)] + if missing: + raise ValueError(f"candidate missing fields: {missing}") + if candidate["prefill_compute_chunk_tokens"] not in (64, 128, 256): + raise ValueError("chunk tokens must be one of 64, 128, 256") + if candidate.get("snapshot_mode", "final_only") != "final_only": + raise ValueError("snapshot mode must remain final_only") + if candidate.get("require_full_context", True) is not True: + raise ValueError("candidate must require full context") + if candidate.get("allow_fallback", False) is not False: + raise ValueError("candidate must forbid fallback") + + +def render_candidate(candidate: dict) -> str: + validate_candidate(candidate) + return ( + '"""AutoResearch agent-editable strategy. Generated by supervisor."""\n\n' + f"CANDIDATE_ID = {candidate['candidate_id']!r}\n" + f"TARGET_OBLIGATION_ID = {candidate['target_obligation_id']!r}\n" + f"HYPOTHESIS = {candidate['hypothesis']!r}\n" + f"GENERATOR_DIRECTIVE = {candidate['generator_directive']!r}\n" + f"CRITIC_DIRECTIVE = {candidate['critic_directive']!r}\n" + f"PREFILL_COMPUTE_CHUNK_TOKENS = " + f"{candidate['prefill_compute_chunk_tokens']}\n" + 'SNAPSHOT_MODE = "final_only"\n' + f"MAX_SEGMENT_SECONDS = {candidate.get('max_segment_seconds', 300.0)!r}\n" + "REQUIRE_FULL_CONTEXT = True\n" + "ALLOW_FALLBACK = False\n" + ) + + +def _extract_json(text: str) -> dict: + stripped = text.strip() + if stripped.startswith("```"): + stripped = re.sub(r"^```(?:json)?\\s*", "", stripped) + stripped = re.sub(r"\\s*```$", "", stripped) + start, end = stripped.find("{"), stripped.rfind("}") + if start < 0 or end <= start: + raise ValueError("strategy agent returned no JSON object") + return json.loads(stripped[start:end + 1]) + + +def propose_candidate( + *, + address: str, + tokenizer_id: str, + program: str, + current: dict, + results_text: str, + ledger: dict, +) -> dict: + from kakeya import Client + from transformers import AutoTokenizer + from scripts.chat_grpc import _resolve_eos_token_ids + + tokenizer = AutoTokenizer.from_pretrained(tokenizer_id) + prompt = ( + "You are the AutoResearch strategy agent. Follow the human-owned " + "program exactly. Select one unresolved proof obligation and propose " + "one falsifiable GAN strategy experiment. Return JSON only with keys: " + + ", ".join(REQUIRED_CANDIDATE_FIELDS) + + ". Allowed prefill_compute_chunk_tokens: 64, 128, 256. " + "Do not weaken full context, final-only snapshots, or no-fallback rules." + f"\n\nPROGRAM:\n{program}\n\nCURRENT:\n{json.dumps(current)}" + f"\n\nRESULTS:\n{results_text[-12000:]}" + f"\n\nLEDGER:\n{json.dumps(ledger)}" + ) + ids = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + tokenize=True, + return_dict=False, + enable_thinking=False, + ) + generated: list[int] = [] + with Client(address) as client: + with client.create_session( + eos_token_ids=_resolve_eos_token_ids(tokenizer), + client_label="autoresearch-strategy", + ) as session: + session.append(ids) + while len(generated) < 2048: + before = len(generated) + generated.extend(int(token) for token in session.generate(max_tokens=64)) + if session.last_stop_reason != 1: + break + if len(generated) == before: + raise RuntimeError("strategy agent made no progress") + if session.last_stop_reason != 2: + raise RuntimeError( + f"strategy agent did not reach EOS: {session.last_stop_reason}", + ) + candidate = _extract_json( + tokenizer.decode(generated, skip_special_tokens=True), + ) + candidate.update({ + "snapshot_mode": "final_only", + "max_segment_seconds": 300.0, + "require_full_context": True, + "allow_fallback": False, + }) + validate_candidate(candidate) + return candidate + + +def deploy_candidate(worker_ssh: str, chunk_tokens: int) -> None: + remote = f"""python3 - <<'PY' +import os, plistlib +from pathlib import Path +p=Path.home()/'Library/LaunchAgents/ai.kakeya.prefill-worker.plist' +d=plistlib.loads(p.read_bytes()) +a=d['ProgramArguments'] +i=a.index('--prefill-compute-chunk-tokens') +a[i+1]='{chunk_tokens}' +t=p.with_suffix('.plist.tmp') +t.write_bytes(plistlib.dumps(d)) +os.chmod(t,0o644) +t.replace(p) +PY +launchctl bootout gui/$(id -u)/ai.kakeya.prefill-worker 2>/dev/null || true +sleep 3 +launchctl bootstrap gui/$(id -u) \"$HOME/Library/LaunchAgents/ai.kakeya.prefill-worker.plist\" +launchctl kickstart -k gui/$(id -u)/ai.kakeya.prefill-worker +""" + subprocess.run( + ["ssh", "-o", "BatchMode=yes", worker_ssh, remote], + check=True, + ) + _wait_port("169.254.27.104", 53051) + probe = subprocess.run( + ["ssh", worker_ssh, "ps -ax -o command="], + check=True, + capture_output=True, + text=True, + ).stdout + expected = f"--prefill-compute-chunk-tokens {chunk_tokens}" + if expected not in probe: + raise RuntimeError("deployed worker chunk size verification failed") + + +def clear_primary_cache() -> None: + subprocess.run( + [ + "launchctl", "kickstart", "-k", + f"gui/{os.getuid()}/ai.kakeya.grpc-runtime-prefill", + ], + check=True, + ) + _wait_port("127.0.0.1", 51051) + _wait_port("127.0.0.1", 8090) + + +def _backup(path: Path) -> bytes | None: + return path.read_bytes() if path.exists() else None + + +def _restore(path: Path, content: bytes | None) -> None: + if content is None: + path.unlink(missing_ok=True) + else: + temporary = path.with_suffix(path.suffix + ".restore") + temporary.write_bytes(content) + os.chmod(temporary, 0o600) + temporary.replace(path) + + +def run_gan_experiment( + *, + repo: Path, + candidate_path: Path, + state_path: Path, + timeout_s: float, +) -> tuple[str, dict, str]: + command = [ + "bash", str(repo / "scripts/run_agent_gan_repl.sh"), + "--skip-ensure", "--no-auto-loop", + "--candidate-file", str(candidate_path), + "--state-file", str(state_path), + ] + result = subprocess.run( + command, + input="/continue\n/quit\n", + text=True, + capture_output=True, + timeout=timeout_s, + cwd=repo, + ) + output = result.stdout + result.stderr + if result.returncode != 0: + raise RuntimeError(f"GAN experiment failed ({result.returncode}): {output[-4000:]}") + matches = re.findall(r"run=(br_[0-9a-f]+)", output) + if not matches: + raise RuntimeError("GAN experiment produced no benchmark run id") + run_id = matches[-1] + report = _json_request( + f"http://127.0.0.1:8090/v1/network/benchmarks/{run_id}", + ) + if report.get("status") != "completed": + raise RuntimeError(f"GAN benchmark is not completed: {report.get('status')}") + return run_id, report, output + + +def read_results(path: Path) -> list[dict]: + if not path.exists(): + return [] + with path.open(newline="") as handle: + return list(csv.DictReader(handle, delimiter="\t")) + + +def best_kept(results: list[dict]) -> dict | None: + kept = [row for row in results if row.get("kept") == "True"] + if not kept: + return None + return min( + kept, + key=lambda row: ( + int(row["proof_obligations_unresolved"]), + float(row["metric_cold_critic_prefill_s"]), + ), + ) + + +def should_keep(result: dict, baseline: dict | None) -> bool: + if not result["accepted"]: + return False + if baseline is None: + return True + new_key = ( + int(result["proof_obligations_unresolved"]), + float(result["metric_cold_critic_prefill_s"]), + ) + old_key = ( + int(baseline["proof_obligations_unresolved"]), + float(baseline["metric_cold_critic_prefill_s"]), + ) + return new_key < old_key + + +RESULT_FIELDS = ( + "timestamp", "experiment_id", "run_id", "candidate_id", + "target_obligation_id", "constraints_pass", "accepted", "kept", + "metric_cold_critic_prefill_s", "baseline_metric_s", + "proof_obligations_total", "proof_obligations_covered", + "proof_obligations_unresolved", "compute_chunk_tokens", + "candidate_sha256", "report_path", +) + + +def append_result(path: Path, row: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + write_header = not path.exists() + with path.open("a", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=RESULT_FIELDS, delimiter="\t") + if write_header: + writer.writeheader() + writer.writerow({field: row.get(field, "") for field in RESULT_FIELDS}) + + +def run_iteration(args, iteration: int) -> dict: + root = Path(__file__).resolve().parents[2] + ar = Path(__file__).resolve().parent + candidate_path = ar / "candidate.py" + results_path = Path(args.results).expanduser() + reports_dir = Path(args.reports_dir).expanduser() + reports_dir.mkdir(parents=True, exist_ok=True) + state_path = Path(args.state_file).expanduser() + ledger_path = Path(args.proof_ledger).expanduser() + program = (ar / "program.md").read_text() + results = read_results(results_path) + baseline = best_kept(results) + current_module = _load_candidate(candidate_path) + current = _candidate_snapshot(current_module) + previous_candidate = candidate_path.read_bytes() + previous_state = _backup(state_path) + previous_ledger = _backup(ledger_path) + previous_chunk = int(current["prefill_compute_chunk_tokens"]) + + if baseline is None and iteration == 0: + proposed = current + else: + proposed = propose_candidate( + address=args.address, + tokenizer_id=args.tokenizer_id, + program=program, + current=current, + results_text=results_path.read_text() if results_path.exists() else "", + ledger=json.loads(ledger_path.read_text()), + ) + candidate_path.write_text(render_candidate(proposed)) + validate_candidate(proposed) + experiment_id = ( + f"ar_{int(time.time())}_{iteration}_" + f"{hashlib.sha256(candidate_path.read_bytes()).hexdigest()[:8]}" + ) + report_path = reports_dir / f"{experiment_id}.json" + try: + deploy_candidate(args.worker_ssh, proposed["prefill_compute_chunk_tokens"]) + clear_primary_cache() + run_id, report, _ = run_gan_experiment( + repo=root, + candidate_path=candidate_path, + state_path=state_path, + timeout_s=args.experiment_timeout_s, + ) + report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2)) + candidate_module = _load_candidate(candidate_path) + result = evaluate(report, candidate_module) + keep = should_keep(result, baseline) + row = { + "timestamp": time.time(), + "experiment_id": experiment_id, + "run_id": run_id, + "candidate_id": proposed["candidate_id"], + "target_obligation_id": proposed["target_obligation_id"], + "constraints_pass": result["accepted"], + "accepted": result["accepted"], + "kept": keep, + "metric_cold_critic_prefill_s": result[ + "metric_cold_critic_prefill_s" + ], + "baseline_metric_s": ( + baseline["metric_cold_critic_prefill_s"] if baseline else "" + ), + "proof_obligations_total": result["proof_obligations_total"], + "proof_obligations_covered": result["proof_obligations_covered"], + "proof_obligations_unresolved": result[ + "proof_obligations_unresolved" + ], + "compute_chunk_tokens": result["compute_chunk_tokens"], + "candidate_sha256": hashlib.sha256( + candidate_path.read_bytes(), + ).hexdigest(), + "report_path": str(report_path), + } + append_result(results_path, row) + if not keep: + candidate_path.write_bytes(previous_candidate) + _restore(state_path, previous_state) + _restore(ledger_path, previous_ledger) + deploy_candidate(args.worker_ssh, previous_chunk) + return row + except Exception: + candidate_path.write_bytes(previous_candidate) + _restore(state_path, previous_state) + _restore(ledger_path, previous_ledger) + deploy_candidate(args.worker_ssh, previous_chunk) + raise + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--iterations", type=int, default=1) + parser.add_argument("--worker-ssh", default="allens") + parser.add_argument("--address", default="127.0.0.1:51051") + parser.add_argument( + "--tokenizer-id", + default=str( + Path.home() + / "kakeya-models/gemma-4-26B-A4B-it-mlx-4bit" + ), + ) + parser.add_argument( + "--results", + default=str(Path.home() / ".kakeya/autoresearch/prefill/results.tsv"), + ) + parser.add_argument( + "--reports-dir", + default=str(Path.home() / ".kakeya/autoresearch/prefill/reports"), + ) + parser.add_argument( + "--state-file", + default=str(Path.home() / ".kakeya/agent_gan_state.json"), + ) + parser.add_argument( + "--proof-ledger", + default=str(Path.home() / ".kakeya/agent_gan_proof_ledger.json"), + ) + parser.add_argument("--experiment-timeout-s", type=float, default=7200) + args = parser.parse_args() + if args.iterations <= 0: + raise SystemExit("iterations must be > 0") + for iteration in range(args.iterations): + row = run_iteration(args, iteration) + print(json.dumps(row, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/ops/distributed-prefill-kv-network.md b/docs/ops/distributed-prefill-kv-network.md index e4b9f87..fd99f47 100644 --- a/docs/ops/distributed-prefill-kv-network.md +++ b/docs/ops/distributed-prefill-kv-network.md @@ -427,6 +427,20 @@ Karpathy-style optimization lives in `autoresearch/prefill/`. Humans edit `prepare.py` evaluates full-context correctness plus cold Critic Prefill time. Candidates are retained only when every semantic/topology constraint passes and the metric improves; `results.tsv` records experiments. +`autoresearch/prefill/supervisor.py` is the top-level runtime: it uses a real +Gemma strategy-agent call to rewrite `candidate.py`, deploys the selected chunk +strategy to allens, clears both cache tiers, runs one real full-context +Generator/Critic/Proof-Ledger experiment, evaluates the fixed report, compares +the lexicographic proof/prefill baseline, atomically keeps or restores +candidate/state/ledger, appends `results.tsv`, and repeats. Any proposal, +deployment, telemetry, GAN, evaluator, or restore failure terminates the +supervisor; there is no fallback path. + +Run: + +```bash +bash scripts/run_autoresearch_gan.sh --iterations 1 +``` Interactive prompt templates are deterministic and contain no per-run nonce, so repeating the same task can reuse allens cold-tier and Primary hot-tier KV. diff --git a/scripts/agent_gan_repl.py b/scripts/agent_gan_repl.py index ac2f71b..79e6020 100644 --- a/scripts/agent_gan_repl.py +++ b/scripts/agent_gan_repl.py @@ -712,6 +712,11 @@ def main() -> int: default="~/.kakeya/agent_gan_proof_ledger.json", help="Private persistent mathematical proof obligations.", ) + parser.add_argument( + "--candidate-file", + default="", + help="AutoResearch candidate strategy applied to this experiment.", + ) parser.add_argument("--recover-run", default="") parser.add_argument( "--recover-log", @@ -736,6 +741,12 @@ def main() -> int: raise SystemExit("output-tokens must be > 0") if args.auto_loop_boundary_wait_s < 0: raise SystemExit("auto-loop-boundary-wait-s must be >= 0") + research_candidate = None + if args.candidate_file: + from autoresearch.prefill.prepare import _load_candidate + research_candidate = _load_candidate( + Path(args.candidate_file).expanduser(), + ) transcript = TimestampedTee( sys.stdout, Path(args.log_file).expanduser(), @@ -863,6 +874,16 @@ def get_stats(): auto_loop_active = bool(args.auto_loop) print(f"[goal] reset: {research_goal}", flush=True) steering = command.payload if command.action == "steer" else "" + generator_steering = steering + critic_strategy = "" + if research_candidate is not None: + generator_steering = "\n\n".join(filter(None, ( + steering, + str(research_candidate.GENERATOR_DIRECTIVE), + ))) + critic_strategy = str( + research_candidate.CRITIC_DIRECTIVE, + ) if command.action in {"continue", "steer"} and args.auto_loop: auto_loop_active = True phase = ReplPhase.RUNNING @@ -942,6 +963,14 @@ def get_stats(): if proof_ledger is not None else 0 ), "proof_obligations_pending": len(turn_obligations), + "autoresearch_candidate_id": ( + str(research_candidate.CANDIDATE_ID) + if research_candidate is not None else "" + ), + "autoresearch_target_obligation": ( + str(research_candidate.TARGET_OBLIGATION_ID) + if research_candidate is not None else "" + ), }, }, ) @@ -958,7 +987,7 @@ def get_stats(): try: generator_messages = build_generator_messages( research_goal, - steering=steering, + steering=generator_steering, previous_generator=previous_generator, previous_critic=( previous_critic + critic_issue_injection @@ -1041,7 +1070,11 @@ def get_stats(): critic_messages = build_critic_messages( research_goal, critic_context, - steering=steering + critic_issue_injection, + steering="\n\n".join(filter(None, ( + steering, + critic_strategy, + critic_issue_injection, + ))), proof_ledger=proof_ledger_text + ( "\nGENERATOR COVERAGE FAILURE: missing " + ", ".join(sorted(missing_issues)) diff --git a/scripts/run_autoresearch_gan.sh b/scripts/run_autoresearch_gan.sh new file mode 100644 index 0000000..5bb096e --- /dev/null +++ b/scripts/run_autoresearch_gan.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PYTHON="${KAKEYA_BENCH_PYTHON:-$HOME/.venv-distwan/bin/python}" + +exec env PYTHONPATH="$ROOT:$ROOT/sdks/python" \ + "$PYTHON" "$ROOT/autoresearch/prefill/supervisor.py" "$@" diff --git a/tests/inference_engine/bench/test_autoresearch_supervisor.py b/tests/inference_engine/bench/test_autoresearch_supervisor.py new file mode 100644 index 0000000..560e4c4 --- /dev/null +++ b/tests/inference_engine/bench/test_autoresearch_supervisor.py @@ -0,0 +1,101 @@ +from autoresearch.prefill.supervisor import ( + append_result, + best_kept, + read_results, + render_candidate, + should_keep, + validate_candidate, +) + + +def _candidate(): + return { + "candidate_id": "trial", + "target_obligation_id": "RH-C1", + "hypothesis": "Construct and attack one explicit operator.", + "generator_directive": "Define the operator.", + "critic_directive": "Falsify the operator.", + "prefill_compute_chunk_tokens": 256, + "snapshot_mode": "final_only", + "max_segment_seconds": 300.0, + "require_full_context": True, + "allow_fallback": False, + } + + +def test_candidate_render_is_executable_and_strict(tmp_path): + candidate = _candidate() + validate_candidate(candidate) + path = tmp_path / "candidate.py" + path.write_text(render_candidate(candidate)) + namespace = {} + exec(compile(path.read_text(), str(path), "exec"), namespace) + assert namespace["CANDIDATE_ID"] == "trial" + assert namespace["PREFILL_COMPUTE_CHUNK_TOKENS"] == 256 + bad = {**candidate, "allow_fallback": True} + try: + validate_candidate(bad) + except ValueError: + pass + else: + raise AssertionError("fallback candidate must be rejected") + + +def test_keep_is_lexicographic_on_proof_then_prefill(): + baseline = { + "proof_obligations_unresolved": "5", + "metric_cold_critic_prefill_s": "500", + } + assert should_keep({ + "accepted": True, + "proof_obligations_unresolved": 4, + "metric_cold_critic_prefill_s": 900, + }, baseline) + assert should_keep({ + "accepted": True, + "proof_obligations_unresolved": 5, + "metric_cold_critic_prefill_s": 499, + }, baseline) + assert not should_keep({ + "accepted": True, + "proof_obligations_unresolved": 5, + "metric_cold_critic_prefill_s": 501, + }, baseline) + assert not should_keep({ + "accepted": False, + "proof_obligations_unresolved": 0, + "metric_cold_critic_prefill_s": 1, + }, baseline) + + +def test_results_are_append_only_and_best_is_selected(tmp_path): + path = tmp_path / "results.tsv" + common = { + "timestamp": 1, + "experiment_id": "e1", + "run_id": "br_1", + "candidate_id": "c1", + "target_obligation_id": "RH-C1", + "constraints_pass": True, + "accepted": True, + "kept": True, + "metric_cold_critic_prefill_s": 500, + "baseline_metric_s": "", + "proof_obligations_total": 5, + "proof_obligations_covered": 5, + "proof_obligations_unresolved": 5, + "compute_chunk_tokens": 256, + "candidate_sha256": "a", + "report_path": "/tmp/r1.json", + } + append_result(path, common) + append_result(path, { + **common, + "timestamp": 2, + "experiment_id": "e2", + "candidate_id": "c2", + "metric_cold_critic_prefill_s": 450, + }) + rows = read_results(path) + assert len(rows) == 2 + assert best_kept(rows)["candidate_id"] == "c2" diff --git a/tests/inference_engine/bench/test_prefill_autoresearch.py b/tests/inference_engine/bench/test_prefill_autoresearch.py index 4bef89c..9c079cd 100644 --- a/tests/inference_engine/bench/test_prefill_autoresearch.py +++ b/tests/inference_engine/bench/test_prefill_autoresearch.py @@ -2,9 +2,13 @@ class Candidate: + CANDIDATE_ID = "test" + TARGET_OBLIGATION_ID = "RH-C1" PREFILL_COMPUTE_CHUNK_TOKENS = 256 SNAPSHOT_MODE = "final_only" MAX_SEGMENT_SECONDS = 300 + REQUIRE_FULL_CONTEXT = True + ALLOW_FALLBACK = False def _report(**overrides): @@ -19,6 +23,9 @@ def _report(**overrides): "critic_context_tokens": 900, "critic_omitted_tokens": 0, "critic_protocol": "goal_anchored_recursive_gan_v3", + "proof_obligations_total": 5, + "proof_obligations_covered": 5, + "proof_obligations_unresolved": 5, "delta": {"fallbacks": 0, "remote_job_failures": 0}, } stage.update(overrides) @@ -38,6 +45,10 @@ def test_autoresearch_rejects_slow_segment_or_semantic_regression(): "PREFILL_COMPUTE_CHUNK_TOKENS": 512, "SNAPSHOT_MODE": "final_only", "MAX_SEGMENT_SECONDS": 300, + "CANDIDATE_ID": "slow", + "TARGET_OBLIGATION_ID": "RH-C1", + "REQUIRE_FULL_CONTEXT": True, + "ALLOW_FALLBACK": False, }) assert not evaluate(_report(), slow)["accepted"] assert not evaluate(