diff --git a/autoresearch/prefill/program.md b/autoresearch/prefill/program.md index ce45134..efd18df 100644 --- a/autoresearch/prefill/program.md +++ b/autoresearch/prefill/program.md @@ -32,7 +32,7 @@ Use a lexicographic objective: ## Experiment loop 1. Read `candidate.py` and `results.tsv`. -2. State one concrete mathematical hypothesis for one unresolved leaf. +2. Let the host deterministically select the deepest unresolved leaf. 3. Modify only `candidate.py`. 4. Verify Primary and allens health without restarting either service. 5. Preserve all KV caches across decomposition iterations. @@ -45,6 +45,10 @@ Use a lexicographic objective: Every candidate must target exactly one current unresolved leaf obligation and contain a falsifiable hypothesis plus distinct Generator and Critic directives. +Normal iterations use the deterministic host candidate and do not call the +Strategy model. Invoke Strategy Gemma only after three consecutive +non-progressing runs, after a falsified branch, or via an explicit CLI/trigger +file request. An unresolved Critic verdict must isolate one strictly smaller missing lemma; the host records that lemma as a deduplicated child obligation. Completed GAN runs, transcripts, checkpoints, and ledger updates remain durable even when the @@ -58,7 +62,7 @@ must be deployed outside this supervisor. Cold benchmarks are explicit, separate invocations of `scripts/benchmark_prefill_architecture.py`. Prefill budgets are hard admission limits, never truncation instructions. -Strategy input must fit 8192 tokens by carrying the complete active leaf +Strategy input must fit 8448 tokens by carrying the complete active leaf ancestry and its exact experiment records. Generator and Critic inputs must fit 6144 tokens; the Critic always receives the complete current Generator output. Repeated Strategy strings are interned once in `text_by_id`; `_ref` fields @@ -73,6 +77,9 @@ model-invented ID. Reject a proposed child when it duplicates an existing statement or lemma name, is highly similar to an ancestor, or is too vague to be falsifiable. A rejected cyclic frontier is `INCONCLUSIVE`, not progress. +`DECOMPOSED` is keepable only when the host actually persisted at least one +new child that passed the ID, novelty, cycle, and falsifiability gates. + Do not optimize output wording, scores, prizes, or other proof-irrelevant content. Prefill performance is a tertiary objective after mathematical decomposition progress, while preserving the complete semantic contract. diff --git a/autoresearch/prefill/supervisor.py b/autoresearch/prefill/supervisor.py index 811c543..e8b992e 100644 --- a/autoresearch/prefill/supervisor.py +++ b/autoresearch/prefill/supervisor.py @@ -367,6 +367,9 @@ def parse_research_verdict(output: str, candidate_id: str) -> dict: "outcome": fields["outcome"], "evidence": fields["evidence"], "new_frontier": fields["new_frontier"], + "created_obligation_ids": list( + fields.get("created_obligation_ids", []), + ), } matches = list(re.finditer( r"^(?:critic>\s*)?### AUTORESEARCH_VERDICT\s*$" @@ -397,6 +400,7 @@ def parse_research_verdict(output: str, candidate_id: str) -> dict: "outcome": fields["Outcome"], "evidence": fields["Evidence"], "new_frontier": fields["New frontier"], + "created_obligation_ids": [], } @@ -626,7 +630,7 @@ def propose_candidate( current: dict, results_text: str, ledger: dict, - max_prefill_tokens: int = 8192, + max_prefill_tokens: int = 8448, ) -> dict: from kakeya import Client from transformers import AutoTokenizer @@ -826,14 +830,102 @@ def best_kept(results: list[dict]) -> dict | None: ) +def _created_ids(row: dict) -> list[str]: + raw = row.get("created_obligation_ids", "") + if isinstance(raw, list): + return [str(item) for item in raw if item] + if not raw: + return [] + try: + value = json.loads(raw) + except (TypeError, json.JSONDecodeError): + return [] + return [str(item) for item in value if item] if isinstance(value, list) else [] + + +def _row_made_progress(row: dict) -> bool: + if row.get("kept") not in {True, "True"}: + return False + outcome = row.get("research_outcome") + if outcome in {"SUPPORTED", "FALSIFIED"}: + return True + if outcome == "DECOMPOSED": + # Legacy rows predate created_obligation_ids but were already admitted + # by the host's child-creation gate. + return bool(_created_ids(row)) or not row.get( + "created_obligation_ids", + ) + return False + + +def strategy_trigger_reason( + results: list[dict], + *, + stagnation_rounds: int, + force: bool = False, + trigger_file: Path | None = None, +) -> str: + if force: + return "manual-cli" + if trigger_file is not None and trigger_file.exists(): + return "manual-trigger-file" + if results and results[-1].get("research_outcome") == "FALSIFIED": + return "branch-falsified" + stagnant = 0 + for row in reversed(results): + if _row_made_progress(row): + break + stagnant += 1 + if stagnant >= stagnation_rounds: + return f"stagnation-{stagnant}" + return "" + + +def build_host_candidate(current: dict, ledger: dict) -> dict: + target_id = _select_repair_target(current, ledger) + target = next( + item + for item in ledger.get("obligations", []) + if item.get("obligation_id") == target_id + ) + statement = str(target.get("statement", "")).strip() + evidence = str(target.get("last_evidence", "")).strip() + digest = hashlib.sha256(target_id.encode()).hexdigest()[:12] + candidate = { + "candidate_id": f"host-leaf-{digest}", + "target_obligation_id": target_id, + "hypothesis": statement, + "generator_directive": ( + f"Resolve or falsify the exact target leaf {target_id}: " + f"{statement} Previous Critic evidence: {evidence or '(none)'}. " + "Provide an explicit derivation or counterexample; do not rename " + "the same gap as a new lemma." + ), + "critic_directive": ( + f"Adversarially test target leaf {target_id}. Reject unsupported " + "existence claims and semantic restatements. Mark PROVED or " + "DISPROVED only with explicit evidence; otherwise identify one " + "strictly smaller, falsifiable missing obligation." + ), + "prefill_compute_chunk_tokens": int( + current["prefill_compute_chunk_tokens"], + ), + "snapshot_mode": "final_only", + "max_segment_seconds": 300.0, + "require_full_context": True, + "allow_fallback": False, + } + validate_candidate(candidate) + return candidate + + def should_keep(result: dict, baseline: dict | None) -> bool: if not result["accepted"]: return False - if result.get("research_outcome") not in { - "SUPPORTED", "FALSIFIED", "DECOMPOSED", - }: + outcome = result.get("research_outcome") + if outcome not in {"SUPPORTED", "FALSIFIED", "DECOMPOSED"}: return False - if not result.get("hypothesis_novel", False): + if outcome == "DECOMPOSED" and not result.get("created_obligation_ids"): return False if baseline is None: return True @@ -850,7 +942,8 @@ def should_keep(result: dict, baseline: dict | None) -> bool: "proof_obligations_unresolved", "compute_chunk_tokens", "candidate_sha256", "report_path", "hypothesis_sha256", "research_outcome", "research_evidence", - "new_frontier", "transcript_path", "error", + "new_frontier", "created_obligation_ids", "strategy_mode", + "transcript_path", "error", ) @@ -910,6 +1003,7 @@ def run_iteration(args, iteration: int) -> dict: transcript_path = reports_dir / "not-started.log" hypothesis_sha256 = "" candidate_sha256 = hashlib.sha256(previous_candidate).hexdigest() + strategy_mode = "baseline" try: print( f"[autoresearch] iteration={iteration} " @@ -926,17 +1020,26 @@ def run_iteration(args, iteration: int) -> dict: f"kv_hit_rate={health.get('kv_hit_rate', 0):.1%}", flush=True, ) + ledger_data = json.loads(ledger_path.read_text()) + trigger_file = Path(args.strategy_trigger_file).expanduser() + trigger_reason = strategy_trigger_reason( + results, + stagnation_rounds=args.strategy_stagnation_rounds, + force=args.force_strategy and iteration == 0, + trigger_file=trigger_file, + ) if baseline is None and iteration == 0: print( "[autoresearch] phase=baseline using current candidate", flush=True, ) - else: + elif trigger_reason: + strategy_mode = "gemma" print( - "[autoresearch] phase=strategy-proposal real-gemma", + "[autoresearch] phase=strategy-proposal " + f"mode=gemma trigger={trigger_reason}", flush=True, ) - ledger_data = json.loads(ledger_path.read_text()) proposed = propose_candidate( address=args.address, tokenizer_id=args.tokenizer_id, @@ -954,13 +1057,24 @@ def run_iteration(args, iteration: int) -> dict: raise ValueError( "strategy agent targeted a non-leaf proof obligation", ) - candidate_path.write_text(render_candidate(proposed)) + if trigger_reason == "manual-trigger-file": + trigger_file.unlink(missing_ok=True) + else: + strategy_mode = "host" + proposed = build_host_candidate(current, ledger_data) print( - f"[autoresearch] phase=candidate-written " - f"candidate={proposed['candidate_id']} " + "[autoresearch] phase=deterministic-candidate " f"target={proposed['target_obligation_id']}", flush=True, ) + candidate_path.write_text(render_candidate(proposed)) + print( + f"[autoresearch] phase=candidate-written " + f"candidate={proposed['candidate_id']} " + f"target={proposed['target_obligation_id']} " + f"mode={strategy_mode}", + flush=True, + ) validate_candidate(proposed) hypothesis_sha256 = hashlib.sha256( proposed["hypothesis"].strip().lower().encode(), @@ -970,7 +1084,8 @@ def run_iteration(args, iteration: int) -> dict: for row in results if row.get("hypothesis_sha256") } - if hypothesis_sha256 in seen_hypotheses: + hypothesis_novel = hypothesis_sha256 not in seen_hypotheses + if strategy_mode == "gemma" and not hypothesis_novel: raise ValueError("strategy agent repeated a previous hypothesis") candidate_sha256 = hashlib.sha256( candidate_path.read_bytes(), @@ -1011,8 +1126,9 @@ def run_iteration(args, iteration: int) -> dict: "research_outcome": verdict["outcome"], "research_evidence": verdict["evidence"], "new_frontier": verdict["new_frontier"], + "created_obligation_ids": verdict["created_obligation_ids"], "transcript_path": str(transcript_path), - "hypothesis_novel": True, + "hypothesis_novel": hypothesis_novel, }) keep = should_keep(result, baseline) print( @@ -1050,6 +1166,10 @@ def run_iteration(args, iteration: int) -> dict: "research_outcome": verdict["outcome"], "research_evidence": verdict["evidence"], "new_frontier": verdict["new_frontier"], + "created_obligation_ids": json.dumps( + verdict["created_obligation_ids"], + ), + "strategy_mode": strategy_mode, "transcript_path": str(transcript_path), } append_result(results_path, row) @@ -1094,6 +1214,7 @@ def run_iteration(args, iteration: int) -> dict: "report_path": str(report_path), "hypothesis_sha256": hypothesis_sha256, "research_outcome": "EVALUATION_FAILED", + "strategy_mode": strategy_mode, "transcript_path": str(transcript_path), "error": f"{type(exc).__name__}: {exc}", } @@ -1117,7 +1238,20 @@ def main() -> int: parser.add_argument( "--strategy-max-prefill-tokens", type=int, - default=8192, + default=8448, + ) + parser.add_argument( + "--strategy-stagnation-rounds", + type=int, + default=3, + ) + parser.add_argument("--force-strategy", action="store_true") + parser.add_argument( + "--strategy-trigger-file", + default=str( + Path.home() + / ".kakeya/autoresearch/request_strategy" + ), ) parser.add_argument( "--tokenizer-id", @@ -1148,6 +1282,8 @@ def main() -> int: raise SystemExit("iterations must be > 0") if args.strategy_max_prefill_tokens <= 0: raise SystemExit("strategy-max-prefill-tokens must be > 0") + if args.strategy_stagnation_rounds <= 0: + raise SystemExit("strategy-stagnation-rounds must be > 0") for iteration in range(args.iterations): row = run_iteration(args, iteration) print(json.dumps(row, indent=2, sort_keys=True)) diff --git a/tests/inference_engine/bench/test_autoresearch_supervisor.py b/tests/inference_engine/bench/test_autoresearch_supervisor.py index 838ee55..5457d68 100644 --- a/tests/inference_engine/bench/test_autoresearch_supervisor.py +++ b/tests/inference_engine/bench/test_autoresearch_supervisor.py @@ -1,6 +1,7 @@ from autoresearch.prefill.supervisor import ( append_result, best_kept, + build_host_candidate, build_strategy_research_state, check_runtime_health, parse_research_verdict, @@ -9,6 +10,7 @@ render_candidate, should_keep, StrategyPrefillHeartbeat, + strategy_trigger_reason, _extract_json, _pending_leaf_ids, validate_candidate, @@ -225,10 +227,19 @@ def test_keep_requires_novel_mathematical_advancement(): assert should_keep({ "accepted": True, "research_outcome": "DECOMPOSED", + "created_obligation_ids": ["RH-C1-child"], "hypothesis_novel": True, "proof_obligations_unresolved": 5, "metric_cold_critic_prefill_s": 900, }, baseline) + assert not should_keep({ + "accepted": True, + "research_outcome": "DECOMPOSED", + "created_obligation_ids": [], + "hypothesis_novel": True, + "proof_obligations_unresolved": 5, + "metric_cold_critic_prefill_s": 1, + }, baseline) assert not should_keep({ "accepted": True, "research_outcome": "INCONCLUSIVE", @@ -282,6 +293,70 @@ def test_pending_leaf_ids_excludes_unresolved_parents(): assert _pending_leaf_ids(ledger) == ["RH-C1-child", "RH-C2"] +def test_host_candidate_targets_deepest_current_branch_leaf(): + current = {**_candidate(), "target_obligation_id": "RH-C2"} + ledger = {"obligations": [ + { + "obligation_id": "RH-C1", + "statement": "Unrelated leaf.", + "status": "UNRESOLVED", + "parent_id": "", + }, + { + "obligation_id": "RH-C2", + "statement": "Root convergence claim.", + "status": "UNRESOLVED", + "parent_id": "", + }, + { + "obligation_id": "RH-C2-child", + "statement": "Construct a compact convergence counterexample.", + "status": "UNRESOLVED", + "parent_id": "RH-C2", + "last_evidence": "The previous approximation failed at a pole.", + }, + ]} + candidate = build_host_candidate(current, ledger) + assert candidate["target_obligation_id"] == "RH-C2-child" + assert candidate["hypothesis"] == ( + "Construct a compact convergence counterexample." + ) + assert "do not rename" in candidate["generator_directive"] + assert candidate["prefill_compute_chunk_tokens"] == 256 + + +def test_strategy_is_triggered_only_by_events(tmp_path): + progress = { + "kept": "True", + "research_outcome": "DECOMPOSED", + "created_obligation_ids": '["child"]', + } + inconclusive = { + "kept": "False", + "research_outcome": "INCONCLUSIVE", + "created_obligation_ids": "[]", + } + assert strategy_trigger_reason( + [progress, inconclusive, inconclusive], + stagnation_rounds=3, + ) == "" + assert strategy_trigger_reason( + [progress, inconclusive, inconclusive, inconclusive], + stagnation_rounds=3, + ) == "stagnation-3" + assert strategy_trigger_reason( + [{"kept": "True", "research_outcome": "FALSIFIED"}], + stagnation_rounds=3, + ) == "branch-falsified" + trigger = tmp_path / "request_strategy" + trigger.write_text("replan") + assert strategy_trigger_reason( + [], + stagnation_rounds=3, + trigger_file=trigger, + ) == "manual-trigger-file" + + def test_strategy_state_keeps_complete_active_ancestry_only(): ledger = {"obligations": [ { @@ -461,11 +536,12 @@ def test_supervisor_preserves_runtime_and_cache_across_iterations(): / "supervisor.py" ).read_text() body = source[source.index("def run_iteration"):source.index("def main")] - assert body.index("check_runtime_health(") < ( - body.index("proposed = propose_candidate") + assert body.index("check_runtime_health(") < body.index( + "strategy_trigger_reason(", ) assert "phase=runtime-health-check" in body - assert "phase=strategy-proposal real-gemma" in body + assert "phase=deterministic-candidate" in body + assert "mode=gemma trigger=" in body assert "if not gan_completed:" in body assert "phase=completed-run-preserved" in body assert "deploy_candidate" not in source